def train_classifier() -> tf.keras.Model: model = get_classification_model() model = compile_model(model) train, val, test = create_datasets() train = configure_dataset_for_performance(train, BATCH_SIZE) val = configure_dataset_for_performance(val, BATCH_SIZE) test = configure_dataset_for_performance(test, BATCH_SIZE) log_dir = "logs/fit/" + datetime.now().strftime("%Y%m%d-%H%M%S") callbacks = [ EarlyStopping(monitor="val_loss", patience=5, restore_best_weights=True), TerminateOnNaN(), TBCallback(log_dir=log_dir, histogram_freq=1), PlotLossesKeras(), ] fit_kwargs = dict( epochs=20, shuffle=True, callbacks=callbacks, ) history = model.fit(train, validation_data=val, **fit_kwargs) plot_history(history.history) scores = model.evaluate(test) metric_dict = dict(zip(model.metrics_names, scores)) print("Test metrics:", metric_dict) return model
def test_customization(): plot_history(pd.read_csv("tests/big_history.csv", index_col=0), path="plots/normal.png", customization_callback=callami) plt.close() plot_history("tests/big_history.csv") plt.close() plot_history(["tests/big_history.csv", "tests/big_history.csv"]) plt.close() with pytest.raises(ValueError): plot_history(78)
def test_plot_small_history(): plot_history(pd.read_json("tests/small_history.json"), path="plots/small_history.png") plt.close() assert os.path.exists("plots/small_history.png") plot_history("tests/small_history.json") plt.close() plot_history(["tests/small_history.json", "tests/small_history.json"]) plt.close() plot_history(compress_json.load("tests/small_history.json")) plt.close()
def test_plot_multi_history(): plot_history([ pd.read_csv("tests/history1.csv"), pd.read_csv("tests/history2.csv"), pd.read_csv("tests/history4.csv"), pd.read_csv("tests/history3.csv") ], path="plots/multiple.png", interpolate=True, max_epochs="min") plt.close() plot_history([ pd.read_csv("tests/history1.csv"), pd.read_csv("tests/history2.csv"), pd.read_csv("tests/history4.csv"), pd.read_csv("tests/history3.csv") ], path="plots/multiple.png", interpolate=True, max_epochs="max") plt.close() assert os.path.exists("plots/multiple.png")
def test_plot(): plot_history(pd.read_csv("tests/big_history.csv", index_col=0)[:16], path="plots/interpolated.png", interpolate=True) plot_history(pd.read_csv("tests/big_history.csv", index_col=0)[:2], path="plots/interpolated.png", interpolate=True) plot_history(pd.read_csv("tests/big_history.csv", index_col=0), path="plots/interpolated.png", interpolate=True) plt.close() assert os.path.exists("plots/interpolated.png")
def evaluate_model_TCN(x_tr, y_tr, x_te, y_te, gamma=2, epochs=200, verbose=0, plot=0, single_run=0): """Training function, to evaluate train set against test set or train set againts validation set Arguments: (x_tr, y_tr) training data (x_te, y_te) testing/validation data gamma: focal loss parameter epochs= number of times, it passes through the training dataset. verbose: if true, print all the metrics plot: if true, print built in plot single_run: fix random seed to ensure reproducibility Returns: loss: Last binary focal loss value on test/validation set accuracy: accuracy on test/validation set wf1: weighted F1 score wf1_: custom weighted F1 score (with proportional weights) mf1: macro F1 score F1_tab: F1 score per label Ptab: precision per label Rtab: recall per label""" batch_size = 1 w = class_weights(y_tr) clear_session() if single_run: np.random.seed(123) python_random.seed(123) tf.random.set_seed(1234) #-------------------------------------model definition---------------------------# #Creation TCN object Tcn = TCN(nb_filters=64, kernel_size=2, nb_stacks=1, dilations=(8, 16, 32, 64, 128, 256, 512, 1024), return_sequences=True, activation=LeakyReLU(0.01), kernel_initializer='he_normal') i = Input(batch_shape=(1, None, x_tr.shape[2])) o = Tcn(i) o = Dense(200, activation=LeakyReLU(0.01), kernel_regularizer=keras.regularizers.l1_l2(0.00001))(o) o = Dense(8, activation='sigmoid')(o) model = Model(inputs=[i], outputs=[o]) model.compile(optimizer='adam', loss=BinaryFocalLoss(gamma), metrics=[ BinaryAccuracy(), Precision(), Recall(), FalseNegatives(), FalsePositives() ]) model.summary() #---------------------fit network---------------------------------------------# hist = model.fit(x_tr, y_tr, epochs=epochs, batch_size=batch_size, verbose=0, validation_data=(x_te, y_te)) #---------------------------------evaluate model----------------------------------------------# #evaluate model on test set (over all classes) loss, accuracy, P, R, FN, FP = model.evaluate(x_te, y_te, batch_size=batch_size, verbose=verbose) #save model if single_run: model.save('Results/opt_TCN_model') print("Model saved to Results") y_pred = model.predict(x_te, batch_size=batch_size, verbose=0) y_pred[y_pred < 0.5] = 0. y_pred[y_pred > 0.5] = 1. y_pred = reshape(y_pred, (y_pred.shape[0] * y_pred.shape[1], 8)) y_te = reshape(y_te, (y_te.shape[0] * y_te.shape[1], 8)) #evaluate F1 score for each label F1_tab, Ptab, Rtab, wf1_ = F1_score(y_te, y_pred, w) #evaluate accuracy per label acc_tab = Acc(y_te, y_pred) print("-> F1 score per label: ", F1_tab) print("-> y_pred ", y_pred[:, 4]) #test f1 score built in f = F1Score(8, threshold=0.5, average='weighted') f.update_state(y_te, y_pred) wf1 = f.result().numpy() print("weighted F1 score built in: ", wf1) f.reset_states() f = F1Score(8, threshold=0.5, average='macro') f.update_state(y_te, y_pred) mf1 = f.result().numpy() print("macro F1 score built in: ", mf1) f.reset_states() #-----------------------------------------print---------------------------------------------# #print all if verbose: print(" -> Accuracy: ", accuracy, "; Mean of labelwise accuracy: ", np.mean(acc_tab)) print("Per label accuracy: ", acc_tab) print("-> Weighted F1 score: ", wf1_) print("-> F1 score per label: ", F1_tab) print("-> Precision: ", P, "; Recall: ", R) print("-> Precision per label: ", Ptab) print("-> Recall per label: ", Rtab) print("-> Loss: ", loss) if plot: plot_history(hist.history) return hist, loss, accuracy, wf1, wf1_, mf1, F1_tab, Ptab, Rtab
def evaluate_model(x_tr, y_tr, x_te, y_te, model_type=1, gamma=2, nodes_nb=600, drop=0.1, epochs=200, reg=0, verbose=0, plot=0, single_run=0): """Training function, to evaluate train set against test set or train set againts validation set Arguments: (x_tr, y_tr) training data (x_te, y_te) testing/validation data model_type: type of model to train/evaluate (on LSTM layers) 0: 1 LSTM layer model 1: 1 bidirectional LSTM layer model 2: 2 bidirectional LSTM layer model gamma: focal loss parameter nodes_nb: number of neurons in the LSTM layers drop: dropout value verbose: if true, print all the metrics plot: if true, print built in plot single_run: fix random seed to ensure reproducibility Returns: loss: Last binary focal loss value on test/validation set accuracy: accuracy on test/validation set wf1: weighted F1 score wf1_: custom weighted F1 score (with proportional weights) mf1: macro F1 score F1_tab: F1 score per label Ptab: precision per label Rtab: recall per label""" batch_size = 32 n_features, n_outputs = x_tr.shape[2], y_tr.shape[2] w = class_weights(y_tr) clear_session() if single_run: np.random.seed(2020) python_random.seed(2020) tf.random.set_seed(2020) #-------------------------------------model definition-------------------------------------# model = Sequential() #model types if model_type == 0: model.add( LSTM(nodes_nb, input_shape=(None, n_features), return_sequences=True)) model.add(Dropout(drop)) model.add(Dense(n_outputs, activation='sigmoid')) if model_type == 1: model.add( Bidirectional( LSTM(nodes_nb, input_shape=(None, n_features), return_sequences=True))) model.add(Dropout(drop)) model.add(Dense(n_outputs, activation='sigmoid')) if model_type == 2: model.add( Bidirectional( LSTM(nodes_nb, input_shape=(None, n_features), return_sequences=True))) model.add(Dropout(drop)) model.add( Bidirectional( LSTM(nodes_nb, input_shape=(None, n_features), return_sequences=True))) model.add(Dropout(drop)) model.add(Dense(n_outputs, activation='sigmoid')) model.compile(loss=BinaryFocalLoss(gamma), optimizer='adam', metrics=[ BinaryAccuracy(), Precision(), Recall(), FalseNegatives(), FalsePositives() ]) #------------------------------------fit network---------------------------------------------# hist = model.fit(x_tr, y_tr, epochs=epochs, batch_size=batch_size, verbose=0, validation_data=(x_te, y_te)) if verbose: model.summary() #---------------------------------evaluate model----------------------------------------------# #evaluate model on test set (over all classes) loss, accuracy, P, R, FN, FP = model.evaluate(x_te, y_te, batch_size=batch_size, verbose=verbose) #save model if single_run: model.save('Results/opt_LSTM_model') print("Model saved to Results") y_pred = model.predict(x_te, batch_size=batch_size, verbose=0) y_pred = reshape(y_pred, (y_pred.shape[0] * y_pred.shape[1], 8)) y_te = reshape(y_te, (y_te.shape[0] * y_te.shape[1], 8)) #evaluate F1 score for each label F1_tab, Ptab, Rtab, wf1_ = F1_score(y_te, y_pred, w) #evaluate accuracy per label acc_tab = Acc(y_te, y_pred) #test f1 score built in f = F1Score(8, threshold=0.5, average='weighted') f.update_state(y_te, y_pred) wf1 = f.result().numpy() f.reset_states() f = F1Score(8, threshold=0.5, average='macro') f.update_state(y_te, y_pred) mf1 = f.result().numpy() f.reset_states() #-----------------------------------------print---------------------------------------------# #print all if verbose: print(" -> Accuracy: ", accuracy, "; Mean of labelwise accuracy: ", np.mean(acc_tab)) print("Per label accuracy: ", acc_tab) print("-> Proportional F1 score: ", wf1_, "; Weighted F1 score: ", wf1, "; Macro F1 score: ", mf1) print("-> F1 score per label: ", F1_tab) print("-> Precision: ", P, "; Recall: ", R) print("-> Precision per label: ", Ptab) print("-> Recall per label: ", Rtab) print("-> Loss: ", loss) if plot: plot_history(hist.history) return hist, loss, accuracy, wf1, wf1_, mf1, F1_tab, Ptab, Rtab
def plot(self, *args, **kwargs): plot_history(self.to_dataframe()[["score"]], *args, **kwargs)
gen_test_data() preprocessed_data() # 3. Load all dataset & normalize X_train, y_train = load_audio_datafiles(conf, conf.X_train, conf.y_train, normalize=True) X_test, y_test = load_audio_datafiles(conf, conf.X_test, conf.y_test, normalize=True) print('Loaded train:test = {}:{} samples.'.format(len(X_train), len(X_test))) # 4. Train folds history, model = train_classifier(conf, fold=0, dataset=[X_train, y_train, X_test, y_test], model=None, show_detail=False, # init_weights=None, # from scratch init_weights=conf.best_weight_file, # from scratch ) # 5. Evaluate evaluate_model(conf, model, X_test, y_test) print('___ training finished ___') fn = glob('weights/*.h5')[-1] shutil.copy(fn, conf.best_weight_file) os.system("python convert_keras_to_tf.py --model_type {} --keras_weight {} --out_prefix {}".format(conf.model, conf.best_weight_file, os.path.splitext(conf.best_weight_file)[0])) print('___ trasfer xxx.h5 to xxx.pb finished ___') plot_history(history.history, path="history.png") plt.close()
def test_plot_singletons(): os.makedirs("plots/singletons", exist_ok=True) plot_history(pd.read_json("tests/history.json"), path="plots/singletons", single_graphs=True) plt.close()
def plot_model(model): plot_history(history.history) plt.show()
def kerasNet(x_data, y_data, NUNITS_INPUT = 64, NUNITS = 32, NHIDDEN = 2, lr = 1e-3, EPOCHS = 100, BATCHSIZE = 32, validation_split = 0.3, VERBOSE = 2, optimizer:str = "sgd", loss = ['mean_squared_error'], use_gpu:bool = True, saveModel: bool = False, plot_results:bool = True, baseline:bool = False ): """ 2 hidden layers, sigmoid tanh """ if use_gpu: try: if is_gpu_available(): print("Using gpu!") except: print("No GPU") model = Sequential() model.add(Dense(NUNITS_INPUT, input_dim=(x_data.shape[1]), activation = "relu", name="First")) for _ in range(NHIDDEN): if baseline: model.name = "Baseline" model.add(Dense(NUNITS, activation = "tanh" )) else: model.name = "Network1" model.add(Dense(NUNITS, activation = "tanh", kernel_initializer = 'random_uniform', #bias_initializer = 'random_normal', #kernel_regularizer=regularizers.l2(0.01), #activity_regularizer=regularizers.l1(0.01) )) model.add(Dense(y_data.shape[1], activation = 'linear', name = "Final")) model.compile(loss=loss, optimizer=optimizer, metrics = ['mse',rmse, max_error, 'mae'] ) print("X_Train : ", x_data.shape," and Y_Train: ", y_data.shape) history = model.fit(x_data, y_data, validation_split = validation_split, epochs = EPOCHS, batch_size= BATCHSIZE, verbose = VERBOSE ).history print(model.summary()) plot_history(history) if saveModel: model.save(name + '.h5') print("Saved Model") return model
model.prepare_valid_positions(X_val[3])) hist = model.fit(X_train, Y_train, validation_data=(X_val, Y_val), epochs=1, batch_size=batch_size) if history: history = { key: history[key] + hist.history[key] for key in hist.history } else: history = hist.history plot_history(history) plt.show() plt.close() print('Saving ..') if not os.path.exists(save_folder_path): os.makedirs(save_folder_path) print('Folder `%s` created' % save_folder_path) model.save(save_folder_path) with open(os.path.join(save_folder_path, 'tags_vectorizer.pkl'), 'wb') as handle: pickle.dump(tags_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL) with open(os.path.join(save_folder_path, 'intents_label_encoder.pkl'), 'wb') as handle: pickle.dump(intents_label_encoder, handle,
def train_model(train_config_path, sess): logging.basicConfig(stream=sys.stdout, format='%(message)s', level=logging.WARNING) with open(os.path.join(train_config_path, 'train_config.json'), 'r') as json_file: train_config = json.load(json_file) data_folder_path = os.path.join(train_config['data_folder_path'], 'train') save_folder_path = train_config['save_folder_path'] epochs = train_config['epochs'] batch_size = train_config['batch_size'] num_bert_fine_tune_layers = train_config['num_bert_fine_tune_layers'] model_hub_path = train_config['model_hub_path'] logging.log(logging.WARNING, 'Reading data ...') text_arr, tags_arr, intents = JointBertModel.read_goo(data_folder_path) logging.log(logging.WARNING, 'Vectorize data ...') bert_vectorizer = BERTVectorizer(sess, model_hub_path) input_ids, input_mask, segment_ids, valid_positions, sequence_lengths = bert_vectorizer.transform( text_arr) logging.log(logging.WARNING, 'Vectorize tags ...') tags_vectorizer = TagsVectorizer() tags_vectorizer.fit(tags_arr) tags = tags_vectorizer.transform(tags_arr, valid_positions) slots_num = len(tags_vectorizer.label_encoder.classes_) logging.log(logging.WARNING, 'Encoding labels ...') intents_label_encoder = LabelEncoder() intents = intents_label_encoder.fit_transform(intents).astype(np.int32) intents_num = len(intents_label_encoder.classes_) model = JointBertModel(slots_num, intents_num, model_hub_path, sess, num_bert_fine_tune_layers) logging.log(logging.WARNING, 'Training model ...') X = np.concatenate( (input_ids, input_mask, segment_ids, valid_positions, tags), axis=1) Y = intents split_width = input_ids.shape[1] history = {} i = 1 while i <= epochs: folds = StratifiedKFold(n_splits=5, shuffle=True).split(X, Y) for train_index, val_index in folds: if i > epochs: break X_train, X_val = X[train_index], X[val_index] Y_train, Y_val = Y[train_index], Y[val_index] Y_train = [ X_train[:, 4 * split_width:5 * split_width], Y_train ] X_train = [ X_train[:, 0:split_width], X_train[:, split_width:2 * split_width], X_train[:, 2 * split_width:3 * split_width], X_train[:, 3 * split_width:4 * split_width] ] Y_val = [X_val[:, 4 * split_width:5 * split_width], Y_val] X_val = [ X_val[:, 0:split_width], X_val[:, split_width:2 * split_width], X_val[:, 2 * split_width:3 * split_width], X_val[:, 3 * split_width:4 * split_width] ] X_train = (X_train[0], X_train[1], X_train[2], model.prepare_valid_positions(X_train[3])) X_val = (X_val[0], X_val[1], X_val[2], model.prepare_valid_positions(X_val[3])) logging.log(logging.WARNING, 'Epoch %i/%i' % (i, epochs)) hist = model.fit(X_train, Y_train, validation_data=(X_val, Y_val), epochs=1, batch_size=batch_size) if history: history = { key: history[key] + hist.history[key] for key in hist.history } else: history = hist.history i += 1 plot_history(history) plt.show() plt.close() logging.log(logging.WARNING, 'Saving model ...') if not os.path.exists(save_folder_path): os.makedirs(save_folder_path) logging.info('Folder `%s` created' % save_folder_path) model.save_model(save_folder_path) with open(os.path.join(save_folder_path, 'tags_vectorizer.pkl'), 'wb') as handle: pickle.dump(tags_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL) with open(os.path.join(save_folder_path, 'intents_label_encoder.pkl'), 'wb') as handle: pickle.dump(intents_label_encoder, handle, protocol=pickle.HIGHEST_PROTOCOL) return model
def calculate_models(epochs=1, neurons=None): Path(STAT_DIR).mkdir(parents=True, exist_ok=True) Path(MODEL_DIR).mkdir(parents=True, exist_ok=True) timestamp = datetime.now().strftime("%Y_%m_%d_(%H-%M-%S)") name_modifier = '_sample_' + str(SAMPLE) if SAMPLE is not None else '_sample_None' name_modifier += '_epochs' + str(EPOCHS) if globals()['USE_NORMED']: name_modifier = '_normed_' + name_modifier model_folder = MODEL_DIR + timestamp + name_modifier + '/' Path(model_folder).mkdir(parents=True, exist_ok=True) results = list() rare_results = list() if neurons is None: min_value = to_categorical(globals()['VALIDATION_LABELS']).shape[1] max_value = len(globals()['DATA'][0])+1 else: min_value = neurons max_value = neurons + 1 for x in range(min_value, max_value): print("Training NN for " + str(x) + " hidden neurons...") clear_session() model = create_neural_network(x) history = train_model(model, epochs) model.save(model_folder + "model_" + str(x)) plot_history(history, path=model_folder + "model_" + str(x) + "/history.png", graphs_per_row=2) pyplot.close() plot_model( model, to_file=model_folder + "model_" + str(x) + "/model.png", show_shapes=True, show_layer_names=True, rankdir="TB", expand_nested=True, dpi=100, ) results.append([x] + validate_model(model)) temp = validate_model_for_all_classes(model) to_append = [x] for key in sorted(temp.keys()): to_append += temp[key] rare_results.append(to_append) output_file = model_folder + "models" + name_modifier + "_" + timestamp + '.csv' output_file = open(output_file, "w", encoding="utf-8") output_file.write('neurons;loss;accuracy;recall;precision\n') for r in results: output_file.write(";".join(list(map(str, r))) + "\n") output_file.close() rare_output_file = model_folder + "models" + name_modifier + "_rare_" + timestamp + '.csv' rare_output_file = open(rare_output_file, "w", encoding="utf-8") rare_output_file.write('neurons;loss 1;accuracy 1;recall 1;precision 1;loss 2;accuracy 2;recall 2;precision 2;' + 'loss 4;accuracy 4;recall 4;precision 4;' + 'loss 6;accuracy 6;recall 6;precision 6;loss 7;accuracy 7;recall 7;precision 7;' + 'loss 8;accuracy 8;recall 8;precision 8;loss 9;accuracy 9;recall 9;precision 9\n') for r in rare_results: rare_output_file.write(";".join(list(map(str, r))) + "\n") rare_output_file.close()
validation_data=(val_x, val_y)).history pd.DataFrame(history).to_csv("historydota2.csv") # lets assume `model` is main model #model_json = model.to_json() #with open("model_in_json.json", "w") as json_file: # json.dump(model_json, json_file) model.save("bobot_model1.h5") model.save_weights("bobot_model2.h5") #model.save_weights('weight.h5') # Predict all Validation data predict = model.predict(val_x) plot_model(model, show_shapes=True, expand_nested=True, to_file='model.png') # Visualize Prediction df = pd.DataFrame(predict) df.columns = ['Strength', 'Agility', 'Intelligent'] df.index = val_data[:, 0] print(df) plot_history(history, style='-', path='singleton', single_graphs=True, side=12, graphs_per_row=1) #plot_history(history) plt.show()
def test_plot(): plot_history(pd.read_csv("tests/big_history.csv", index_col=0), path="plots/normal.png") plt.close() assert os.path.exists("plots/normal.png")