def evaluate_attention_weights(train_words_type: str) -> None: """MAKEDOC: what is evaluate_attention_weights doing?""" logg = logging.getLogger(f"c.{__name__}.evaluate_attention_weights") # logg.setLevel("INFO") logg.debug("Start evaluate_attention_weights") # magic to fix the GPUs setup_gpus() # VAN_opa1_lr05_bs32_en15_dsaug07_wLTall hypa = { "batch_size_type": "32", "dataset_name": "aug07", "epoch_num_type": "15", "learning_rate_type": "05", "net_type": "VAN", "optimizer_type": "a1", "words_type": "LTall", } use_validation = True dataset_name = hypa["dataset_name"] batch_size = int(hypa["batch_size_type"]) # get the model name model_name = build_area_name(hypa, use_validation) logg.debug(f"model_name: {model_name}") # load the model model_folder = Path("trained_models") / "area" model_path = model_folder / f"{model_name}.h5" model = tf_models.load_model(model_path) # get the output layer because you forgot to name it name_output_layer = model.layers[-1].name logg.debug(f"name_output_layer: {name_output_layer}") # build a model on top of that to get the weights att_weight_model = tf_models.Model( inputs=model.input, outputs=[ model.get_layer(name_output_layer).output, model.get_layer("area_values").output, ], ) att_weight_model.summary() # get the training words train_words = words_types[train_words_type] perm_pred = compute_permutation(train_words) logg.debug(f"perm_pred: {perm_pred}") sorted_train_words = sorted(train_words) logg.debug(f"sorted(train_words): {sorted(train_words)}") # load data if you do not want to record new audios processed_folder = Path("data_proc") processed_path = processed_folder / f"{dataset_name}" logg.debug(f"processed_path: {processed_path}") # # evaluate on all data because im confused # data, labels = load_processed(processed_path, train_words) # logg.debug(f"data['testing'].shape: {data['testing'].shape}") # logg.debug(f"labels['testing'].shape: {labels['testing'].shape}") # eval_testing = model.evaluate(data["testing"], labels["testing"]) # for metrics_name, value in zip(model.metrics_names, eval_testing): # logg.debug(f"{metrics_name}: {value}") # which word in the dataset to plot # word_id = 5 # word_id = 7 word_id = 12 # the loaded spectrograms rec_data_l: ty.List[np.ndarray] = [] # for now we do not record new words rec_words = train_words[30:32] num_rec_words = len(rec_words) logg.debug(f"processed_path: {processed_path}") for i, word in enumerate(rec_words): logg.debug(f"\nword: {word}") data, labels = load_processed(processed_path, [word]) logg.debug(f"data['testing'].shape: {data['testing'].shape}") logg.debug(f"labels['testing'].shape: {labels['testing'].shape}") # eval_testing = model.evaluate(data["testing"], labels["testing"]) # for metrics_name, value in zip(model.metrics_names, eval_testing): # logg.debug(f"{metrics_name}: {value}") # get one of the spectrograms word_data = data["testing"][word_id] rec_data_l.append(word_data) pred, att_weights = att_weight_model.predict(data["testing"]) logg.debug(f"pred.shape: {pred.shape}") logg.debug(f"pred[0].shape: {pred[0].shape}") pred_am_all = np.argmax(pred, axis=1) logg.debug(f"pred_am_all: {pred_am_all}") pred_index = np.argmax(pred[0]) pred_word = sorted_train_words[pred_index] logg.debug(f"sorted pred_word: {pred_word} pred_index {pred_index}") # test EVERY SINGLE spectrogram spec_num = data["testing"].shape[0] for wid in range(spec_num): # get the word word_data = data["testing"][wid] logg.debug(f"word_data.shape: {word_data.shape}") batch_word_data = np.expand_dims(word_data, axis=0) logg.debug(f"batch_word_data.shape: {batch_word_data.shape}") shape_batch = (batch_size, *word_data.shape) logg.debug(f"shape_batch: {shape_batch}") batch_word_data_big = np.zeros(shape_batch, dtype=np.float32) for i in range(batch_size): batch_word_data_big[i, :, :, :] = batch_word_data # batch_word_data_big[0, :, :, :] = batch_word_data # predict it # pred, att_weights = att_weight_model.predict(batch_word_data) pred, att_weights = att_weight_model.predict(batch_word_data_big) # show all prediction # pred_am = np.argmax(pred, axis=1) # logg.debug(f"pred_am: {pred_am}") # focus on first prediction word_pred = pred[0] pred_index = np.argmax(word_pred) pred_word = sorted_train_words[pred_index] recap = "" if pred_word == word: recap += "correct " else: recap += " wrong " pred_am = np.argmax(pred, axis=1) logg.debug(f"pred_am: {pred_am}") recap += f"sorted pred_word: {pred_word} pred_index {pred_index}" recap += f" word_pred.shape {word_pred.shape}" recap += f" pred_am_all[wid] {pred_am_all[wid]}" # pred_f = ", ".join([f"{p:.3f}" for p in pred[0]]) # recap += f" pred_f: {pred_f}" logg.debug(recap) # break # turn the list into np array rec_data = np.stack(rec_data_l) logg.debug(f"\nrec_data.shape: {rec_data.shape}") # get prediction and attention weights pred, att_weights = att_weight_model.predict(rec_data) logg.debug(f"att_weights.shape: {att_weights.shape}") logg.debug(f"att_weights[0].shape: {att_weights[0].shape}") plot_size = 5 fw = plot_size * num_rec_words nrows = 2 fh = plot_size * nrows fig, axes = plt.subplots(nrows=nrows, ncols=num_rec_words, figsize=(fw, fh)) fig.suptitle("Attention weights computed with VerticalAreaNet", fontsize=20) for i, word in enumerate(rec_words): logg.debug(f"recword: {word}") # show the spectrogram word_spec = rec_data[i][:, :, 0] # logg.debug(f"word_spec: {word_spec}") axes[0][i].set_title(f"Spectrogram for {word}", fontsize=20) axes[0][i].imshow(word_spec, origin="lower") axes[1][i].set_title(f"Attention weights for {word}", fontsize=20) att_w = att_weights[i][:, :, 0] axes[1][i].imshow(att_w, origin="lower") logg.debug(f"att_w.max(): {att_w.max()}") # axes[0][i].imshow( # att_w, origin="lower", extent=img.get_extent(), cmap="gray", alpha=0.4 # ) # weighted = word_spec * att_w # axes[2][i].imshow(weighted, origin="lower") word_pred = pred[i] pred_index = np.argmax(word_pred) pred_word = sorted_train_words[pred_index] logg.debug(f"sorted pred_word: {pred_word} pred_index {pred_index}") # # plot the predictions word_pred = pred[i] # logg.debug(f"word_pred: {word_pred}") # # permute the prediction from sorted to the order you have word_pred = word_pred[perm_pred] # logg.debug(f"word_pred permuted: {word_pred}") pred_index = np.argmax(word_pred) pred_word = train_words[pred_index] logg.debug(f"pred_word: {pred_word} pred_index {pred_index}") # title = f"Predictions for {word}" # plot_pred(word_pred, train_words, axes[2][i], title, pred_index) fig.tight_layout() fig_name = f"{model_name}" fig_name += "_0002.{}" plot_folder = Path("plot_results") results_path = plot_folder / fig_name.format("pdf") fig.savefig(results_path) plt.show()
def evaluate_model_area(model_name: str, test_words_type: str) -> None: r"""MAKEDOC: what is evaluate_model_area doing?""" logg = logging.getLogger(f"c.{__name__}.evaluate_model_area") # logg.setLevel("INFO") logg.debug("Start evaluate_model_area") # magic to fix the GPUs setup_gpus() # # VAN_opa1_lr05_bs32_en15_dsaug07_wLTall # hypa = { # "batch_size_type": "32", # "dataset_name": "aug07", # "epoch_num_type": "15", # "learning_rate_type": "03", # "net_type": "VAN", # "optimizer_type": "a1", # # "words_type": "LTall", # "words_type": train_words_type, # } # # use_validation = True # use_validation = False # dataset_name = hypa["dataset_name"] # get the model name # model_name = build_area_name(hypa, use_validation) logg.debug(f"model_name: {model_name}") dataset_re = re.compile("_ds(.*?)_") match = dataset_re.search(model_name) if match is not None: logg.debug(f"match[1]: {match[1]}") dataset_name = match[1] train_words_type_re = re.compile("_w(.*?)[_.]") match = train_words_type_re.search(model_name) if match is not None: logg.debug(f"match[1]: {match[1]}") train_words_type = match[1] # load the model model_folder = Path("trained_models") / "area" model_path = model_folder / f"{model_name}.h5" model = tf_models.load_model(model_path) # model.summary() train_words = words_types[train_words_type] logg.debug(f"train_words: {train_words}") test_words = words_types[test_words_type] logg.debug(f"test_words: {test_words}") # input data processed_path = Path("data_proc") / f"{dataset_name}" data, labels = load_processed(processed_path, test_words) logg.debug(f"list(data.keys()): {list(data.keys())}") logg.debug(f"data['testing'].shape: {data['testing'].shape}") # evaluate on the words you trained on logg.debug("Evaluate on test data:") model.evaluate(data["testing"], labels["testing"]) # model.evaluate(data["validation"], labels["validation"]) # predict labels/cm/fscore y_pred = model.predict(data["testing"]) cm = pred_hot_2_cm(labels["testing"], y_pred, test_words) # y_pred = model.predict(data["validation"]) # cm = pred_hot_2_cm(labels["validation"], y_pred, test_words) fscore = analyze_confusion(cm, test_words) logg.debug(f"fscore: {fscore}") fig, ax = plt.subplots(figsize=(12, 12)) plot_confusion_matrix(cm, ax, model_name, test_words, fscore, train_words) fig_name = f"{model_name}_test{test_words_type}_cm.{{}}" cm_folder = Path("plot_results") / "cm" if not cm_folder.exists(): cm_folder.mkdir(parents=True, exist_ok=True) plot_cm_path = cm_folder / fig_name.format("png") fig.savefig(plot_cm_path) plot_cm_path = cm_folder / fig_name.format("pdf") fig.savefig(plot_cm_path) plt.show()
def recompute_fscore_cnn() -> None: """MAKEDOC: what is recompute_fscore_cnn doing?""" logg = logging.getLogger(f"c.{__name__}.recompute_fscore_cnn") # logg.setLevel("INFO") logg.debug("Start recompute_fscore_cnn") info_folder = Path("info") trained_folder = Path("trained_models") for model_folder in info_folder.iterdir(): # logg.debug(f"model_folder: {model_folder}") # check that it is a CNN model_name = model_folder.name if not model_name.startswith("CNN"): continue # check that the model is trained and not a placeholder model_path = trained_folder / f"{model_name}.h5" found_model = False if model_path.exists(): if model_path.stat().st_size > 100: found_model = True if not found_model: continue # load it model = models.load_model(model_path) res_recap_path = model_folder / "results_recap.json" if not res_recap_path.exists(): continue results_recap = json.loads(res_recap_path.read_text()) # logg.debug(f"results_recap['cm']: {results_recap['cm']}") recap_path = model_folder / "recap.json" recap = json.loads(recap_path.read_text()) # logg.debug(f"recap['words']: {recap['words']}") words = recap["words"] hypa = recap["hypa"] # check that the data is available dn = hypa["dataset"] wt = hypa["words"] if dn.startswith("mel") or dn.startswith("mfcc"): preprocess_spec(dn, wt) elif dn.startswith("aug"): do_augmentation(dn, wt) processed_path = Path("data_proc") / f"{hypa['dataset']}" data, labels = load_processed(processed_path, words) y_pred = model.predict(data["testing"]) cm = pred_hot_2_cm(labels["testing"], y_pred, words) fscore = analyze_confusion(cm, words) # logg.debug(f"fscore: {fscore}") # overwrite the cm results_recap["cm"] = cm.tolist() # add the fscore results_recap["fscore"] = fscore # increase the version results_recap["results_recap_version"] = "002" # write the new results res_recap_path.write_text(json.dumps(results_recap, indent=4)) # increase the recap version (shows that it is after this debacle) recap["version"] = "002" recap_path.write_text(json.dumps(recap, indent=4)) # save the new plots fig, ax = plt.subplots(figsize=(12, 12)) plot_confusion_matrix(cm, ax, model_name, words, fscore) plot_cm_path = info_folder / "test_confusion_matrix.png" fig.savefig(plot_cm_path) plt.close(fig)
def train_model(hypa, force_retrain): """MAKEDOC: What is train_model doing?""" logg = logging.getLogger(f"c.{__name__}.train_model") # logg.debug("Starting train_model") # get the words words = words_types[hypa["words"]] # name the model model_name = build_cnn_name(hypa) logg.debug(f"model_name: {model_name}") # save the trained model here model_folder = Path("trained_models") / "cnn" if not model_folder.exists(): model_folder.mkdir(parents=True, exist_ok=True) model_path = model_folder / f"{model_name}.h5" # logg.debug(f"model_path: {model_path}") placeholder_path = model_folder / f"{model_name}.txt" # check if this model has already been trained if placeholder_path.exists(): if force_retrain: logg.warn("\nRETRAINING MODEL!!\n") else: logg.debug("Already trained") return # save info regarding the model training in this folder info_folder = Path("info") / "cnn" / model_name if not info_folder.exists(): info_folder.mkdir(parents=True, exist_ok=True) # magic to fix the GPUs setup_gpus() # input data processed_path = Path("data_proc") / f"{hypa['dataset']}" data, labels = load_processed(processed_path, words) # from hypa extract model param model_param = {} model_param["num_labels"] = len(words) model_param["input_shape"] = data["training"][0].shape model_param["base_filters"] = hypa["base_filters"] model_param["base_dense_width"] = hypa["base_dense_width"] # translate types to actual values kernel_size_types = { "01": [(2, 2), (2, 2), (2, 2)], "02": [(5, 1), (3, 3), (3, 3)], "03": [(1, 5), (3, 3), (3, 3)], } model_param["kernel_sizes"] = kernel_size_types[hypa["kernel_size_type"]] pool_size_types = { "01": [(2, 2), (2, 2), (2, 2)], "02": [(2, 1), (2, 2), (2, 2)], "03": [(1, 2), (2, 2), (2, 2)], } model_param["pool_sizes"] = pool_size_types[hypa["pool_size_type"]] dropout_types = {"01": [0.03, 0.01], "02": [0.3, 0.1]} model_param["dropouts"] = dropout_types[hypa["dropout_type"]] # a dict to recreate this training recap = {} recap["words"] = words recap["hypa"] = hypa recap["model_param"] = model_param recap["model_name"] = model_name recap["version"] = "002" # logg.debug(f"recap: {recap}") recap_path = info_folder / "recap.json" recap_path.write_text(json.dumps(recap, indent=4)) learning_rate_types = { "01": "fixed01", "02": "fixed02", "03": "fixed03", "e1": "exp_decay_keras_01", "04": "exp_decay_step_01", "05": "exp_decay_smooth_01", "06": "exp_decay_smooth_02", } learning_rate_type = hypa["learning_rate_type"] lr_value = learning_rate_types[learning_rate_type] # setup opt fixed lr values if lr_value.startswith("fixed"): if lr_value == "fixed01": lr = 1e-2 elif lr_value == "fixed02": lr = 1e-3 elif lr_value == "fixed03": lr = 1e-4 else: lr = 1e-3 if lr_value == "exp_decay_keras_01": lr = ExponentialDecay(0.1, decay_steps=100000, decay_rate=0.96, staircase=True) optimizer_types = { "a1": Adam(learning_rate=lr), "r1": RMSprop(learning_rate=lr), } opt = optimizer_types[hypa["optimizer_type"]] # create the model model = CNNmodel(**model_param) # model.summary() metrics = [ tf.keras.metrics.CategoricalAccuracy(), tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), ] model.compile( optimizer=opt, loss=tf.keras.losses.CategoricalCrossentropy(), metrics=metrics, ) # setup callbacks callbacks = [] # setup exp decay step / smooth if lr_value.startswith("exp_decay"): if lr_value == "exp_decay_step_01": exp_decay_part = partial(exp_decay_step, epochs_drop=5) elif lr_value == "exp_decay_smooth_01": exp_decay_part = partial(exp_decay_smooth, epochs_drop=5) elif lr_value == "exp_decay_smooth_02": exp_decay_part = partial( exp_decay_smooth, epochs_drop=5, initial_lrate=1e-2 ) lrate = LearningRateScheduler(exp_decay_part) callbacks.append(lrate) # # setup early stopping # early_stop = EarlyStopping( # # monitor="val_categorical_accuracy", # monitor="val_loss", # patience=4, # verbose=1, # restore_best_weights=True, # ) # callbacks.append(early_stop) # get training parameters BATCH_SIZE = hypa["batch_size"] SHUFFLE_BUFFER_SIZE = BATCH_SIZE EPOCH_NUM = hypa["epoch_num"] # load the datasets datasets = {} for which in ["training", "validation", "testing"]: # logg.debug(f"data[{which}].shape: {data[which].shape}") datasets[which] = Dataset.from_tensor_slices((data[which], labels[which])) # logg.debug(f"datasets[{which}]: {datasets[which]}") datasets[which] = datasets[which].shuffle(SHUFFLE_BUFFER_SIZE).batch(BATCH_SIZE) # logg.debug(f"datasets[{which}]: {datasets[which]}") # train the model results = model.fit( data["training"], labels["training"], # validation_data=datasets["validation"], validation_data=(data["validation"], labels["validation"]), batch_size=BATCH_SIZE, epochs=EPOCH_NUM, verbose=1, callbacks=callbacks, ) # save the trained model model.save(model_path) results_recap = {} results_recap["model_name"] = model_name # version of the results saved results_recap["results_recap_version"] = "002" # quickly evaluate the results # logg.debug(f"\nmodel.metrics_names: {model.metrics_names}") # for which in ["training", "validation", "testing"]: # model_eval = model.evaluate(datasets[which]) # logg.debug(f"{which}: model_eval: {model_eval}") # save the evaluation results logg.debug("Evaluate on test data:") # eval_testing = model.evaluate(datasets["testing"]) # results_recap[model.metrics_names[0]] = eval_testing[0] # results_recap[model.metrics_names[1]] = eval_testing[1] eval_testing = model.evaluate(data["testing"], labels["testing"]) for metrics_name, value in zip(model.metrics_names, eval_testing): logg.debug(f"{metrics_name}: {value}") results_recap[metrics_name] = value # compute the confusion matrix # y_pred = model.predict(datasets["testing"]) y_pred = model.predict(data["testing"]) cm = pred_hot_2_cm(labels["testing"], y_pred, words) # logg.debug(f"cm: {cm}") results_recap["cm"] = cm.tolist() # compute the fscore fscore = analyze_confusion(cm, words) logg.debug(f"fscore: {fscore}") # plot the cm fig, ax = plt.subplots(figsize=(12, 12)) plot_confusion_matrix(cm, ax, model_name, words, fscore) plot_cm_path = info_folder / "test_confusion_matrix.png" fig.savefig(plot_cm_path) plt.close(fig) # save the histories results_recap["history"] = { "loss": results.history["loss"], "val_loss": results.history["val_loss"], "categorical_accuracy": results.history["categorical_accuracy"], "val_categorical_accuracy": results.history["val_categorical_accuracy"], } # save the results res_recap_path = info_folder / "results_recap.json" res_recap_path.write_text(json.dumps(results_recap, indent=4)) y_pred_dataset = model.predict(datasets["testing"]) cm_dataset = pred_hot_2_cm(labels["testing"], y_pred_dataset, words) fscore_dataset = analyze_confusion(cm_dataset, words) logg.debug(f"fscore_dataset: {fscore_dataset} fscore {fscore}") # for i, (ys, yd) in enumerate(zip(y_pred, y_pred_dataset)): # pred_split = np.argmax(ys) # pred_dataset = np.argmax(yd) # logg.debug(f"i: {i} pred_split: {pred_split} pred_dataset: {pred_dataset}") # plt.show() placeholder_path.write_text(f"Trained. F-score: {fscore}") return "done_training"
def train_attention(hypa: ty.Dict[str, str], force_retrain: bool, use_validation: bool) -> None: """MAKEDOC: what is train_attention doing?""" logg = logging.getLogger(f"c.{__name__}.train_attention") # logg.setLevel("INFO") logg.debug("Start train_attention") # build the model name model_name = build_attention_name(hypa, use_validation) logg.debug(f"model_name: {model_name}") # save the trained model here model_folder = Path("trained_models") / "attention" if not model_folder.exists(): model_folder.mkdir(parents=True, exist_ok=True) model_path = model_folder / f"{model_name}.h5" placeholder_path = model_folder / f"{model_name}.txt" # check if this model has already been trained if placeholder_path.exists(): if force_retrain: logg.warn("\nRETRAINING MODEL!!\n") else: logg.debug("Already trained") return # save info regarding the model training in this folder info_folder = Path("info") / "attention" / model_name if not info_folder.exists(): info_folder.mkdir(parents=True, exist_ok=True) # get the word list words = words_types[hypa["words_type"]] num_labels = len(words) # load data processed_folder = Path("data_proc") processed_path = processed_folder / f"{hypa['dataset_name']}" data, labels = load_processed(processed_path, words) # concatenate train and val for final train val_data = None if use_validation: x = data["training"] y = labels["training"] val_data = (data["validation"], labels["validation"]) logg.debug("Using validation data") else: x = np.concatenate((data["training"], data["validation"])) y = np.concatenate((labels["training"], labels["validation"])) logg.debug("NOT using validation data") # the shape of each sample input_shape = data["training"][0].shape # from hypa extract model param model_param = get_model_param_attention(hypa, num_labels, input_shape) batch_size_types = {"01": 32, "02": 16} batch_size = batch_size_types[hypa["batch_size_type"]] epoch_num_types = {"01": 15, "02": 30, "03": 2, "04": 4} epoch_num = epoch_num_types[hypa["epoch_num_type"]] # magic to fix the GPUs setup_gpus() model = AttentionModel(**model_param) # model.summary() metrics = [ tf.keras.metrics.CategoricalAccuracy(), tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), ] learning_rate_types = { "01": "fixed01", "02": "fixed02", "03": "exp_decay_step_01", "04": "exp_decay_smooth_01", "05": "clr_triangular2_01", "06": "clr_triangular2_02", "07": "clr_triangular2_03", "08": "clr_triangular2_04", "09": "clr_triangular2_05", "10": "exp_decay_smooth_02", } learning_rate_type = hypa["learning_rate_type"] lr_value = learning_rate_types[learning_rate_type] # setup opt fixed lr values if lr_value.startswith("fixed"): if lr_value == "fixed01": lr = 1e-3 elif lr_value == "fixed02": lr = 1e-4 else: lr = 1e-3 optimizer_types = { "a1": Adam(learning_rate=lr), "r1": RMSprop(learning_rate=lr) } opt = optimizer_types[hypa["optimizer_type"]] model.compile( optimizer=opt, loss=tf.keras.losses.CategoricalCrossentropy(), metrics=metrics, ) # setup callbacks callbacks = [] # setup exp decay step / smooth if lr_value.startswith("exp_decay"): if lr_value == "exp_decay_step_01": exp_decay_part = partial(exp_decay_step, epochs_drop=5) elif lr_value == "exp_decay_smooth_01": exp_decay_part = partial(exp_decay_smooth, epochs_drop=5) elif lr_value == "exp_decay_smooth_02": exp_decay_part = partial(exp_decay_smooth, epochs_drop=5, initial_lrate=1e-2) lrate = LearningRateScheduler(exp_decay_part) callbacks.append(lrate) # setup cyclic learning rate if lr_value.startswith("clr_triangular2"): base_lr = 1e-5 max_lr = 1e-3 # training iteration per epoch = num samples // batch size # step size suggested = 2~8 * iterations if lr_value == "clr_triangular2_01": step_factor = 8 step_size = step_factor * x.shape[0] // batch_size elif lr_value == "clr_triangular2_02": step_factor = 2 step_size = step_factor * x.shape[0] // batch_size # target_cycles = the number of cycles we want in those epochs # it_per_epoch = num_samples // batch_size # total_iterations = it_per_epoch * epoch_num # step_size = total_iterations // target_cycles elif lr_value == "clr_triangular2_03": # the number of cycles we want in those epochs target_cycles = 4 it_per_epoch = x.shape[0] // batch_size total_iterations = it_per_epoch * epoch_num step_size = total_iterations // (target_cycles * 2) elif lr_value == "clr_triangular2_04": # the number of cycles we want in those epochs target_cycles = 2 it_per_epoch = x.shape[0] // batch_size total_iterations = it_per_epoch * epoch_num step_size = total_iterations // (target_cycles * 2) elif lr_value == "clr_triangular2_05": # the number of cycles we want in those epochs target_cycles = 2 it_per_epoch = x.shape[0] // batch_size total_iterations = it_per_epoch * epoch_num step_size = total_iterations // (target_cycles * 2) # set bigger starting value max_lr = 1e-2 logg.debug(f"x.shape[0]: {x.shape[0]}") logg.debug(f"CLR is using step_size: {step_size}") mode = "triangular2" cyclic_lr = CyclicLR(base_lr, max_lr, step_size, mode) callbacks.append(cyclic_lr) # setup early stopping if learning_rate_type in ["01", "02", "03", "04"]: metric_to_monitor = "val_loss" if use_validation else "loss" early_stop = EarlyStopping( monitor=metric_to_monitor, patience=4, restore_best_weights=True, verbose=1, ) callbacks.append(early_stop) # model_checkpoint = ModelCheckpoint( # model_name, # monitor="val_loss", # save_best_only=True, # ) # a dict to recreate this training # FIXME this should be right before fit and have epoch_num/batch_size/lr info recap: ty.Dict[str, ty.Any] = {} recap["words"] = words recap["hypa"] = hypa recap["model_param"] = model_param recap["use_validation"] = use_validation recap["model_name"] = model_name recap["version"] = "001" # logg.debug(f"recap: {recap}") recap_path = info_folder / "recap.json" recap_path.write_text(json.dumps(recap, indent=4)) results = model.fit( x, y, validation_data=val_data, epochs=epoch_num, batch_size=batch_size, callbacks=callbacks, ) results_recap: ty.Dict[str, ty.Any] = {} results_recap["model_name"] = model_name results_recap["results_recap_version"] = "002" # eval performance on the various metrics eval_testing = model.evaluate(data["testing"], labels["testing"]) for metrics_name, value in zip(model.metrics_names, eval_testing): logg.debug(f"{metrics_name}: {value}") results_recap[metrics_name] = value # compute the confusion matrix y_pred = model.predict(data["testing"]) cm = pred_hot_2_cm(labels["testing"], y_pred, words) # logg.debug(f"cm: {cm}") results_recap["cm"] = cm.tolist() # compute the fscore fscore = analyze_confusion(cm, words) logg.debug(f"fscore: {fscore}") results_recap["fscore"] = fscore # save the histories results_recap["history_train"] = { mn: results.history[mn] for mn in model.metrics_names } if use_validation: results_recap["history_val"] = { f"val_{mn}": results.history[f"val_{mn}"] for mn in model.metrics_names } # plot the cm fig, ax = plt.subplots(figsize=(12, 12)) plot_confusion_matrix(cm, ax, model_name, words, fscore) plot_cm_path = info_folder / "test_confusion_matrix.png" fig.savefig(plot_cm_path) plt.close(fig) # save the results res_recap_path = info_folder / "results_recap.json" res_recap_path.write_text(json.dumps(results_recap, indent=4)) # if cyclic_lr was used save the history if lr_value.startswith("clr_triangular2"): logg.debug(f"cyclic_lr.history.keys(): {cyclic_lr.history.keys()}") clr_recap = {} for metric_name, values in cyclic_lr.history.items(): clr_recap[metric_name] = list(float(v) for v in values) clr_recap_path = info_folder / "clr_recap.json" clr_recap_path.write_text(json.dumps(clr_recap, indent=4)) # save the trained model model.save(model_path) placeholder_path.write_text(f"Trained. F-score: {fscore}")
def find_best_lr(hypa: ty.Dict[str, str]) -> None: """MAKEDOC: what is find_best_lr doing?""" logg = logging.getLogger(f"c.{__name__}.find_best_lr") # logg.setLevel("INFO") logg.debug("Start find_best_lr") # get the word list words = words_types[hypa["words_type"]] num_labels = len(words) # load data processed_folder = Path("data_proc") processed_path = processed_folder / f"{hypa['dataset_name']}" data, labels = load_processed(processed_path, words) # no need for validation x = np.concatenate((data["training"], data["validation"])) y = np.concatenate((labels["training"], labels["validation"])) # the shape of each sample input_shape = data["training"][0].shape # from hypa extract model param model_param = get_model_param_attention(hypa, num_labels, input_shape) # magic to fix the GPUs setup_gpus() model = AttentionModel(**model_param) # model.summary() start_lr = 1e-9 end_lr = 1e1 batch_size_types = {"01": 32, "02": 16} batch_size = batch_size_types[hypa["batch_size_type"]] epoch_num_types = {"01": 15, "02": 30, "03": 2} epoch_num = epoch_num_types[hypa["epoch_num_type"]] optimizer_types = {"a1": Adam(), "r1": RMSprop()} opt = optimizer_types[hypa["optimizer_type"]] metrics = [ tf.keras.metrics.CategoricalAccuracy(), tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), ] model.compile( optimizer=opt, loss=tf.keras.losses.CategoricalCrossentropy(), metrics=metrics, ) # find the best values lrf = LearningRateFinder(model) lrf.find((x, y), start_lr, end_lr, epochs=epoch_num, batchSize=batch_size) model_name = build_attention_name(hypa, False) fig_title = "LR_sweep" fig_title += f"_bs{batch_size}" fig_title += f"_en{epoch_num}" fig_title += f"__{model_name}" fig, ax = plt.subplots(figsize=(8, 8)) # get the plot lrf.plot_loss(ax=ax, title=fig_title) # save the plot plot_fol = Path("plot_results") / "att" / "find_best_lr" if not plot_fol.exists(): plot_fol.mkdir(parents=True, exist_ok=True) fig_name = fig_title + ".{}" fig.savefig(plot_fol / fig_name.format("png")) fig.savefig(plot_fol / fig_name.format("pdf")) # TODO: save the loss history plt.show()
def evaluate_model_cnn(which_dataset: str, train_words_type: str, test_words_type: str) -> None: """MAKEDOC: what is evaluate_model_cnn doing?""" logg = logging.getLogger(f"c.{__name__}.evaluate_model_cnn") # logg.setLevel("INFO") logg.debug("Start evaluate_model_cnn") # magic to fix the GPUs setup_gpus() # setup the parameters # hypa: ty.Dict[str, ty.Union[str, int]] = {} # hypa["base_dense_width"] = 32 # hypa["base_filters"] = 20 # hypa["batch_size"] = 32 # hypa["dropout_type"] = "01" # # hypa["epoch_num"] = 16 # hypa["epoch_num"] = 15 # hypa["kernel_size_type"] = "02" # # hypa["pool_size_type"] = "02" # hypa["pool_size_type"] = "01" # # hypa["learning_rate_type"] = "02" # hypa["learning_rate_type"] = "04" # hypa["optimizer_type"] = "a1" # hypa["dataset"] = which_dataset # hypa["words"] = train_words_type # hypa: ty.Dict[str, ty.Union[str, int]] = {} # hypa["base_dense_width"] = 32 # hypa["base_filters"] = 32 # hypa["batch_size"] = 32 # hypa["dropout_type"] = "02" # hypa["epoch_num"] = 15 # hypa["kernel_size_type"] = "02" # hypa["pool_size_type"] = "01" # hypa["learning_rate_type"] = "04" # hypa["optimizer_type"] = "a1" # hypa["dataset"] = which_dataset # hypa["words"] = train_words_type hypa: ty.Dict[str, ty.Union[str, int]] = { "base_dense_width": 32, "base_filters": 32, "batch_size": 32, # "dataset": "aug07", "dropout_type": "01", "epoch_num": 15, "kernel_size_type": "02", "learning_rate_type": "04", "optimizer_type": "a1", "pool_size_type": "01", # "words": "all", } hypa["dataset"] = which_dataset hypa["words"] = train_words_type # get the words # train_words = words_types[train_words_type] test_words = words_types[test_words_type] model_name = build_cnn_name(hypa) logg.debug(f"model_name: {model_name}") model_folder = Path("trained_models") / "cnn" model_path = model_folder / f"{model_name}.h5" if not model_path.exists(): logg.error(f"Model not found at: {model_path}") raise FileNotFoundError model = tf.keras.models.load_model(model_path) model.summary() # input data processed_path = Path("data_proc") / f"{which_dataset}" data, labels = load_processed(processed_path, test_words) logg.debug(f"data['testing'].shape: {data['testing'].shape}") # evaluate on the words you trained on logg.debug("Evaluate on test data:") model.evaluate(data["testing"], labels["testing"]) # model.evaluate(data["validation"], labels["validation"]) # predict labels/cm/fscore y_pred = model.predict(data["testing"]) cm = pred_hot_2_cm(labels["testing"], y_pred, test_words) # y_pred = model.predict(data["validation"]) # cm = pred_hot_2_cm(labels["validation"], y_pred, test_words) fscore = analyze_confusion(cm, test_words) logg.debug(f"fscore: {fscore}") fig, ax = plt.subplots(figsize=(12, 12)) plot_confusion_matrix(cm, ax, model_name, test_words, fscore) plt.show()
def evaluate_model_cm(model_name: str, test_words_type: str) -> float: r"""MAKEDOC: what is evaluate_model_cm doing?""" logg = logging.getLogger(f"c.{__name__}.evaluate_model_cm") # logg.setLevel("INFO") # logg.debug("\nStart evaluate_model_cm") # magic to fix the GPUs setup_gpus() logg.debug(f"\nmodel_name: {model_name}") dataset_re = re.compile("_ds(.*?)_") match = dataset_re.search(model_name) if match is not None: logg.debug(f"match[1]: {match[1]}") dataset_name = match[1] train_words_type_re = re.compile("_w(.*?)[_.]") match = train_words_type_re.search(model_name) if match is not None: logg.debug(f"match[1]: {match[1]}") train_words_type = match[1] arch_type = model_name[:3] if arch_type == "ATT": train_type_tag = "attention" else: train_type_tag = "area" # load the model model_folder = Path("trained_models") / train_type_tag model_path = model_folder / f"{model_name}.h5" model = tf_models.load_model(model_path) # model.summary() train_words = words_types[train_words_type] logg.debug(f"train_words: {train_words}") test_words = words_types[test_words_type] logg.debug(f"test_words: {test_words}") # input data must exist if dataset_name.startswith("mel"): preprocess_spec(dataset_name, test_words_type) elif dataset_name.startswith("aug"): do_augmentation(dataset_name, test_words_type) # input data processed_path = Path("data_proc") / f"{dataset_name}" data, labels = load_processed(processed_path, test_words) logg.debug(f"list(data.keys()): {list(data.keys())}") logg.debug(f"data['testing'].shape: {data['testing'].shape}") # evaluate on the words you trained on logg.debug("Evaluate on test data:") model.evaluate(data["testing"], labels["testing"]) # model.evaluate(data["validation"], labels["validation"]) # predict labels/cm/fscore y_pred = model.predict(data["testing"]) cm = pred_hot_2_cm(labels["testing"], y_pred, test_words) # y_pred = model.predict(data["validation"]) # cm = pred_hot_2_cm(labels["validation"], y_pred, test_words) fscore = analyze_confusion(cm, test_words) logg.debug(f"fscore: {fscore}") fig, ax = plt.subplots(figsize=(12, 12)) plot_confusion_matrix(cm, ax, model_name, test_words, fscore, train_words) fig_name = f"{model_name}_test{test_words_type}_cm.{{}}" cm_folder = Path("plot_results") / "cm_all01" if not cm_folder.exists(): cm_folder.mkdir(parents=True, exist_ok=True) plot_cm_path = cm_folder / fig_name.format("png") fig.savefig(plot_cm_path) plot_cm_path = cm_folder / fig_name.format("pdf") fig.savefig(plot_cm_path) # plt.show() return fscore
def evaluate_attention_weights(train_words_type: str, rec_words_type: str, do_new_record: bool = False) -> None: """MAKEDOC: what is evaluate_attention_weights doing?""" logg = logging.getLogger(f"c.{__name__}.evaluate_attention_weights") # logg.setLevel("INFO") logg.debug("Start evaluate_attention_weights") # magic to fix the GPUs setup_gpus() # ATT_ct02_dr02_ks02_lu01_as01_qt01_dw01_opa1_lr01_bs01_en01_dsmel04_wk1 # hypa: ty.Dict[str, str] = {} # hypa["conv_size_type"] = "02" # hypa["dropout_type"] = "02" # hypa["kernel_size_type"] = "02" # hypa["lstm_units_type"] = "01" # hypa["query_style_type"] = "01" # hypa["dense_width_type"] = "01" # hypa["optimizer_type"] = "a1" # hypa["learning_rate_type"] = "01" # hypa["batch_size_type"] = "01" # hypa["epoch_num_type"] = "01" # dataset_name = "mel04" # hypa["dataset_name"] = dataset_name # hypa["words_type"] = train_words_type # use_validation = True # ATT_ct02_dr01_ks01_lu01_qt05_dw01_opa1_lr03_bs02_en02_dsaug07_wLTnum hypa = { "batch_size_type": "02", "conv_size_type": "02", "dataset_name": "aug07", "dense_width_type": "01", "dropout_type": "01", "epoch_num_type": "02", "kernel_size_type": "01", "learning_rate_type": "03", "lstm_units_type": "01", "optimizer_type": "a1", "query_style_type": "05", "words_type": "LTnum", } use_validation = True dataset_name = hypa["dataset_name"] model_name = build_attention_name(hypa, use_validation) logg.debug(f"model_name: {model_name}") # load the model model_folder = Path("trained_models") / "attention" model_path = model_folder / f"{model_name}.h5" # model = tf.keras.models.load_model(model_path) # https://github.com/keras-team/keras/issues/5088#issuecomment-401498334 model = tf.keras.models.load_model( model_path, custom_objects={"backend": tf.keras.backend}) model.summary() logg.debug(f"ascii_model(model): {ascii_model(model)}") att_weight_model = tf.keras.models.Model( inputs=model.input, outputs=[ model.get_layer("output").output, model.get_layer("att_softmax").output, model.get_layer("bidirectional_1").output, ], ) att_weight_model.summary() # logg.debug(f"att_weight_model.outputs: {att_weight_model.outputs}") # get the training words train_words = words_types[train_words_type] # logg.debug(f"train_words: {train_words}") perm_pred = compute_permutation(train_words) rec_words_type = args.rec_words_type if rec_words_type == "train": rec_words = train_words[-3:] # rec_words = train_words[:] logg.debug(f"Using rec_words: {rec_words}") else: rec_words = words_types[rec_words_type] num_rec_words = len(rec_words) # record new audios if do_new_record: # where to save the audios audio_folder = Path("recorded_audio") if not audio_folder.exists(): audio_folder.mkdir(parents=True, exist_ok=True) # record the audios and save them in audio_folder audio_path_fmt = "{}_02.wav" audios = record_audios(rec_words, audio_folder, audio_path_fmt, timeout=0) # compute the spectrograms and build the dataset of correct shape img_specs = [] spec_dict = get_spec_dict() spec_kwargs = spec_dict[dataset_name] p2d_kwargs = {"ref": np.max} for word in rec_words: # get the name audio_path = audio_folder / audio_path_fmt.format(word) # convert it to mel log_spec = wav2mel(audio_path, spec_kwargs, p2d_kwargs) img_spec = log_spec.reshape((*log_spec.shape, 1)) # logg.debug(f"img_spec.shape: {img_spec.shape}") # img_spec.shape: (128, 32, 1) img_specs.append(img_spec) # the data needs to look like this data['testing'].shape: (735, 128, 32, 1) rec_data = np.stack(img_specs) # logg.debug(f"rec_data.shape: {rec_data.shape}") # load data if you do not want to record new audios else: # input data processed_folder = Path("data_proc") processed_path = processed_folder / f"{dataset_name}" # which word in the dataset to plot word_id = 2 # the loaded spectrograms rec_data_l: ty.List[np.ndarray] = [] for i, word in enumerate(rec_words): data, labels = load_processed(processed_path, [word]) # get one of the spectrograms word_data = data["testing"][word_id] rec_data_l.append(word_data) # turn the list into np array rec_data = np.stack(rec_data_l) # get prediction and attention weights pred, att_weights, LSTM_out = att_weight_model.predict(rec_data) # logg.debug(f"att_weights.shape: {att_weights.shape}") # logg.debug(f"att_weights[0].shape: {att_weights[0].shape}") # if we recorded fresh audios we also have the waveform to plot ax_add = 1 if do_new_record else 0 # plot the wave, spectrogram, weights and predictions in each column plot_size = 5 fw = plot_size * num_rec_words nrows = 3 + ax_add # nrows = 4 + ax_add fh = plot_size * nrows * 0.7 fig, axes = plt.subplots(nrows=nrows, ncols=num_rec_words, figsize=(fw, fh), sharey="row") fig.suptitle(f"Attention weights and predictions for {rec_words}", fontsize=20) for i, word in enumerate(rec_words): word_spec = rec_data[i][:, :, 0] # logg.debug(f"word_spec.shape: {word_spec.shape}") # plot the waveform if do_new_record: plot_waveform(audios[i], axes[0][i]) # plot the spectrogram title = f"Spectrogram for {word}" plot_spec(word_spec, axes[0 + ax_add][i], title=title) # plot the weights word_att_weights = att_weights[i] # plot_att_weights(word_att_weights, axes[1 + ax_add][i], title) word_att_weights_img = np.expand_dims(word_att_weights, axis=-1).T axes[1 + ax_add][i].imshow(word_att_weights_img, origin="lower", aspect="auto") title = f"Attention weights for {word}" axes[1 + ax_add][i].set_title(title) # plot the predictions word_pred = pred[i] # permute the prediction from sorted to the order you have word_pred = word_pred[perm_pred] pred_index = np.argmax(word_pred) title = f"Predictions for {word}" plot_pred(word_pred, train_words, axes[2 + ax_add][i], title, pred_index) # axes[3 + ax_add][i].imshow(LSTM_out[i], origin="lower") # fig.tight_layout() fig.tight_layout(h_pad=3, rect=[0, 0.03, 1, 0.97]) fig_name = f"{model_name}" fig_name += f"_{train_words_type}" fig_name += f"_{rec_words_type}_img" if do_new_record: fig_name += "_new.{}" else: fig_name += "_data.{}" plot_folder = Path("plot_results") results_path = plot_folder / fig_name.format("png") fig.savefig(results_path) results_path = plot_folder / fig_name.format("pdf") fig.savefig(results_path) if num_rec_words <= 6: plt.show()
def find_best_lr(hypa: ty.Dict[str, str]) -> None: """MAKEDOC: what is find_best_lr doing?""" logg = logging.getLogger(f"c.{__name__}.find_best_lr") # logg.setLevel("INFO") logg.debug("Start find_best_lr") # get the word list words = words_types[hypa["words_type"]] num_labels = len(words) # no validation just find the LR use_validation = False # name the model model_name = build_area_name(hypa, use_validation) logg.debug(f"model_name: {model_name}") # load data processed_folder = Path("data_proc") processed_path = processed_folder / f"{hypa['dataset_name']}" data, labels = load_processed(processed_path, words) # the shape of each sample input_shape = data["training"][0].shape # from hypa extract model param model_param = get_model_param_area(hypa, num_labels, input_shape) # no need for validation x = np.concatenate((data["training"], data["validation"])) y = np.concatenate((labels["training"], labels["validation"])) # magic to fix the GPUs setup_gpus() # get the model with the chosen params net_type = hypa["net_type"] if net_type == "ARN": model = AreaNet.build(**model_param) elif net_type == "AAN": model = ActualAreaNet.build(**model_param) elif net_type == "VAN": model = VerticalAreaNet.build(**model_param) elif net_type.startswith("SI"): if net_type == "SIM": sim_type = "1" elif net_type == "SI2": sim_type = "2" model = SimpleNet.build(sim_type=sim_type, **model_param) num_samples = x.shape[0] logg.debug(f"num_samples: {num_samples}") # from hypa extract training param (epochs, batch, opt, ...) training_param = get_training_param_area(hypa, use_validation, model_path=None, num_samples=num_samples) # a few metrics to track metrics = [ tf.keras.metrics.CategoricalAccuracy(), tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), ] # compile the model model.compile( optimizer=training_param["opt"], loss=tf.keras.losses.CategoricalCrossentropy(), metrics=metrics, ) # boundary values start_lr = 1e-9 end_lr = 1e1 # find the best values lrf = LearningRateFinder(model) lrf.find( (x, y), start_lr, end_lr, epochs=training_param["epochs"], batchSize=training_param["batch_size"], ) fig_title = "LR_sweep" fig_title += f"__{model_name}" fig, ax = plt.subplots(figsize=(8, 8)) # get the plot lrf.plot_loss(ax=ax, title=fig_title) # save the plot plot_fol = Path("plot_results") / "area" / "find_best_lr" if not plot_fol.exists(): plot_fol.mkdir(parents=True, exist_ok=True) fig_name = fig_title + ".{}" fig.savefig(plot_fol / fig_name.format("png")) fig.savefig(plot_fol / fig_name.format("pdf")) recap_loss = {} recap_loss["lrs"] = [float(lr) for lr in lrf.lrs[:]] recap_loss["losses"] = [float(loss) for loss in lrf.losses[:]] loss_path = plot_fol / f"loss_{fig_title}.json" loss_path.write_text(json.dumps(recap_loss, indent=4)) plt.show()
def train_area( hypa: ty.Dict[str, str], force_retrain: bool, use_validation: bool, trained_folder: Path, root_info_folder: Path, ) -> None: """MAKEDOC: what is train_area doing?""" logg = logging.getLogger(f"c.{__name__}.train_area") # logg.setLevel("INFO") logg.debug("Start train_area") ########################################################## # Setup folders ########################################################## # name the model model_name = build_area_name(hypa, use_validation) logg.debug(f"model_name: {model_name}") # save the trained model here model_path = trained_folder / f"{model_name}.h5" placeholder_path = trained_folder / f"{model_name}.txt" # check if this model has already been trained if placeholder_path.exists(): if force_retrain: logg.warn("\nRETRAINING MODEL!!\n") else: logg.debug("Already trained") return # save info regarding the model training in this folder model_info_folder = root_info_folder / model_name if not model_info_folder.exists(): model_info_folder.mkdir(parents=True, exist_ok=True) # magic to fix the GPUs setup_gpus() ########################################################## # Load data ########################################################## # get the words words = words_types[hypa["words_type"]] num_labels = len(words) # load data processed_folder = Path("data_proc") processed_path = processed_folder / f"{hypa['dataset_name']}" data, labels = load_processed(processed_path, words) # concatenate train and val for final train val_data = None if use_validation: x = data["training"] y = labels["training"] val_data = (data["validation"], labels["validation"]) logg.debug("Using validation data") else: x = np.concatenate((data["training"], data["validation"])) y = np.concatenate((labels["training"], labels["validation"])) logg.debug("NOT using validation data") ########################################################## # Setup model ########################################################## # the shape of each sample input_shape = data["training"][0].shape # from hypa extract model param model_param = get_model_param_area(hypa, num_labels, input_shape) # get the model with the chosen params net_type = hypa["net_type"] if net_type == "ARN": model = AreaNet.build(**model_param) elif net_type == "AAN": model = ActualAreaNet.build(**model_param) elif net_type == "VAN": model = VerticalAreaNet.build(**model_param) elif net_type.startswith("SI"): if net_type == "SIM": sim_type = "1" elif net_type == "SI2": sim_type = "2" model = SimpleNet.build(sim_type=sim_type, **model_param) num_samples = x.shape[0] logg.debug(f"num_samples: {num_samples}") # from hypa extract training param (epochs, batch, opt, ...) training_param = get_training_param_area(hypa, use_validation, model_path, num_samples) # a few metrics to track metrics = [ tf.keras.metrics.CategoricalAccuracy(), tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), ] # compile the model model.compile( optimizer=training_param["opt"], loss=tf.keras.losses.CategoricalCrossentropy(), metrics=metrics, ) # recap recap: ty.Dict[str, ty.Any] = {} recap["model_name"] = model_name recap["words"] = words recap["hypa"] = hypa recap["model_param"] = model_param recap["use_validation"] = use_validation recap["batch_size"] = training_param["batch_size"] recap["epochs"] = training_param["epochs"] recap["lr_name"] = training_param["lr_name"] recap["version"] = "002" # logg.debug(f"recap: {recap}") recap_path = model_info_folder / "recap.json" recap_path.write_text(json.dumps(recap, indent=4)) # https://stackoverflow.com/a/45546663/2237151 model_summary_path = model_info_folder / "model_summary.txt" with model_summary_path.open("w") as msf: model.summary(line_length=150, print_fn=lambda x: msf.write(x + "\n")) ########################################################## # Fit model ########################################################## results = model.fit( x, y, validation_data=val_data, epochs=training_param["epochs"], batch_size=training_param["batch_size"], callbacks=training_param["callbacks"], ) ########################################################## # Save results, history, performance ########################################################## # results_recap results_recap: ty.Dict[str, ty.Any] = {} results_recap["model_name"] = model_name results_recap["results_recap_version"] = "001" # evaluate performance eval_testing = model.evaluate(data["testing"], labels["testing"]) for metrics_name, value in zip(model.metrics_names, eval_testing): logg.debug(f"{metrics_name}: {value}") results_recap[metrics_name] = value # confusion matrix y_pred = model.predict(data["testing"]) cm = pred_hot_2_cm(labels["testing"], y_pred, words) results_recap["cm"] = cm.tolist() # fscore fscore = analyze_confusion(cm, words) logg.debug(f"fscore: {fscore}") results_recap["fscore"] = fscore # save the histories results_recap["history_train"] = { mn: results.history[mn] for mn in model.metrics_names } if use_validation: results_recap["history_val"] = { f"val_{mn}": results.history[f"val_{mn}"] for mn in model.metrics_names } # save the results res_recap_path = model_info_folder / "results_recap.json" res_recap_path.write_text(json.dumps(results_recap, indent=4)) # plot the cm fig, ax = plt.subplots(figsize=(12, 12)) plot_confusion_matrix(cm, ax, model_name, words, fscore) plot_cm_path = model_info_folder / "test_confusion_matrix.png" fig.savefig(plot_cm_path) plt.close(fig) # save the trained model model.save(model_path) # save the placeholder placeholder_path.write_text(f"Trained. F-score: {fscore}")