def evaluate_attention_weights(train_words_type: str) -> None:
"""MAKEDOC: what is evaluate_attention_weights doing?"""
logg = logging.getLogger(f"c.{__name__}.evaluate_attention_weights")
# logg.setLevel("INFO")
logg.debug("Start evaluate_attention_weights")
# magic to fix the GPUs
setup_gpus()
# VAN_opa1_lr05_bs32_en15_dsaug07_wLTall
hypa = {
"batch_size_type": "32",
"dataset_name": "aug07",
"epoch_num_type": "15",
"learning_rate_type": "05",
"net_type": "VAN",
"optimizer_type": "a1",
"words_type": "LTall",
}
use_validation = True
dataset_name = hypa["dataset_name"]
batch_size = int(hypa["batch_size_type"])
# get the model name
model_name = build_area_name(hypa, use_validation)
logg.debug(f"model_name: {model_name}")
# load the model
model_folder = Path("trained_models") / "area"
model_path = model_folder / f"{model_name}.h5"
model = tf_models.load_model(model_path)
# get the output layer because you forgot to name it
name_output_layer = model.layers[-1].name
logg.debug(f"name_output_layer: {name_output_layer}")
# build a model on top of that to get the weights
att_weight_model = tf_models.Model(
inputs=model.input,
outputs=[
model.get_layer(name_output_layer).output,
model.get_layer("area_values").output,
],
)
att_weight_model.summary()
# get the training words
train_words = words_types[train_words_type]
perm_pred = compute_permutation(train_words)
logg.debug(f"perm_pred: {perm_pred}")
sorted_train_words = sorted(train_words)
logg.debug(f"sorted(train_words): {sorted(train_words)}")
# load data if you do not want to record new audios
processed_folder = Path("data_proc")
processed_path = processed_folder / f"{dataset_name}"
logg.debug(f"processed_path: {processed_path}")
# # evaluate on all data because im confused
# data, labels = load_processed(processed_path, train_words)
# logg.debug(f"data['testing'].shape: {data['testing'].shape}")
# logg.debug(f"labels['testing'].shape: {labels['testing'].shape}")
# eval_testing = model.evaluate(data["testing"], labels["testing"])
# for metrics_name, value in zip(model.metrics_names, eval_testing):
# logg.debug(f"{metrics_name}: {value}")
# which word in the dataset to plot
# word_id = 5
# word_id = 7
word_id = 12
# the loaded spectrograms
rec_data_l: ty.List[np.ndarray] = []
# for now we do not record new words
rec_words = train_words[30:32]
num_rec_words = len(rec_words)
logg.debug(f"processed_path: {processed_path}")
for i, word in enumerate(rec_words):
logg.debug(f"\nword: {word}")
data, labels = load_processed(processed_path, [word])
logg.debug(f"data['testing'].shape: {data['testing'].shape}")
logg.debug(f"labels['testing'].shape: {labels['testing'].shape}")
# eval_testing = model.evaluate(data["testing"], labels["testing"])
# for metrics_name, value in zip(model.metrics_names, eval_testing):
# logg.debug(f"{metrics_name}: {value}")
# get one of the spectrograms
word_data = data["testing"][word_id]
rec_data_l.append(word_data)
pred, att_weights = att_weight_model.predict(data["testing"])
logg.debug(f"pred.shape: {pred.shape}")
logg.debug(f"pred[0].shape: {pred[0].shape}")
pred_am_all = np.argmax(pred, axis=1)
logg.debug(f"pred_am_all: {pred_am_all}")
pred_index = np.argmax(pred[0])
pred_word = sorted_train_words[pred_index]
logg.debug(f"sorted pred_word: {pred_word} pred_index {pred_index}")
# test EVERY SINGLE spectrogram
spec_num = data["testing"].shape[0]
for wid in range(spec_num):
# get the word
word_data = data["testing"][wid]
logg.debug(f"word_data.shape: {word_data.shape}")
batch_word_data = np.expand_dims(word_data, axis=0)
logg.debug(f"batch_word_data.shape: {batch_word_data.shape}")
shape_batch = (batch_size, *word_data.shape)
logg.debug(f"shape_batch: {shape_batch}")
batch_word_data_big = np.zeros(shape_batch, dtype=np.float32)
for i in range(batch_size):
batch_word_data_big[i, :, :, :] = batch_word_data
# batch_word_data_big[0, :, :, :] = batch_word_data
# predict it
# pred, att_weights = att_weight_model.predict(batch_word_data)
pred, att_weights = att_weight_model.predict(batch_word_data_big)
# show all prediction
# pred_am = np.argmax(pred, axis=1)
# logg.debug(f"pred_am: {pred_am}")
# focus on first prediction
word_pred = pred[0]
pred_index = np.argmax(word_pred)
pred_word = sorted_train_words[pred_index]
recap = ""
if pred_word == word:
recap += "correct "
else:
recap += " wrong "
pred_am = np.argmax(pred, axis=1)
logg.debug(f"pred_am: {pred_am}")
recap += f"sorted pred_word: {pred_word} pred_index {pred_index}"
recap += f" word_pred.shape {word_pred.shape}"
recap += f" pred_am_all[wid] {pred_am_all[wid]}"
# pred_f = ", ".join([f"{p:.3f}" for p in pred[0]])
# recap += f" pred_f: {pred_f}"
logg.debug(recap)
# break
# turn the list into np array
rec_data = np.stack(rec_data_l)
logg.debug(f"\nrec_data.shape: {rec_data.shape}")
# get prediction and attention weights
pred, att_weights = att_weight_model.predict(rec_data)
logg.debug(f"att_weights.shape: {att_weights.shape}")
logg.debug(f"att_weights[0].shape: {att_weights[0].shape}")
plot_size = 5
fw = plot_size * num_rec_words
nrows = 2
fh = plot_size * nrows
fig, axes = plt.subplots(nrows=nrows,
ncols=num_rec_words,
figsize=(fw, fh))
fig.suptitle("Attention weights computed with VerticalAreaNet",
fontsize=20)
for i, word in enumerate(rec_words):
logg.debug(f"recword: {word}")
# show the spectrogram
word_spec = rec_data[i][:, :, 0]
# logg.debug(f"word_spec: {word_spec}")
axes[0][i].set_title(f"Spectrogram for {word}", fontsize=20)
axes[0][i].imshow(word_spec, origin="lower")
axes[1][i].set_title(f"Attention weights for {word}", fontsize=20)
att_w = att_weights[i][:, :, 0]
axes[1][i].imshow(att_w, origin="lower")
logg.debug(f"att_w.max(): {att_w.max()}")
# axes[0][i].imshow(
# att_w, origin="lower", extent=img.get_extent(), cmap="gray", alpha=0.4
# )
# weighted = word_spec * att_w
# axes[2][i].imshow(weighted, origin="lower")
word_pred = pred[i]
pred_index = np.argmax(word_pred)
pred_word = sorted_train_words[pred_index]
logg.debug(f"sorted pred_word: {pred_word} pred_index {pred_index}")
# # plot the predictions
word_pred = pred[i]
# logg.debug(f"word_pred: {word_pred}")
# # permute the prediction from sorted to the order you have
word_pred = word_pred[perm_pred]
# logg.debug(f"word_pred permuted: {word_pred}")
pred_index = np.argmax(word_pred)
pred_word = train_words[pred_index]
logg.debug(f"pred_word: {pred_word} pred_index {pred_index}")
# title = f"Predictions for {word}"
# plot_pred(word_pred, train_words, axes[2][i], title, pred_index)
fig.tight_layout()
fig_name = f"{model_name}"
fig_name += "_0002.{}"
plot_folder = Path("plot_results")
results_path = plot_folder / fig_name.format("pdf")
fig.savefig(results_path)
plt.show()
def evaluate_model_area(model_name: str, test_words_type: str) -> None:
r"""MAKEDOC: what is evaluate_model_area doing?"""
logg = logging.getLogger(f"c.{__name__}.evaluate_model_area")
# logg.setLevel("INFO")
logg.debug("Start evaluate_model_area")
# magic to fix the GPUs
setup_gpus()
# # VAN_opa1_lr05_bs32_en15_dsaug07_wLTall
# hypa = {
# "batch_size_type": "32",
# "dataset_name": "aug07",
# "epoch_num_type": "15",
# "learning_rate_type": "03",
# "net_type": "VAN",
# "optimizer_type": "a1",
# # "words_type": "LTall",
# "words_type": train_words_type,
# }
# # use_validation = True
# use_validation = False
# dataset_name = hypa["dataset_name"]
# get the model name
# model_name = build_area_name(hypa, use_validation)
logg.debug(f"model_name: {model_name}")
dataset_re = re.compile("_ds(.*?)_")
match = dataset_re.search(model_name)
if match is not None:
logg.debug(f"match[1]: {match[1]}")
dataset_name = match[1]
train_words_type_re = re.compile("_w(.*?)[_.]")
match = train_words_type_re.search(model_name)
if match is not None:
logg.debug(f"match[1]: {match[1]}")
train_words_type = match[1]
# load the model
model_folder = Path("trained_models") / "area"
model_path = model_folder / f"{model_name}.h5"
model = tf_models.load_model(model_path)
# model.summary()
train_words = words_types[train_words_type]
logg.debug(f"train_words: {train_words}")
test_words = words_types[test_words_type]
logg.debug(f"test_words: {test_words}")
# input data
processed_path = Path("data_proc") / f"{dataset_name}"
data, labels = load_processed(processed_path, test_words)
logg.debug(f"list(data.keys()): {list(data.keys())}")
logg.debug(f"data['testing'].shape: {data['testing'].shape}")
# evaluate on the words you trained on
logg.debug("Evaluate on test data:")
model.evaluate(data["testing"], labels["testing"])
# model.evaluate(data["validation"], labels["validation"])
# predict labels/cm/fscore
y_pred = model.predict(data["testing"])
cm = pred_hot_2_cm(labels["testing"], y_pred, test_words)
# y_pred = model.predict(data["validation"])
# cm = pred_hot_2_cm(labels["validation"], y_pred, test_words)
fscore = analyze_confusion(cm, test_words)
logg.debug(f"fscore: {fscore}")
fig, ax = plt.subplots(figsize=(12, 12))
plot_confusion_matrix(cm, ax, model_name, test_words, fscore, train_words)
fig_name = f"{model_name}_test{test_words_type}_cm.{{}}"
cm_folder = Path("plot_results") / "cm"
if not cm_folder.exists():
cm_folder.mkdir(parents=True, exist_ok=True)
plot_cm_path = cm_folder / fig_name.format("png")
fig.savefig(plot_cm_path)
plot_cm_path = cm_folder / fig_name.format("pdf")
fig.savefig(plot_cm_path)
plt.show()
def recompute_fscore_cnn() -> None:
"""MAKEDOC: what is recompute_fscore_cnn doing?"""
logg = logging.getLogger(f"c.{__name__}.recompute_fscore_cnn")
# logg.setLevel("INFO")
logg.debug("Start recompute_fscore_cnn")
info_folder = Path("info")
trained_folder = Path("trained_models")
for model_folder in info_folder.iterdir():
# logg.debug(f"model_folder: {model_folder}")
# check that it is a CNN
model_name = model_folder.name
if not model_name.startswith("CNN"):
continue
# check that the model is trained and not a placeholder
model_path = trained_folder / f"{model_name}.h5"
found_model = False
if model_path.exists():
if model_path.stat().st_size > 100:
found_model = True
if not found_model:
continue
# load it
model = models.load_model(model_path)
res_recap_path = model_folder / "results_recap.json"
if not res_recap_path.exists():
continue
results_recap = json.loads(res_recap_path.read_text())
# logg.debug(f"results_recap['cm']: {results_recap['cm']}")
recap_path = model_folder / "recap.json"
recap = json.loads(recap_path.read_text())
# logg.debug(f"recap['words']: {recap['words']}")
words = recap["words"]
hypa = recap["hypa"]
# check that the data is available
dn = hypa["dataset"]
wt = hypa["words"]
if dn.startswith("mel") or dn.startswith("mfcc"):
preprocess_spec(dn, wt)
elif dn.startswith("aug"):
do_augmentation(dn, wt)
processed_path = Path("data_proc") / f"{hypa['dataset']}"
data, labels = load_processed(processed_path, words)
y_pred = model.predict(data["testing"])
cm = pred_hot_2_cm(labels["testing"], y_pred, words)
fscore = analyze_confusion(cm, words)
# logg.debug(f"fscore: {fscore}")
# overwrite the cm
results_recap["cm"] = cm.tolist()
# add the fscore
results_recap["fscore"] = fscore
# increase the version
results_recap["results_recap_version"] = "002"
# write the new results
res_recap_path.write_text(json.dumps(results_recap, indent=4))
# increase the recap version (shows that it is after this debacle)
recap["version"] = "002"
recap_path.write_text(json.dumps(recap, indent=4))
# save the new plots
fig, ax = plt.subplots(figsize=(12, 12))
plot_confusion_matrix(cm, ax, model_name, words, fscore)
plot_cm_path = info_folder / "test_confusion_matrix.png"
fig.savefig(plot_cm_path)
plt.close(fig)
def train_model(hypa, force_retrain):
"""MAKEDOC: What is train_model doing?"""
logg = logging.getLogger(f"c.{__name__}.train_model")
# logg.debug("Starting train_model")
# get the words
words = words_types[hypa["words"]]
# name the model
model_name = build_cnn_name(hypa)
logg.debug(f"model_name: {model_name}")
# save the trained model here
model_folder = Path("trained_models") / "cnn"
if not model_folder.exists():
model_folder.mkdir(parents=True, exist_ok=True)
model_path = model_folder / f"{model_name}.h5"
# logg.debug(f"model_path: {model_path}")
placeholder_path = model_folder / f"{model_name}.txt"
# check if this model has already been trained
if placeholder_path.exists():
if force_retrain:
logg.warn("\nRETRAINING MODEL!!\n")
else:
logg.debug("Already trained")
return
# save info regarding the model training in this folder
info_folder = Path("info") / "cnn" / model_name
if not info_folder.exists():
info_folder.mkdir(parents=True, exist_ok=True)
# magic to fix the GPUs
setup_gpus()
# input data
processed_path = Path("data_proc") / f"{hypa['dataset']}"
data, labels = load_processed(processed_path, words)
# from hypa extract model param
model_param = {}
model_param["num_labels"] = len(words)
model_param["input_shape"] = data["training"][0].shape
model_param["base_filters"] = hypa["base_filters"]
model_param["base_dense_width"] = hypa["base_dense_width"]
# translate types to actual values
kernel_size_types = {
"01": [(2, 2), (2, 2), (2, 2)],
"02": [(5, 1), (3, 3), (3, 3)],
"03": [(1, 5), (3, 3), (3, 3)],
}
model_param["kernel_sizes"] = kernel_size_types[hypa["kernel_size_type"]]
pool_size_types = {
"01": [(2, 2), (2, 2), (2, 2)],
"02": [(2, 1), (2, 2), (2, 2)],
"03": [(1, 2), (2, 2), (2, 2)],
}
model_param["pool_sizes"] = pool_size_types[hypa["pool_size_type"]]
dropout_types = {"01": [0.03, 0.01], "02": [0.3, 0.1]}
model_param["dropouts"] = dropout_types[hypa["dropout_type"]]
# a dict to recreate this training
recap = {}
recap["words"] = words
recap["hypa"] = hypa
recap["model_param"] = model_param
recap["model_name"] = model_name
recap["version"] = "002"
# logg.debug(f"recap: {recap}")
recap_path = info_folder / "recap.json"
recap_path.write_text(json.dumps(recap, indent=4))
learning_rate_types = {
"01": "fixed01",
"02": "fixed02",
"03": "fixed03",
"e1": "exp_decay_keras_01",
"04": "exp_decay_step_01",
"05": "exp_decay_smooth_01",
"06": "exp_decay_smooth_02",
}
learning_rate_type = hypa["learning_rate_type"]
lr_value = learning_rate_types[learning_rate_type]
# setup opt fixed lr values
if lr_value.startswith("fixed"):
if lr_value == "fixed01":
lr = 1e-2
elif lr_value == "fixed02":
lr = 1e-3
elif lr_value == "fixed03":
lr = 1e-4
else:
lr = 1e-3
if lr_value == "exp_decay_keras_01":
lr = ExponentialDecay(0.1, decay_steps=100000, decay_rate=0.96, staircase=True)
optimizer_types = {
"a1": Adam(learning_rate=lr),
"r1": RMSprop(learning_rate=lr),
}
opt = optimizer_types[hypa["optimizer_type"]]
# create the model
model = CNNmodel(**model_param)
# model.summary()
metrics = [
tf.keras.metrics.CategoricalAccuracy(),
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall(),
]
model.compile(
optimizer=opt,
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=metrics,
)
# setup callbacks
callbacks = []
# setup exp decay step / smooth
if lr_value.startswith("exp_decay"):
if lr_value == "exp_decay_step_01":
exp_decay_part = partial(exp_decay_step, epochs_drop=5)
elif lr_value == "exp_decay_smooth_01":
exp_decay_part = partial(exp_decay_smooth, epochs_drop=5)
elif lr_value == "exp_decay_smooth_02":
exp_decay_part = partial(
exp_decay_smooth, epochs_drop=5, initial_lrate=1e-2
)
lrate = LearningRateScheduler(exp_decay_part)
callbacks.append(lrate)
# # setup early stopping
# early_stop = EarlyStopping(
# # monitor="val_categorical_accuracy",
# monitor="val_loss",
# patience=4,
# verbose=1,
# restore_best_weights=True,
# )
# callbacks.append(early_stop)
# get training parameters
BATCH_SIZE = hypa["batch_size"]
SHUFFLE_BUFFER_SIZE = BATCH_SIZE
EPOCH_NUM = hypa["epoch_num"]
# load the datasets
datasets = {}
for which in ["training", "validation", "testing"]:
# logg.debug(f"data[{which}].shape: {data[which].shape}")
datasets[which] = Dataset.from_tensor_slices((data[which], labels[which]))
# logg.debug(f"datasets[{which}]: {datasets[which]}")
datasets[which] = datasets[which].shuffle(SHUFFLE_BUFFER_SIZE).batch(BATCH_SIZE)
# logg.debug(f"datasets[{which}]: {datasets[which]}")
# train the model
results = model.fit(
data["training"],
labels["training"],
# validation_data=datasets["validation"],
validation_data=(data["validation"], labels["validation"]),
batch_size=BATCH_SIZE,
epochs=EPOCH_NUM,
verbose=1,
callbacks=callbacks,
)
# save the trained model
model.save(model_path)
results_recap = {}
results_recap["model_name"] = model_name
# version of the results saved
results_recap["results_recap_version"] = "002"
# quickly evaluate the results
# logg.debug(f"\nmodel.metrics_names: {model.metrics_names}")
# for which in ["training", "validation", "testing"]:
# model_eval = model.evaluate(datasets[which])
# logg.debug(f"{which}: model_eval: {model_eval}")
# save the evaluation results
logg.debug("Evaluate on test data:")
# eval_testing = model.evaluate(datasets["testing"])
# results_recap[model.metrics_names[0]] = eval_testing[0]
# results_recap[model.metrics_names[1]] = eval_testing[1]
eval_testing = model.evaluate(data["testing"], labels["testing"])
for metrics_name, value in zip(model.metrics_names, eval_testing):
logg.debug(f"{metrics_name}: {value}")
results_recap[metrics_name] = value
# compute the confusion matrix
# y_pred = model.predict(datasets["testing"])
y_pred = model.predict(data["testing"])
cm = pred_hot_2_cm(labels["testing"], y_pred, words)
# logg.debug(f"cm: {cm}")
results_recap["cm"] = cm.tolist()
# compute the fscore
fscore = analyze_confusion(cm, words)
logg.debug(f"fscore: {fscore}")
# plot the cm
fig, ax = plt.subplots(figsize=(12, 12))
plot_confusion_matrix(cm, ax, model_name, words, fscore)
plot_cm_path = info_folder / "test_confusion_matrix.png"
fig.savefig(plot_cm_path)
plt.close(fig)
# save the histories
results_recap["history"] = {
"loss": results.history["loss"],
"val_loss": results.history["val_loss"],
"categorical_accuracy": results.history["categorical_accuracy"],
"val_categorical_accuracy": results.history["val_categorical_accuracy"],
}
# save the results
res_recap_path = info_folder / "results_recap.json"
res_recap_path.write_text(json.dumps(results_recap, indent=4))
y_pred_dataset = model.predict(datasets["testing"])
cm_dataset = pred_hot_2_cm(labels["testing"], y_pred_dataset, words)
fscore_dataset = analyze_confusion(cm_dataset, words)
logg.debug(f"fscore_dataset: {fscore_dataset} fscore {fscore}")
# for i, (ys, yd) in enumerate(zip(y_pred, y_pred_dataset)):
# pred_split = np.argmax(ys)
# pred_dataset = np.argmax(yd)
# logg.debug(f"i: {i} pred_split: {pred_split} pred_dataset: {pred_dataset}")
# plt.show()
placeholder_path.write_text(f"Trained. F-score: {fscore}")
return "done_training"
def train_attention(hypa: ty.Dict[str, str], force_retrain: bool,
use_validation: bool) -> None:
"""MAKEDOC: what is train_attention doing?"""
logg = logging.getLogger(f"c.{__name__}.train_attention")
# logg.setLevel("INFO")
logg.debug("Start train_attention")
# build the model name
model_name = build_attention_name(hypa, use_validation)
logg.debug(f"model_name: {model_name}")
# save the trained model here
model_folder = Path("trained_models") / "attention"
if not model_folder.exists():
model_folder.mkdir(parents=True, exist_ok=True)
model_path = model_folder / f"{model_name}.h5"
placeholder_path = model_folder / f"{model_name}.txt"
# check if this model has already been trained
if placeholder_path.exists():
if force_retrain:
logg.warn("\nRETRAINING MODEL!!\n")
else:
logg.debug("Already trained")
return
# save info regarding the model training in this folder
info_folder = Path("info") / "attention" / model_name
if not info_folder.exists():
info_folder.mkdir(parents=True, exist_ok=True)
# get the word list
words = words_types[hypa["words_type"]]
num_labels = len(words)
# load data
processed_folder = Path("data_proc")
processed_path = processed_folder / f"{hypa['dataset_name']}"
data, labels = load_processed(processed_path, words)
# concatenate train and val for final train
val_data = None
if use_validation:
x = data["training"]
y = labels["training"]
val_data = (data["validation"], labels["validation"])
logg.debug("Using validation data")
else:
x = np.concatenate((data["training"], data["validation"]))
y = np.concatenate((labels["training"], labels["validation"]))
logg.debug("NOT using validation data")
# the shape of each sample
input_shape = data["training"][0].shape
# from hypa extract model param
model_param = get_model_param_attention(hypa, num_labels, input_shape)
batch_size_types = {"01": 32, "02": 16}
batch_size = batch_size_types[hypa["batch_size_type"]]
epoch_num_types = {"01": 15, "02": 30, "03": 2, "04": 4}
epoch_num = epoch_num_types[hypa["epoch_num_type"]]
# magic to fix the GPUs
setup_gpus()
model = AttentionModel(**model_param)
# model.summary()
metrics = [
tf.keras.metrics.CategoricalAccuracy(),
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall(),
]
learning_rate_types = {
"01": "fixed01",
"02": "fixed02",
"03": "exp_decay_step_01",
"04": "exp_decay_smooth_01",
"05": "clr_triangular2_01",
"06": "clr_triangular2_02",
"07": "clr_triangular2_03",
"08": "clr_triangular2_04",
"09": "clr_triangular2_05",
"10": "exp_decay_smooth_02",
}
learning_rate_type = hypa["learning_rate_type"]
lr_value = learning_rate_types[learning_rate_type]
# setup opt fixed lr values
if lr_value.startswith("fixed"):
if lr_value == "fixed01":
lr = 1e-3
elif lr_value == "fixed02":
lr = 1e-4
else:
lr = 1e-3
optimizer_types = {
"a1": Adam(learning_rate=lr),
"r1": RMSprop(learning_rate=lr)
}
opt = optimizer_types[hypa["optimizer_type"]]
model.compile(
optimizer=opt,
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=metrics,
)
# setup callbacks
callbacks = []
# setup exp decay step / smooth
if lr_value.startswith("exp_decay"):
if lr_value == "exp_decay_step_01":
exp_decay_part = partial(exp_decay_step, epochs_drop=5)
elif lr_value == "exp_decay_smooth_01":
exp_decay_part = partial(exp_decay_smooth, epochs_drop=5)
elif lr_value == "exp_decay_smooth_02":
exp_decay_part = partial(exp_decay_smooth,
epochs_drop=5,
initial_lrate=1e-2)
lrate = LearningRateScheduler(exp_decay_part)
callbacks.append(lrate)
# setup cyclic learning rate
if lr_value.startswith("clr_triangular2"):
base_lr = 1e-5
max_lr = 1e-3
# training iteration per epoch = num samples // batch size
# step size suggested = 2~8 * iterations
if lr_value == "clr_triangular2_01":
step_factor = 8
step_size = step_factor * x.shape[0] // batch_size
elif lr_value == "clr_triangular2_02":
step_factor = 2
step_size = step_factor * x.shape[0] // batch_size
# target_cycles = the number of cycles we want in those epochs
# it_per_epoch = num_samples // batch_size
# total_iterations = it_per_epoch * epoch_num
# step_size = total_iterations // target_cycles
elif lr_value == "clr_triangular2_03":
# the number of cycles we want in those epochs
target_cycles = 4
it_per_epoch = x.shape[0] // batch_size
total_iterations = it_per_epoch * epoch_num
step_size = total_iterations // (target_cycles * 2)
elif lr_value == "clr_triangular2_04":
# the number of cycles we want in those epochs
target_cycles = 2
it_per_epoch = x.shape[0] // batch_size
total_iterations = it_per_epoch * epoch_num
step_size = total_iterations // (target_cycles * 2)
elif lr_value == "clr_triangular2_05":
# the number of cycles we want in those epochs
target_cycles = 2
it_per_epoch = x.shape[0] // batch_size
total_iterations = it_per_epoch * epoch_num
step_size = total_iterations // (target_cycles * 2)
# set bigger starting value
max_lr = 1e-2
logg.debug(f"x.shape[0]: {x.shape[0]}")
logg.debug(f"CLR is using step_size: {step_size}")
mode = "triangular2"
cyclic_lr = CyclicLR(base_lr, max_lr, step_size, mode)
callbacks.append(cyclic_lr)
# setup early stopping
if learning_rate_type in ["01", "02", "03", "04"]:
metric_to_monitor = "val_loss" if use_validation else "loss"
early_stop = EarlyStopping(
monitor=metric_to_monitor,
patience=4,
restore_best_weights=True,
verbose=1,
)
callbacks.append(early_stop)
# model_checkpoint = ModelCheckpoint(
# model_name,
# monitor="val_loss",
# save_best_only=True,
# )
# a dict to recreate this training
# FIXME this should be right before fit and have epoch_num/batch_size/lr info
recap: ty.Dict[str, ty.Any] = {}
recap["words"] = words
recap["hypa"] = hypa
recap["model_param"] = model_param
recap["use_validation"] = use_validation
recap["model_name"] = model_name
recap["version"] = "001"
# logg.debug(f"recap: {recap}")
recap_path = info_folder / "recap.json"
recap_path.write_text(json.dumps(recap, indent=4))
results = model.fit(
x,
y,
validation_data=val_data,
epochs=epoch_num,
batch_size=batch_size,
callbacks=callbacks,
)
results_recap: ty.Dict[str, ty.Any] = {}
results_recap["model_name"] = model_name
results_recap["results_recap_version"] = "002"
# eval performance on the various metrics
eval_testing = model.evaluate(data["testing"], labels["testing"])
for metrics_name, value in zip(model.metrics_names, eval_testing):
logg.debug(f"{metrics_name}: {value}")
results_recap[metrics_name] = value
# compute the confusion matrix
y_pred = model.predict(data["testing"])
cm = pred_hot_2_cm(labels["testing"], y_pred, words)
# logg.debug(f"cm: {cm}")
results_recap["cm"] = cm.tolist()
# compute the fscore
fscore = analyze_confusion(cm, words)
logg.debug(f"fscore: {fscore}")
results_recap["fscore"] = fscore
# save the histories
results_recap["history_train"] = {
mn: results.history[mn]
for mn in model.metrics_names
}
if use_validation:
results_recap["history_val"] = {
f"val_{mn}": results.history[f"val_{mn}"]
for mn in model.metrics_names
}
# plot the cm
fig, ax = plt.subplots(figsize=(12, 12))
plot_confusion_matrix(cm, ax, model_name, words, fscore)
plot_cm_path = info_folder / "test_confusion_matrix.png"
fig.savefig(plot_cm_path)
plt.close(fig)
# save the results
res_recap_path = info_folder / "results_recap.json"
res_recap_path.write_text(json.dumps(results_recap, indent=4))
# if cyclic_lr was used save the history
if lr_value.startswith("clr_triangular2"):
logg.debug(f"cyclic_lr.history.keys(): {cyclic_lr.history.keys()}")
clr_recap = {}
for metric_name, values in cyclic_lr.history.items():
clr_recap[metric_name] = list(float(v) for v in values)
clr_recap_path = info_folder / "clr_recap.json"
clr_recap_path.write_text(json.dumps(clr_recap, indent=4))
# save the trained model
model.save(model_path)
placeholder_path.write_text(f"Trained. F-score: {fscore}")
def find_best_lr(hypa: ty.Dict[str, str]) -> None:
"""MAKEDOC: what is find_best_lr doing?"""
logg = logging.getLogger(f"c.{__name__}.find_best_lr")
# logg.setLevel("INFO")
logg.debug("Start find_best_lr")
# get the word list
words = words_types[hypa["words_type"]]
num_labels = len(words)
# load data
processed_folder = Path("data_proc")
processed_path = processed_folder / f"{hypa['dataset_name']}"
data, labels = load_processed(processed_path, words)
# no need for validation
x = np.concatenate((data["training"], data["validation"]))
y = np.concatenate((labels["training"], labels["validation"]))
# the shape of each sample
input_shape = data["training"][0].shape
# from hypa extract model param
model_param = get_model_param_attention(hypa, num_labels, input_shape)
# magic to fix the GPUs
setup_gpus()
model = AttentionModel(**model_param)
# model.summary()
start_lr = 1e-9
end_lr = 1e1
batch_size_types = {"01": 32, "02": 16}
batch_size = batch_size_types[hypa["batch_size_type"]]
epoch_num_types = {"01": 15, "02": 30, "03": 2}
epoch_num = epoch_num_types[hypa["epoch_num_type"]]
optimizer_types = {"a1": Adam(), "r1": RMSprop()}
opt = optimizer_types[hypa["optimizer_type"]]
metrics = [
tf.keras.metrics.CategoricalAccuracy(),
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall(),
]
model.compile(
optimizer=opt,
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=metrics,
)
# find the best values
lrf = LearningRateFinder(model)
lrf.find((x, y), start_lr, end_lr, epochs=epoch_num, batchSize=batch_size)
model_name = build_attention_name(hypa, False)
fig_title = "LR_sweep"
fig_title += f"_bs{batch_size}"
fig_title += f"_en{epoch_num}"
fig_title += f"__{model_name}"
fig, ax = plt.subplots(figsize=(8, 8))
# get the plot
lrf.plot_loss(ax=ax, title=fig_title)
# save the plot
plot_fol = Path("plot_results") / "att" / "find_best_lr"
if not plot_fol.exists():
plot_fol.mkdir(parents=True, exist_ok=True)
fig_name = fig_title + ".{}"
fig.savefig(plot_fol / fig_name.format("png"))
fig.savefig(plot_fol / fig_name.format("pdf"))
# TODO: save the loss history
plt.show()
def evaluate_model_cnn(which_dataset: str, train_words_type: str,
test_words_type: str) -> None:
"""MAKEDOC: what is evaluate_model_cnn doing?"""
logg = logging.getLogger(f"c.{__name__}.evaluate_model_cnn")
# logg.setLevel("INFO")
logg.debug("Start evaluate_model_cnn")
# magic to fix the GPUs
setup_gpus()
# setup the parameters
# hypa: ty.Dict[str, ty.Union[str, int]] = {}
# hypa["base_dense_width"] = 32
# hypa["base_filters"] = 20
# hypa["batch_size"] = 32
# hypa["dropout_type"] = "01"
# # hypa["epoch_num"] = 16
# hypa["epoch_num"] = 15
# hypa["kernel_size_type"] = "02"
# # hypa["pool_size_type"] = "02"
# hypa["pool_size_type"] = "01"
# # hypa["learning_rate_type"] = "02"
# hypa["learning_rate_type"] = "04"
# hypa["optimizer_type"] = "a1"
# hypa["dataset"] = which_dataset
# hypa["words"] = train_words_type
# hypa: ty.Dict[str, ty.Union[str, int]] = {}
# hypa["base_dense_width"] = 32
# hypa["base_filters"] = 32
# hypa["batch_size"] = 32
# hypa["dropout_type"] = "02"
# hypa["epoch_num"] = 15
# hypa["kernel_size_type"] = "02"
# hypa["pool_size_type"] = "01"
# hypa["learning_rate_type"] = "04"
# hypa["optimizer_type"] = "a1"
# hypa["dataset"] = which_dataset
# hypa["words"] = train_words_type
hypa: ty.Dict[str, ty.Union[str, int]] = {
"base_dense_width": 32,
"base_filters": 32,
"batch_size": 32,
# "dataset": "aug07",
"dropout_type": "01",
"epoch_num": 15,
"kernel_size_type": "02",
"learning_rate_type": "04",
"optimizer_type": "a1",
"pool_size_type": "01",
# "words": "all",
}
hypa["dataset"] = which_dataset
hypa["words"] = train_words_type
# get the words
# train_words = words_types[train_words_type]
test_words = words_types[test_words_type]
model_name = build_cnn_name(hypa)
logg.debug(f"model_name: {model_name}")
model_folder = Path("trained_models") / "cnn"
model_path = model_folder / f"{model_name}.h5"
if not model_path.exists():
logg.error(f"Model not found at: {model_path}")
raise FileNotFoundError
model = tf.keras.models.load_model(model_path)
model.summary()
# input data
processed_path = Path("data_proc") / f"{which_dataset}"
data, labels = load_processed(processed_path, test_words)
logg.debug(f"data['testing'].shape: {data['testing'].shape}")
# evaluate on the words you trained on
logg.debug("Evaluate on test data:")
model.evaluate(data["testing"], labels["testing"])
# model.evaluate(data["validation"], labels["validation"])
# predict labels/cm/fscore
y_pred = model.predict(data["testing"])
cm = pred_hot_2_cm(labels["testing"], y_pred, test_words)
# y_pred = model.predict(data["validation"])
# cm = pred_hot_2_cm(labels["validation"], y_pred, test_words)
fscore = analyze_confusion(cm, test_words)
logg.debug(f"fscore: {fscore}")
fig, ax = plt.subplots(figsize=(12, 12))
plot_confusion_matrix(cm, ax, model_name, test_words, fscore)
plt.show()
def evaluate_model_cm(model_name: str, test_words_type: str) -> float:
r"""MAKEDOC: what is evaluate_model_cm doing?"""
logg = logging.getLogger(f"c.{__name__}.evaluate_model_cm")
# logg.setLevel("INFO")
# logg.debug("\nStart evaluate_model_cm")
# magic to fix the GPUs
setup_gpus()
logg.debug(f"\nmodel_name: {model_name}")
dataset_re = re.compile("_ds(.*?)_")
match = dataset_re.search(model_name)
if match is not None:
logg.debug(f"match[1]: {match[1]}")
dataset_name = match[1]
train_words_type_re = re.compile("_w(.*?)[_.]")
match = train_words_type_re.search(model_name)
if match is not None:
logg.debug(f"match[1]: {match[1]}")
train_words_type = match[1]
arch_type = model_name[:3]
if arch_type == "ATT":
train_type_tag = "attention"
else:
train_type_tag = "area"
# load the model
model_folder = Path("trained_models") / train_type_tag
model_path = model_folder / f"{model_name}.h5"
model = tf_models.load_model(model_path)
# model.summary()
train_words = words_types[train_words_type]
logg.debug(f"train_words: {train_words}")
test_words = words_types[test_words_type]
logg.debug(f"test_words: {test_words}")
# input data must exist
if dataset_name.startswith("mel"):
preprocess_spec(dataset_name, test_words_type)
elif dataset_name.startswith("aug"):
do_augmentation(dataset_name, test_words_type)
# input data
processed_path = Path("data_proc") / f"{dataset_name}"
data, labels = load_processed(processed_path, test_words)
logg.debug(f"list(data.keys()): {list(data.keys())}")
logg.debug(f"data['testing'].shape: {data['testing'].shape}")
# evaluate on the words you trained on
logg.debug("Evaluate on test data:")
model.evaluate(data["testing"], labels["testing"])
# model.evaluate(data["validation"], labels["validation"])
# predict labels/cm/fscore
y_pred = model.predict(data["testing"])
cm = pred_hot_2_cm(labels["testing"], y_pred, test_words)
# y_pred = model.predict(data["validation"])
# cm = pred_hot_2_cm(labels["validation"], y_pred, test_words)
fscore = analyze_confusion(cm, test_words)
logg.debug(f"fscore: {fscore}")
fig, ax = plt.subplots(figsize=(12, 12))
plot_confusion_matrix(cm, ax, model_name, test_words, fscore, train_words)
fig_name = f"{model_name}_test{test_words_type}_cm.{{}}"
cm_folder = Path("plot_results") / "cm_all01"
if not cm_folder.exists():
cm_folder.mkdir(parents=True, exist_ok=True)
plot_cm_path = cm_folder / fig_name.format("png")
fig.savefig(plot_cm_path)
plot_cm_path = cm_folder / fig_name.format("pdf")
fig.savefig(plot_cm_path)
# plt.show()
return fscore
def evaluate_attention_weights(train_words_type: str,
rec_words_type: str,
do_new_record: bool = False) -> None:
"""MAKEDOC: what is evaluate_attention_weights doing?"""
logg = logging.getLogger(f"c.{__name__}.evaluate_attention_weights")
# logg.setLevel("INFO")
logg.debug("Start evaluate_attention_weights")
# magic to fix the GPUs
setup_gpus()
# ATT_ct02_dr02_ks02_lu01_as01_qt01_dw01_opa1_lr01_bs01_en01_dsmel04_wk1
# hypa: ty.Dict[str, str] = {}
# hypa["conv_size_type"] = "02"
# hypa["dropout_type"] = "02"
# hypa["kernel_size_type"] = "02"
# hypa["lstm_units_type"] = "01"
# hypa["query_style_type"] = "01"
# hypa["dense_width_type"] = "01"
# hypa["optimizer_type"] = "a1"
# hypa["learning_rate_type"] = "01"
# hypa["batch_size_type"] = "01"
# hypa["epoch_num_type"] = "01"
# dataset_name = "mel04"
# hypa["dataset_name"] = dataset_name
# hypa["words_type"] = train_words_type
# use_validation = True
# ATT_ct02_dr01_ks01_lu01_qt05_dw01_opa1_lr03_bs02_en02_dsaug07_wLTnum
hypa = {
"batch_size_type": "02",
"conv_size_type": "02",
"dataset_name": "aug07",
"dense_width_type": "01",
"dropout_type": "01",
"epoch_num_type": "02",
"kernel_size_type": "01",
"learning_rate_type": "03",
"lstm_units_type": "01",
"optimizer_type": "a1",
"query_style_type": "05",
"words_type": "LTnum",
}
use_validation = True
dataset_name = hypa["dataset_name"]
model_name = build_attention_name(hypa, use_validation)
logg.debug(f"model_name: {model_name}")
# load the model
model_folder = Path("trained_models") / "attention"
model_path = model_folder / f"{model_name}.h5"
# model = tf.keras.models.load_model(model_path)
# https://github.com/keras-team/keras/issues/5088#issuecomment-401498334
model = tf.keras.models.load_model(
model_path, custom_objects={"backend": tf.keras.backend})
model.summary()
logg.debug(f"ascii_model(model): {ascii_model(model)}")
att_weight_model = tf.keras.models.Model(
inputs=model.input,
outputs=[
model.get_layer("output").output,
model.get_layer("att_softmax").output,
model.get_layer("bidirectional_1").output,
],
)
att_weight_model.summary()
# logg.debug(f"att_weight_model.outputs: {att_weight_model.outputs}")
# get the training words
train_words = words_types[train_words_type]
# logg.debug(f"train_words: {train_words}")
perm_pred = compute_permutation(train_words)
rec_words_type = args.rec_words_type
if rec_words_type == "train":
rec_words = train_words[-3:]
# rec_words = train_words[:]
logg.debug(f"Using rec_words: {rec_words}")
else:
rec_words = words_types[rec_words_type]
num_rec_words = len(rec_words)
# record new audios
if do_new_record:
# where to save the audios
audio_folder = Path("recorded_audio")
if not audio_folder.exists():
audio_folder.mkdir(parents=True, exist_ok=True)
# record the audios and save them in audio_folder
audio_path_fmt = "{}_02.wav"
audios = record_audios(rec_words,
audio_folder,
audio_path_fmt,
timeout=0)
# compute the spectrograms and build the dataset of correct shape
img_specs = []
spec_dict = get_spec_dict()
spec_kwargs = spec_dict[dataset_name]
p2d_kwargs = {"ref": np.max}
for word in rec_words:
# get the name
audio_path = audio_folder / audio_path_fmt.format(word)
# convert it to mel
log_spec = wav2mel(audio_path, spec_kwargs, p2d_kwargs)
img_spec = log_spec.reshape((*log_spec.shape, 1))
# logg.debug(f"img_spec.shape: {img_spec.shape}")
# img_spec.shape: (128, 32, 1)
img_specs.append(img_spec)
# the data needs to look like this data['testing'].shape: (735, 128, 32, 1)
rec_data = np.stack(img_specs)
# logg.debug(f"rec_data.shape: {rec_data.shape}")
# load data if you do not want to record new audios
else:
# input data
processed_folder = Path("data_proc")
processed_path = processed_folder / f"{dataset_name}"
# which word in the dataset to plot
word_id = 2
# the loaded spectrograms
rec_data_l: ty.List[np.ndarray] = []
for i, word in enumerate(rec_words):
data, labels = load_processed(processed_path, [word])
# get one of the spectrograms
word_data = data["testing"][word_id]
rec_data_l.append(word_data)
# turn the list into np array
rec_data = np.stack(rec_data_l)
# get prediction and attention weights
pred, att_weights, LSTM_out = att_weight_model.predict(rec_data)
# logg.debug(f"att_weights.shape: {att_weights.shape}")
# logg.debug(f"att_weights[0].shape: {att_weights[0].shape}")
# if we recorded fresh audios we also have the waveform to plot
ax_add = 1 if do_new_record else 0
# plot the wave, spectrogram, weights and predictions in each column
plot_size = 5
fw = plot_size * num_rec_words
nrows = 3 + ax_add
# nrows = 4 + ax_add
fh = plot_size * nrows * 0.7
fig, axes = plt.subplots(nrows=nrows,
ncols=num_rec_words,
figsize=(fw, fh),
sharey="row")
fig.suptitle(f"Attention weights and predictions for {rec_words}",
fontsize=20)
for i, word in enumerate(rec_words):
word_spec = rec_data[i][:, :, 0]
# logg.debug(f"word_spec.shape: {word_spec.shape}")
# plot the waveform
if do_new_record:
plot_waveform(audios[i], axes[0][i])
# plot the spectrogram
title = f"Spectrogram for {word}"
plot_spec(word_spec, axes[0 + ax_add][i], title=title)
# plot the weights
word_att_weights = att_weights[i]
# plot_att_weights(word_att_weights, axes[1 + ax_add][i], title)
word_att_weights_img = np.expand_dims(word_att_weights, axis=-1).T
axes[1 + ax_add][i].imshow(word_att_weights_img,
origin="lower",
aspect="auto")
title = f"Attention weights for {word}"
axes[1 + ax_add][i].set_title(title)
# plot the predictions
word_pred = pred[i]
# permute the prediction from sorted to the order you have
word_pred = word_pred[perm_pred]
pred_index = np.argmax(word_pred)
title = f"Predictions for {word}"
plot_pred(word_pred, train_words, axes[2 + ax_add][i], title,
pred_index)
# axes[3 + ax_add][i].imshow(LSTM_out[i], origin="lower")
# fig.tight_layout()
fig.tight_layout(h_pad=3, rect=[0, 0.03, 1, 0.97])
fig_name = f"{model_name}"
fig_name += f"_{train_words_type}"
fig_name += f"_{rec_words_type}_img"
if do_new_record:
fig_name += "_new.{}"
else:
fig_name += "_data.{}"
plot_folder = Path("plot_results")
results_path = plot_folder / fig_name.format("png")
fig.savefig(results_path)
results_path = plot_folder / fig_name.format("pdf")
fig.savefig(results_path)
if num_rec_words <= 6:
plt.show()
def find_best_lr(hypa: ty.Dict[str, str]) -> None:
"""MAKEDOC: what is find_best_lr doing?"""
logg = logging.getLogger(f"c.{__name__}.find_best_lr")
# logg.setLevel("INFO")
logg.debug("Start find_best_lr")
# get the word list
words = words_types[hypa["words_type"]]
num_labels = len(words)
# no validation just find the LR
use_validation = False
# name the model
model_name = build_area_name(hypa, use_validation)
logg.debug(f"model_name: {model_name}")
# load data
processed_folder = Path("data_proc")
processed_path = processed_folder / f"{hypa['dataset_name']}"
data, labels = load_processed(processed_path, words)
# the shape of each sample
input_shape = data["training"][0].shape
# from hypa extract model param
model_param = get_model_param_area(hypa, num_labels, input_shape)
# no need for validation
x = np.concatenate((data["training"], data["validation"]))
y = np.concatenate((labels["training"], labels["validation"]))
# magic to fix the GPUs
setup_gpus()
# get the model with the chosen params
net_type = hypa["net_type"]
if net_type == "ARN":
model = AreaNet.build(**model_param)
elif net_type == "AAN":
model = ActualAreaNet.build(**model_param)
elif net_type == "VAN":
model = VerticalAreaNet.build(**model_param)
elif net_type.startswith("SI"):
if net_type == "SIM":
sim_type = "1"
elif net_type == "SI2":
sim_type = "2"
model = SimpleNet.build(sim_type=sim_type, **model_param)
num_samples = x.shape[0]
logg.debug(f"num_samples: {num_samples}")
# from hypa extract training param (epochs, batch, opt, ...)
training_param = get_training_param_area(hypa,
use_validation,
model_path=None,
num_samples=num_samples)
# a few metrics to track
metrics = [
tf.keras.metrics.CategoricalAccuracy(),
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall(),
]
# compile the model
model.compile(
optimizer=training_param["opt"],
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=metrics,
)
# boundary values
start_lr = 1e-9
end_lr = 1e1
# find the best values
lrf = LearningRateFinder(model)
lrf.find(
(x, y),
start_lr,
end_lr,
epochs=training_param["epochs"],
batchSize=training_param["batch_size"],
)
fig_title = "LR_sweep"
fig_title += f"__{model_name}"
fig, ax = plt.subplots(figsize=(8, 8))
# get the plot
lrf.plot_loss(ax=ax, title=fig_title)
# save the plot
plot_fol = Path("plot_results") / "area" / "find_best_lr"
if not plot_fol.exists():
plot_fol.mkdir(parents=True, exist_ok=True)
fig_name = fig_title + ".{}"
fig.savefig(plot_fol / fig_name.format("png"))
fig.savefig(plot_fol / fig_name.format("pdf"))
recap_loss = {}
recap_loss["lrs"] = [float(lr) for lr in lrf.lrs[:]]
recap_loss["losses"] = [float(loss) for loss in lrf.losses[:]]
loss_path = plot_fol / f"loss_{fig_title}.json"
loss_path.write_text(json.dumps(recap_loss, indent=4))
plt.show()
def train_area(
hypa: ty.Dict[str, str],
force_retrain: bool,
use_validation: bool,
trained_folder: Path,
root_info_folder: Path,
) -> None:
"""MAKEDOC: what is train_area doing?"""
logg = logging.getLogger(f"c.{__name__}.train_area")
# logg.setLevel("INFO")
logg.debug("Start train_area")
##########################################################
# Setup folders
##########################################################
# name the model
model_name = build_area_name(hypa, use_validation)
logg.debug(f"model_name: {model_name}")
# save the trained model here
model_path = trained_folder / f"{model_name}.h5"
placeholder_path = trained_folder / f"{model_name}.txt"
# check if this model has already been trained
if placeholder_path.exists():
if force_retrain:
logg.warn("\nRETRAINING MODEL!!\n")
else:
logg.debug("Already trained")
return
# save info regarding the model training in this folder
model_info_folder = root_info_folder / model_name
if not model_info_folder.exists():
model_info_folder.mkdir(parents=True, exist_ok=True)
# magic to fix the GPUs
setup_gpus()
##########################################################
# Load data
##########################################################
# get the words
words = words_types[hypa["words_type"]]
num_labels = len(words)
# load data
processed_folder = Path("data_proc")
processed_path = processed_folder / f"{hypa['dataset_name']}"
data, labels = load_processed(processed_path, words)
# concatenate train and val for final train
val_data = None
if use_validation:
x = data["training"]
y = labels["training"]
val_data = (data["validation"], labels["validation"])
logg.debug("Using validation data")
else:
x = np.concatenate((data["training"], data["validation"]))
y = np.concatenate((labels["training"], labels["validation"]))
logg.debug("NOT using validation data")
##########################################################
# Setup model
##########################################################
# the shape of each sample
input_shape = data["training"][0].shape
# from hypa extract model param
model_param = get_model_param_area(hypa, num_labels, input_shape)
# get the model with the chosen params
net_type = hypa["net_type"]
if net_type == "ARN":
model = AreaNet.build(**model_param)
elif net_type == "AAN":
model = ActualAreaNet.build(**model_param)
elif net_type == "VAN":
model = VerticalAreaNet.build(**model_param)
elif net_type.startswith("SI"):
if net_type == "SIM":
sim_type = "1"
elif net_type == "SI2":
sim_type = "2"
model = SimpleNet.build(sim_type=sim_type, **model_param)
num_samples = x.shape[0]
logg.debug(f"num_samples: {num_samples}")
# from hypa extract training param (epochs, batch, opt, ...)
training_param = get_training_param_area(hypa, use_validation, model_path,
num_samples)
# a few metrics to track
metrics = [
tf.keras.metrics.CategoricalAccuracy(),
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall(),
]
# compile the model
model.compile(
optimizer=training_param["opt"],
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=metrics,
)
# recap
recap: ty.Dict[str, ty.Any] = {}
recap["model_name"] = model_name
recap["words"] = words
recap["hypa"] = hypa
recap["model_param"] = model_param
recap["use_validation"] = use_validation
recap["batch_size"] = training_param["batch_size"]
recap["epochs"] = training_param["epochs"]
recap["lr_name"] = training_param["lr_name"]
recap["version"] = "002"
# logg.debug(f"recap: {recap}")
recap_path = model_info_folder / "recap.json"
recap_path.write_text(json.dumps(recap, indent=4))
# https://stackoverflow.com/a/45546663/2237151
model_summary_path = model_info_folder / "model_summary.txt"
with model_summary_path.open("w") as msf:
model.summary(line_length=150, print_fn=lambda x: msf.write(x + "\n"))
##########################################################
# Fit model
##########################################################
results = model.fit(
x,
y,
validation_data=val_data,
epochs=training_param["epochs"],
batch_size=training_param["batch_size"],
callbacks=training_param["callbacks"],
)
##########################################################
# Save results, history, performance
##########################################################
# results_recap
results_recap: ty.Dict[str, ty.Any] = {}
results_recap["model_name"] = model_name
results_recap["results_recap_version"] = "001"
# evaluate performance
eval_testing = model.evaluate(data["testing"], labels["testing"])
for metrics_name, value in zip(model.metrics_names, eval_testing):
logg.debug(f"{metrics_name}: {value}")
results_recap[metrics_name] = value
# confusion matrix
y_pred = model.predict(data["testing"])
cm = pred_hot_2_cm(labels["testing"], y_pred, words)
results_recap["cm"] = cm.tolist()
# fscore
fscore = analyze_confusion(cm, words)
logg.debug(f"fscore: {fscore}")
results_recap["fscore"] = fscore
# save the histories
results_recap["history_train"] = {
mn: results.history[mn]
for mn in model.metrics_names
}
if use_validation:
results_recap["history_val"] = {
f"val_{mn}": results.history[f"val_{mn}"]
for mn in model.metrics_names
}
# save the results
res_recap_path = model_info_folder / "results_recap.json"
res_recap_path.write_text(json.dumps(results_recap, indent=4))
# plot the cm
fig, ax = plt.subplots(figsize=(12, 12))
plot_confusion_matrix(cm, ax, model_name, words, fscore)
plot_cm_path = model_info_folder / "test_confusion_matrix.png"
fig.savefig(plot_cm_path)
plt.close(fig)
# save the trained model
model.save(model_path)
# save the placeholder
placeholder_path.write_text(f"Trained. F-score: {fscore}")
