def run_experiment(self):
"""
Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch
:return: The summary current_epoch_losses from starting epoch to total_epochs.
"""
total_losses = {
"train_acc": [],
"train_loss": [],
"val_acc": [],
"val_loss": []
} # initialize a dict to keep the per-epoch metrics
for i, epoch_idx in enumerate(
range(self.starting_epoch, self.num_epochs)):
epoch_start_time = time.time()
current_epoch_losses = {
"train_acc": [],
"train_loss": [],
"val_acc": [],
"val_loss": []
}
with tqdm.tqdm(
total=self.train_data.num_batches
) as pbar_train: # create a progress bar for training
for idx, (x,
y) in enumerate(self.train_data): # get data batches
loss, accuracy = self.run_train_iter(
x=x, y=y) # take a training iter step
current_epoch_losses["train_loss"].append(loss.cpu(
)) # add current iter loss to the train loss list
current_epoch_losses["train_acc"].append(
accuracy) # add current iter acc to the train acc list
pbar_train.update(1)
pbar_train.set_description(
"loss: {:.4f}, accuracy: {:.4f}".format(
loss, accuracy))
with tqdm.tqdm(
total=self.val_data.num_batches
) as pbar_val: # create a progress bar for validation
for x, y in self.val_data: # get data batches
loss, accuracy = self.run_evaluation_iter(
x=x, y=y) # run a validation iter
current_epoch_losses["val_loss"].append(
loss.cpu()) # add current iter loss to val loss list.
current_epoch_losses["val_acc"].append(
accuracy) # add current iter acc to val acc lst.
pbar_val.update(1) # add 1 step to the progress bar
pbar_val.set_description(
"loss: {:.4f}, accuracy: {:.4f}".format(
loss, accuracy))
val_mean_accuracy = np.mean(current_epoch_losses['val_acc'])
if val_mean_accuracy > self.best_val_model_acc: # if current epoch's mean val acc is greater than the saved best val acc then
self.best_val_model_acc = val_mean_accuracy # set the best val model acc to be current epoch's val accuracy
self.best_val_model_idx = epoch_idx # set the experiment-wise best val idx to be the current epoch's idx
for key, value in current_epoch_losses.items():
total_losses[key].append(
np.mean(value)
) # get mean of all metrics of current epoch metrics dict, to get them ready for storage and output on the terminal.
save_statistics(experiment_log_dir=self.experiment_logs,
filename='summary.csv',
stats_dict=total_losses,
current_epoch=i,
continue_from_mode=True if
(self.starting_epoch != 0 or i > 0) else
False) # save statistics to stats file.
# load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to
out_string = "_".join([
"{}_{:.4f}".format(key, np.mean(value))
for key, value in current_epoch_losses.items()
])
# create a string to use to report our epoch metrics
epoch_elapsed_time = time.time(
) - epoch_start_time # calculate time taken for epoch
epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time)
print("Epoch {}:".format(epoch_idx), out_string, "epoch time",
epoch_elapsed_time, "seconds")
self.save_model(
model_save_dir=self.experiment_saved_models,
# save model and best val idx and best val acc, using the model dir, model name and model idx
model_save_name="train_model",
model_idx=epoch_idx,
best_validation_model_idx=self.best_val_model_idx,
best_validation_model_acc=self.best_val_model_acc)
print("Generating test set evaluation metrics")
self.load_model(
model_save_dir=self.experiment_saved_models,
model_idx=self.best_val_model_idx,
# load best validation model
model_save_name="train_model")
print(self.best_val_model_idx)
current_epoch_losses = {
"test_acc": [],
"test_f1_score": []
} # initialize a statistics dict
with tqdm.tqdm(total=self.test_data.num_batches
) as pbar_test: # ini a progress bar
for x, y in self.test_data: # sample batch
loss, accuracy = self.compute_Fscore(
x=x, y=y
) # compute loss and accuracy by running an evaluation step
current_epoch_losses["test_f1_score"].append(
loss) # save test loss
current_epoch_losses["test_acc"].append(
accuracy) # save test accuracy
pbar_test.update(1) # update progress bar status
pbar_test.set_description(
"f1_score: {:.4f}, accuracy: {:.4f}".format(
loss, accuracy)) # update progress bar string output
test_losses = {
key: [np.mean(value)]
for key, value in current_epoch_losses.items()
} # save test set metrics in dict format
save_statistics(
experiment_log_dir=self.experiment_logs,
filename='test_summary.csv',
# save test set metrics on disk in .csv format
stats_dict=test_losses,
current_epoch=0,
continue_from_mode=False)
return total_losses, test_losses
def run_experiment(self):
"""
Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch
:return: The summary current_epoch_losses from starting epoch to total_epochs.
"""
total_losses = {
"train_acc": [],
"train_loss": [],
"train_files": [],
"train_labels": [],
"val_acc": [],
"val_loss": [],
"val_files": [],
"val_labels": [],
"val_pred_labels": []
} # initialize a dict to keep the per-epoch metrics
for i, epoch_idx in enumerate(
range(self.starting_epoch, self.num_epochs)):
epoch_start_time = time.time()
current_epoch_losses = {
"train_acc": [],
"train_loss": [],
"train_files": [],
"train_labels": [],
"val_acc": [],
"val_loss": [],
"val_files": [],
"val_labels": [],
"val_pred_labels": []
}
with tqdm.tqdm(
total=self.train_data.num_batches
) as pbar_train: # create a progress bar for training
for idx, (x, y, f_n_train) in enumerate(
self.train_data): # get data batches
loss, accuracy = self.run_train_iter(
x=x, y=y) # take a training iter step
current_epoch_losses["train_loss"].append(
np.array(loss.cpu())
) # add current iter loss to the train loss list
current_epoch_losses["train_acc"].append(
accuracy) # add current iter acc to the train acc list
current_epoch_losses["train_files"].append(f_n_train)
current_epoch_losses["train_labels"].append(y)
pbar_train.update(1)
pbar_train.set_description(
"loss: {:.4f}, accuracy: {:.4f}".format(
loss, accuracy))
with tqdm.tqdm(
total=self.val_data.num_batches
) as pbar_val: # create a progress bar for validation
for x, y_val, f_n_val in self.val_data: # get data batches
loss, accuracy, y_targets, predicted_labels = self.run_evaluation_iter(
x=x, y=y_val) # run a validation iter
current_epoch_losses["val_loss"].append(
np.array(loss.cpu())
) # add current iter loss to val loss list.
current_epoch_losses["val_acc"].append(
accuracy) # add current iter acc to val acc lst.
current_epoch_losses["val_files"].append(f_n_val)
current_epoch_losses["val_labels"].append(y_targets)
current_epoch_losses["val_pred_labels"].append(
predicted_labels)
pbar_val.update(1) # add 1 step to the progress bar
pbar_val.set_description(
"loss: {:.4f}, accuracy: {:.4f}".format(
loss, accuracy))
val_mean_accuracy = np.mean(current_epoch_losses['val_acc'])
if val_mean_accuracy > self.best_val_model_acc: # if current epoch's mean val acc is greater than the saved best val acc then
self.best_val_model_acc = val_mean_accuracy # set the best val model acc to be current epoch's val accuracy
self.best_val_model_idx = epoch_idx # set the experiment-wise best val idx to be the current epoch's idx
total_losses["val_files"] = current_epoch_losses["val_files"]
total_losses["val_labels"] = current_epoch_losses["val_labels"]
total_losses["val_pred_labels"] = current_epoch_losses[
"val_pred_labels"]
total_losses["train_files"] = current_epoch_losses[
"train_files"]
total_losses["train_labels"] = current_epoch_losses[
"train_labels"]
for key, value in current_epoch_losses.items():
if key not in [
"val_files", "val_labels", "val_preds",
"val_pred_labels", "train_files", "train_labels",
"test_files", "test_labels", "test_pred_labels"
]:
total_losses[key].append(
np.mean(value)
) # get mean of all metrics of current epoch metrics dict, to get them ready for storage and output on the terminal.
save_statistics(experiment_log_dir=self.experiment_logs,
filename='summary.csv',
stats_dict=total_losses,
current_epoch=i) # save statistics to stats file.
# load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to
out_string = "_".join([
"{}_{:.4f}".format(key, np.mean(value))
for key, value in current_epoch_losses.items() if key not in [
"val_files", "val_labels", "val_preds", "val_pred_labels",
"train_files", "train_labels", "test_files", "test_labels",
"test_pred_labels"
]
])
# create a string to use to report our epoch metrics
epoch_elapsed_time = time.time(
) - epoch_start_time # calculate time taken for epoch
epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time)
print("Epoch {}:".format(epoch_idx), out_string, "epoch time",
epoch_elapsed_time, "seconds")
self.save_model(
model_save_dir=self.experiment_saved_models,
# save model and best val idx and best val acc, using the model dir, model name and model idx
model_save_name="train_model",
model_idx=epoch_idx,
best_validation_model_idx=self.best_val_model_idx,
best_validation_model_acc=self.best_val_model_acc)
total_losses["val_files"] = [
item for sublist in total_losses["val_files"] for item in sublist
]
total_losses["val_labels"] = [
item for sublist in total_losses["val_labels"] for item in sublist
]
#total_losses["val_preds"] = [item for sublist in total_losses["val_files"] for item in sublist]
total_losses["val_pred_labels"] = [
item for sublist in total_losses["val_pred_labels"]
for item in sublist
]
total_losses["train_files"] = [
item for sublist in total_losses["train_files"] for item in sublist
]
total_losses["train_labels"] = [
item for sublist in total_losses["train_labels"]
for item in sublist
]
# for i in total_losses["val_pred_labels"]:
# i = i.numpy()[0]
filename = self.experiment_folder + '/total_losses_pickle.pkl'
print("Generated file " + filename)
outfile = open(filename, 'wb')
pickle.dump(total_losses, outfile)
outfile.close()
print("Generating test set evaluation metrics")
self.load_model(
model_save_dir=self.experiment_saved_models,
model_idx=self.best_val_model_idx,
# load best validation model
model_save_name="train_model")
current_epoch_losses = {
"test_acc": [],
"test_loss": [],
"test_files": [],
"test_labels": [],
"test_pred_labels": []
} # initialize a statistics dict
with tqdm.tqdm(total=self.test_data.num_batches
) as pbar_test: # ini a progress bar
for x, y, f_name_test in self.test_data: # sample batch
loss, accuracy, y_targets, predicted_labels = self.run_evaluation_iter(
x=x, y=y
) # compute loss and accuracy by running an evaluation step
current_epoch_losses["test_loss"].append(np.array(
loss.cpu())) # save test loss
current_epoch_losses["test_acc"].append(
accuracy) # save test accuracy
current_epoch_losses["test_files"].append(
f_name_test) # save test accuracy
current_epoch_losses["test_labels"].append(y_targets)
current_epoch_losses["test_pred_labels"].append(
predicted_labels)
pbar_test.update(1) # update progress bar status
pbar_test.set_description(
"loss: {:.4f}, accuracy: {:.4f}".format(
loss, accuracy)) # update progress bar string output
filename = self.experiment_folder + '/total_losses_pickle_test.pkl'
print("Generated file " + filename)
outfile = open(filename, 'wb')
pickle.dump(current_epoch_losses, outfile)
outfile.close()
test_losses = {
key: [np.mean(value)]
for key, value in current_epoch_losses.items()
if key not in ["test_files", "test_labels", "test_pred_labels"]
} # save test set metrics in dict format
save_statistics(
experiment_log_dir=self.experiment_logs,
filename='test_summary.csv',
# save test set metrics on disk in .csv format
stats_dict=test_losses,
current_epoch=0)
return total_losses, test_losses