def evaluate(self):
print("Generating test set evaluation metrics")
self.load_model(
model_save_dir=self.experiment_saved_models,
model_idx=self.best_val_model_idx,
# load best validation model
model_save_name="train_model")
current_epoch_losses = {
"test_aps": [],
"test_loss": []
} # initialize a statistics dict
predictions = {"y_pred": []}
with tqdm.tqdm(
total=len(self.test_data)) as pbar_test: # ini a progress bar
for idx, batch in enumerate(self.test_data): # sample batch
desc_1_batch = batch['desc1'].to(self.device)
desc_2_batch = batch['desc2'].to(self.device)
img_1_batch = batch['image_1'].to(self.device)
img_2_batch = batch['image_2'].to(self.device)
y = batch['target']
x = [desc_1_batch, img_1_batch, desc_2_batch, img_2_batch]
loss, aps, predicted = self.run_evaluation_iter(
x=x, y=y
) # compute loss and accuracy by running an evaluation step
current_epoch_losses["test_loss"].append(
loss) # save test loss
current_epoch_losses["test_aps"].append(
aps) # save test accuracy
predicted = predicted.flatten()
for p in predicted:
predictions['y_pred'].append(p)
pbar_test.update(1) # update progress bar status
pbar_test.set_description(
"loss: {:.4f}, average precision score: {:.4f}".format(
loss, aps)) # update progress bar string output
test_losses = {
key: [np.mean(value)]
for key, value in current_epoch_losses.items()
} # save test set metrics in dict format
save_statistics(
experiment_log_dir=self.experiment_logs,
filename='test_summary.csv',
# save test set metrics on disk in .csv format
stats_dict=test_losses,
current_epoch=0,
continue_from_mode=False)
save_statistics(
experiment_log_dir=self.experiment_logs,
filename='test_predictions.csv',
# save test set metrics on disk in .csv format
save_full_dict=True,
stats_dict=predictions,
current_epoch=0,
continue_from_mode=False)
def output_summary(self, type, epoch_idx):
print("Generating {} set evaluation metrics".format(type))
self.load_model(model_save_dir=self.experiment_saved_models, model_idx=self.best_valid_model_idx,
# load best validation model
model_save_name="train_model")
current_epoch_losses = {type + "_loss": [], type+"_l_s": [], type+"_l_t": [],
type + "_acc": [], type+"_a_s": [], type+"_a_t": [],
type + "_f1": [], type+"_f_s": [], type+"_f_t": [],
type + "_prec": [], type + "_p_s": [], type + "_p_t": [],
type + "_rec": [], type+"_r_s": [], type+"_r_t": []} # initialize a statistics dict
current_epoch_losses, y_out, pred_out = self.run_epoch(current_epoch_losses, type, output=True, epoch_idx=epoch_idx)
# save test set metrics in dict format
losses = {}
losses[type+"_loss"] = [np.mean(current_epoch_losses[type+"_loss"])]
losses[type + "_l_s"] = [np.mean(current_epoch_losses[type + "_l_s"])]
losses[type + "_l_t"] = [np.mean(current_epoch_losses[type + "_l_t"])]
losses[type + "_acc"] = [np.mean(current_epoch_losses[type + "_acc"])]
losses[type + "_a_s"] = [np.mean(current_epoch_losses[type + "_a_s"])]
losses[type + "_a_t"] = [np.mean(current_epoch_losses[type + "_a_t"])]
losses[type + "_f1"] = [np.mean(current_epoch_losses[type + "_f1"])]
losses[type + "_f_s"] = [np.mean(current_epoch_losses[type + "_f_s"])]
losses[type + "_f_t"] = [np.mean(current_epoch_losses[type + "_f_t"])]
losses[type + "_prec"] = [np.mean(current_epoch_losses[type + "_prec"])]
losses[type + "_p_s"] = [np.mean(current_epoch_losses[type + "_p_s"])]
losses[type + "_p_t"] = [np.mean(current_epoch_losses[type + "_p_t"])]
losses[type + "_rec"] = [np.mean(current_epoch_losses[type + "_rec"])]
losses[type + "_r_s"] = [np.mean(current_epoch_losses[type + "_r_s"])]
losses[type + "_r_t"] = [np.mean(current_epoch_losses[type + "_r_t"])]
y_out = y_out.detach().cpu().numpy().flatten()
# losses[type+'_y'] = [str(y_arr)]
pred_out = pred_out.detach().cpu().numpy() #.flatten()
# pred_out = np.exp(pred_arr) # pred_arr is log probabilities, make probabilities
# losses[type+'_pred'] = [str(self.sigmoid_array(pred_arr))]
print(type + " evaluation:")
print("shape of y: ", y_out.shape)
print("shape of pred:", pred_out.shape)
out_path = os.path.join(self.experiment_logs, "{}_out.npz".format(type))
# print("Saving in file {}: y_out {}, and pred_out {}".format(out_path, y_out, pred_out))
np.savez(out_path, y=y_out, pred=pred_out)
save_statistics(experiment_log_dir=self.experiment_logs, filename=type+'_summary.csv',
# save test set metrics on disk in .csv format
stats_dict=losses, current_epoch=0, continue_from_mode=False)
return losses
def run_experiment(self):
"""
Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch
:return: The summary current_epoch_losses from starting epoch to total_epochs.
"""
total_losses = {
"train_acc": [],
"train_loss": [],
"val_acc": [],
"val_loss": [],
"curr_epoch": []
} # initialize a dict to keep the per-epoch metrics
for i, epoch_idx in enumerate(
range(self.starting_epoch, self.num_epochs)):
epoch_start_time = time.time()
current_epoch_losses = {
"train_acc": [],
"train_loss": [],
"val_acc": [],
"val_loss": []
}
with tqdm.tqdm(
total=len(self.train_data)
) as pbar_train: # create a progress bar for training
for idx, (x,
y) in enumerate(self.train_data): # get data batches
loss, accuracy = self.run_train_iter(
x=x, y=y) # take a training iter step
current_epoch_losses["train_loss"].append(
loss) # add current iter loss to the train loss list
current_epoch_losses["train_acc"].append(
accuracy) # add current iter acc to the train acc list
pbar_train.update(1)
pbar_train.set_description(
"loss: {:.4f}, accuracy: {:.4f}".format(
loss, accuracy))
with tqdm.tqdm(
total=len(self.val_data)
) as pbar_val: # create a progress bar for validation
for x, y in self.val_data: # get data batches
loss, accuracy = self.run_evaluation_iter(
x=x, y=y) # run a validation iter
current_epoch_losses["val_loss"].append(
loss) # add current iter loss to val loss list.
current_epoch_losses["val_acc"].append(
accuracy) # add current iter acc to val acc lst.
pbar_val.update(1) # add 1 step to the progress bar
pbar_val.set_description(
"loss: {:.4f}, accuracy: {:.4f}".format(
loss, accuracy))
val_mean_accuracy = np.mean(current_epoch_losses['val_acc'])
if val_mean_accuracy > self.best_val_model_acc: # if current epoch's mean val acc is greater than the saved best val acc then
self.best_val_model_acc = val_mean_accuracy # set the best val model acc to be current epoch's val accuracy
self.best_val_model_idx = epoch_idx # set the experiment-wise best val idx to be the current epoch's idx
for key, value in current_epoch_losses.items():
total_losses[key].append(
np.mean(value)
) # get mean of all metrics of current epoch metrics dict, to get them ready for storage and output on the terminal.
total_losses['curr_epoch'].append(epoch_idx)
save_statistics(experiment_log_dir=self.experiment_logs,
filename='summary.csv',
stats_dict=total_losses,
current_epoch=i,
continue_from_mode=True if
(self.starting_epoch != 0 or i > 0) else
False) # save statistics to stats file.
# load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to
out_string = "_".join([
"{}_{:.4f}".format(key, np.mean(value))
for key, value in current_epoch_losses.items()
])
# create a string to use to report our epoch metrics
epoch_elapsed_time = time.time(
) - epoch_start_time # calculate time taken for epoch
epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time)
print("Epoch {}:".format(epoch_idx), out_string, "epoch time",
epoch_elapsed_time, "seconds")
self.state['current_epoch_idx'] = epoch_idx
self.state['best_val_model_acc'] = self.best_val_model_acc
self.state['best_val_model_idx'] = self.best_val_model_idx
self.save_model(
model_save_dir=self.experiment_saved_models,
# save model and best val idx and best val acc, using the model dir, model name and model idx
model_save_name="train_model",
model_idx=epoch_idx,
state=self.state)
self.save_model(
model_save_dir=self.experiment_saved_models,
# save model and best val idx and best val acc, using the model dir, model name and model idx
model_save_name="train_model",
model_idx='latest',
state=self.state)
print("Generating test set evaluation metrics")
self.load_model(
model_save_dir=self.experiment_saved_models,
model_idx=self.best_val_model_idx,
# load best validation model
model_save_name="train_model")
current_epoch_losses = {
"test_acc": [],
"test_loss": []
} # initialize a statistics dict
with tqdm.tqdm(
total=len(self.test_data)) as pbar_test: # ini a progress bar
for x, y in self.test_data: # sample batch
loss, accuracy = self.run_evaluation_iter(
x=x, y=y
) # compute loss and accuracy by running an evaluation step
current_epoch_losses["test_loss"].append(
loss) # save test loss
current_epoch_losses["test_acc"].append(
accuracy) # save test accuracy
pbar_test.update(1) # update progress bar status
pbar_test.set_description(
"loss: {:.4f}, accuracy: {:.4f}".format(
loss, accuracy)) # update progress bar string output
test_losses = {
key: [np.mean(value)]
for key, value in current_epoch_losses.items()
} # save test set metrics in dict format
save_statistics(
experiment_log_dir=self.experiment_logs,
filename='test_summary.csv',
# save test set metrics on disk in .csv format
stats_dict=test_losses,
current_epoch=0,
continue_from_mode=False)
return total_losses, test_losses
def run_experiment(self):
"""
Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch
:return: The summary current_epoch_losses from starting epoch to total_epochs.
"""
total_losses = {
"train_acc": [],
"train_loss": [],
"val_acc": [],
"val_loss": [],
"curr_epoch": []
} # initialize a dict to keep the per-epoch metrics
for i, epoch_idx in enumerate(
range(self.starting_epoch, self.num_epochs)):
epoch_start_time = time.time()
current_epoch_losses = {
"train_acc": [],
"train_loss": [],
"val_acc": [],
"val_loss": []
}
current_epoch_losses = self.run_training_epoch(
current_epoch_losses)
current_epoch_losses = self.run_validation_epoch(
current_epoch_losses)
val_mean_accuracy = np.mean(current_epoch_losses['val_acc'])
if val_mean_accuracy > self.best_val_model_acc: # if current epoch's mean val acc is greater than the saved best val acc then
self.best_val_model_acc = val_mean_accuracy # set the best val model acc to be current epoch's val accuracy
self.best_val_model_idx = epoch_idx # set the experiment-wise best val idx to be the current epoch's idx
for key, value in current_epoch_losses.items():
total_losses[key].append(np.mean(value))
# get mean of all metrics of current epoch metrics dict,
# to get them ready for storage and output on the terminal.
total_losses['curr_epoch'].append(epoch_idx)
save_statistics(experiment_log_dir=self.experiment_logs,
filename='summary.csv',
stats_dict=total_losses,
current_epoch=i,
continue_from_mode=True if
(self.starting_epoch != 0 or i > 0) else
False) # save statistics to stats file.
# load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to
out_string = "_".join([
"{}_{:.4f}".format(key, np.mean(value))
for key, value in current_epoch_losses.items()
])
# create a string to use to report our epoch metrics
epoch_elapsed_time = time.time(
) - epoch_start_time # calculate time taken for epoch
epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time)
print("Epoch {}:".format(epoch_idx), out_string, "epoch time",
epoch_elapsed_time, "seconds")
self.state['current_epoch_idx'] = epoch_idx
self.state['best_val_model_acc'] = self.best_val_model_acc
self.state['best_val_model_idx'] = self.best_val_model_idx
self.save_model(
model_save_dir=self.experiment_saved_models,
# save model and best val idx and best val acc, using the model dir, model name and model idx
model_save_name="train_model",
model_idx=epoch_idx,
state=self.state)
self.save_model(
model_save_dir=self.experiment_saved_models,
# save model and best val idx and best val acc, using the model dir, model name and model idx
model_save_name="train_model",
model_idx='latest',
state=self.state)
print("Generating test set evaluation metrics")
self.load_model(
model_save_dir=self.experiment_saved_models,
model_idx=self.best_val_model_idx,
# load best validation model
model_save_name="train_model")
current_epoch_losses = {
"test_acc": [],
"test_loss": []
} # initialize a statistics dict
current_epoch_losses = self.run_testing_epoch(
current_epoch_losses=current_epoch_losses)
test_losses = {
key: [np.mean(value)]
for key, value in current_epoch_losses.items()
} # save test set metrics in dict format
save_statistics(
experiment_log_dir=self.experiment_logs,
filename='test_summary.csv',
# save test set metrics on disk in .csv format
stats_dict=test_losses,
current_epoch=0,
continue_from_mode=False)
return total_losses, test_losses
def run_experiment(self):
"""
Runs experiment train and evaluation iterations, saving the model and best val model and val model roc_auc after each cycle
:return: The summary current_cycle_losses from starting cycle to total_cycles.
"""
total_losses = {
"curr_cycle": [],
"train_loss": [],
"train_acc": [],
"val_loss": [],
"val_acc": [],
"val_roc_auc": []
} # initialize a dict to keep the per-epoch metrics
# new_cycle = False
current_cycle_losses = {
"train_loss": [],
"train_acc": [],
"train_bs": [],
"val_loss": [],
"val_acc": [],
"val_bs": []
}
cycle_start_time = time.time()
for i, epoch_idx in enumerate(
range(self.starting_epoch, self.num_epochs)):
''' TRAIN '''
with tqdm(total=len(self.train_data),
disable=bool(os.environ.get("DISABLE_TQDM", False))
) as pbar_train: # create a progress bar for training
for idx, batch in enumerate(
self.train_data): # get data batches
new_cycle = self.scheduler.get_lr(
self.global_step + 1,
self.scheduler.warmup) > self.scheduler.get_lr(
self.global_step, self.scheduler.warmup)
new_cycle &= self.scheduler.get_lr(
self.global_step - 1,
self.scheduler.warmup) > self.scheduler.get_lr(
self.global_step, self.scheduler.warmup)
new_cycle &= (self.global_step != 0)
if new_cycle:
print('New cycle:', new_cycle)
current_cycle_losses = {
"train_loss": [],
"train_acc": [],
"train_bs": [],
"val_loss": [],
"val_acc": [],
"val_bs": []
}
cycle_start_time = time.time()
self.iteration_step = idx
try:
loss, accuracy = self.run_train_iter(
batch) # take a training iter step
except RuntimeError as e:
if 'out of memory' in str(e):
print('| WARNING: ran out of memory, exiting')
for p in self.model.parameters():
del p # free all memory
_ = gc.collect()
torch.cuda.empty_cache()
return
else:
raise e
current_cycle_losses["train_bs"].append(len(
batch[0])) # add batch size!
current_cycle_losses["train_loss"].append(
loss) # add current iter loss to the train loss list
current_cycle_losses["train_acc"].append(
accuracy) # add current iter acc to the train acc list
pbar_train.update(1)
if not pbar_train.disable:
pbar_train.set_description("loss: {:.4f}".format(loss))
''' if new cycle start save!'''
# self.global_step-1 because it is incremented in the iteration loop before
if new_cycle:
''' VALIDATION '''
pred_all = []
y_all = []
with tqdm(
total=len(self.val_data),
disable=bool(
os.environ.get("DISABLE_TQDM", False))
) as pbar_val: # create a progress bar for validation
for batch in self.val_data: # get data batches
try:
loss, accuracy, target = self.run_evaluation_iter(
batch) # run a validation iter
except RuntimeError as e:
if 'out of memory' in str(e):
print(
'| WARNING: ran out of memory, exiting'
)
for p in self.model.parameters():
del p # free all memory
_ = gc.collect()
torch.cuda.empty_cache()
return
else:
raise e
# keep log
pred_all.append(target)
y_all.append(batch[-1])
current_cycle_losses["val_bs"].append(
len(batch[0])) # add batch size!
current_cycle_losses["val_loss"].append(
loss
) # add current iter loss to val loss list.
current_cycle_losses["val_acc"].append(
accuracy
) # add current iter acc to val acc lst.
pbar_val.update(
1) # add 1 step to the progress bar
if not pbar_val.disable:
pbar_val.set_description(
"loss: {:.4f}".format(loss))
''' SAVE LEARNING LOGS'''
# val_roc_auc = roc_auc(np.vstack(y_all), np.vstack(pred_all))
# current_cycle_losses["val_roc_auc"] = [val_roc_auc]
# REMINDER NEW: keep track of bias roc_auc
# val_roc_auc = #.get_final_metric()
try:
val_roc_auc = self.evaluator.get_final_metric(
np.vstack(pred_all)[:, 0])
current_cycle_losses["val_roc_auc"] = [val_roc_auc]
except:
print('Cannot compute bias val_roc_auc')
val_roc_auc = 0
current_cycle_losses["val_roc_auc"] = [val_roc_auc]
for key, value in current_cycle_losses.items():
if "bs" in key: continue
# get mean of all metrics of current epoch metrics dict, to get them ready for storage and output on the terminal.
# weights for incremental mean; or stacked evaluation
try:
total_losses[key].append(
np.average(
value,
axis=0,
weights=total_losses[key.split("_")[0]
+ "_bs"]))
except:
total_losses[key].append(
np.average(value, axis=0))
if self.val_metrics == 'loss':
val_main_metrics = total_losses['val_loss'][-1]
if val_main_metrics < self.best_val_model_metrics or self.cycle == 0: # if current epoch's mean val pargsaroc_auc is greater than the saved best val main metrics then
self.best_val_model_metrics = val_main_metrics # set the best val model pargsaroc_auc to be current epoch's val main metrics
# TODO: epoch_idx is ALREADY OFFSET!
self.best_val_model_idx = self.cycle # set the experiment-wise best val idx to be the current epoch's idx
else:
val_main_metrics = total_losses['val_acc'][
-1] if "acc" in self.val_metrics else total_losses[
'val_roc_auc'][-1]
if val_main_metrics > self.best_val_model_metrics or self.cycle == 0: # if current epoch's mean val pargsaroc_auc is greater than the saved best val main metrics then
self.best_val_model_metrics = val_main_metrics # set the best val model pargsaroc_auc to be current epoch's val main metrics
self.best_val_model_idx = self.cycle # set the experiment-wise best val idx to be the current epoch's idx
total_losses['curr_cycle'].append(self.cycle)
# print(total_losses)
save_statistics(
experiment_log_dir=self.experiment_logs,
filename='summary.csv',
stats_dict=total_losses,
current_epoch=self.cycle,
continue_from_mode=True if
(self.starting_epoch != 0 or self.cycle > 0) else
False) # save statistics to stats file.
# load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to
out_string = "\n".join([
"{}_{}".format(key, value[-1])
for key, value in total_losses.items()
])
# create a string to use to report our epoch metrics
cycle_elapsed_time = time.time(
) - cycle_start_time # calculate time taken for epoch
cycle_elapsed_time = "{:.4f}".format(
cycle_elapsed_time)
print("Cycle {}:".format(self.cycle), out_string,
"cycle time", cycle_elapsed_time, "seconds",
"best cycle", self.best_val_model_idx)
self.state['current_cycle_idx'] = self.cycle
self.state[
'best_val_model_roc_auc'] = self.best_val_model_metrics
self.state[
'best_val_model_idx'] = self.best_val_model_idx
# save model and best val idx and best val metrics, using the model dir, model name and model idx
if self.best_val_model_idx == self.cycle:
print('SAVING BEST', self.best_val_model_idx,
self.cycle)
self.patience = 0
self.save_model(
model_save_dir=self.experiment_saved_models,
model_save_name="train_model",
model_idx='best',
state=self.state)
else:
self.patience += 1
self.save_model(
model_save_dir=self.experiment_saved_models,
model_save_name="train_model",
model_idx=self.cycle,
state=self.state)
self.cycle += 1
if self.patience >= self.max_patience: # hard code patience
break
self.save_model(model_save_dir=self.experiment_saved_models,
model_save_name="train_model",
model_idx='latest',
state=self.state)
''' EXIT TRAINING, DO PREDICTION '''
if self.test_data is None:
pass
else:
return self.run_prediction()
def run_experiment(self):
"""
Runs experiment train and evaluation iterations, saving the model and best valid model and valid model accuracy after each epoch
:return: The summary current_epoch_losses from starting epoch to total_epochs.
"""
total_losses = {"train_loss": [], "train_l_s": [], "train_l_t": [],
"train_acc": [], "train_a_s": [], "train_a_t": [],
"train_f1": [], "train_f_s": [], "train_f_t": [],
"train_prec": [], "train_p_s": [], "train_p_t": [],
"train_rec": [], "train_r_s": [], "train_r_t": [],
"valid_loss": [], "valid_l_s": [], "valid_l_t": [],
"valid_acc": [], "valid_a_s": [], "valid_a_t": [],
"valid_f1": [], "valid_f_s": [], "valid_f_t": [],
"valid_prec": [], "valid_p_s": [], "valid_p_t": [],
"valid_rec": [], "valid_r_s": [], "valid_r_t": [],
"curr_epoch": []} # initialize a dict to keep the per-epoch metrics
for i, epoch_idx in enumerate(range(self.starting_epoch, self.starting_epoch + self.num_epochs)):
epoch_start_time = time.time()
current_epoch_losses = {"train_loss": [], "train_l_s": [], "train_l_t": [],
"train_acc": [], "train_a_s": [], "train_a_t": [],
"train_f1": [], "train_f_s": [], "train_f_t": [],
"train_prec": [], "train_p_s": [], "train_p_t": [],
"train_rec": [], "train_r_s": [], "train_r_t": [],
"valid_loss": [], "valid_l_s": [], "valid_l_t": [],
"valid_acc": [], "valid_a_s": [], "valid_a_t": [],
"valid_f1": [], "valid_f_s": [], "valid_f_t": [],
"valid_prec": [], "valid_p_s": [], "valid_p_t": [],
"valid_rec": [], "valid_r_s": [], "valid_r_t": []}
training_start_time = time.time()
print("debug_batch_cnt_max={}".format(self.debug_batch_cnt_max))
current_epoch_losses, _, _ = self.run_epoch(current_epoch_losses, "train", output=False, epoch_idx=epoch_idx)
print("Total epoch training time is {0:.2f} secs".format(time.time() - training_start_time))
validation_start_time = time.time()
current_epoch_losses, _, _ = self.run_epoch(current_epoch_losses, "valid", output=False, epoch_idx=epoch_idx)
print("Total epoch validation time is {0:.2f} secs".format(time.time() - validation_start_time))
# valid_mean_accuracy = np.mean(current_epoch_losses['valid_acc'])
valid_mean_acc = np.mean(current_epoch_losses['valid_acc'])
valid_mean_f1 = np.mean(current_epoch_losses['valid_f1'])
if valid_mean_acc > self.best_valid_model_acc: # if current epoch's mean valid acc is greater than the saved best valid acc then
self.best_valid_model_acc = valid_mean_acc # set the best valid model acc to be current epoch's valid acc
self.best_valid_model_f1 = valid_mean_f1
self.best_valid_model_idx = epoch_idx # set the experiment-wise best valid idx to be the current epoch's idx
for key, value in current_epoch_losses.items():
total_losses[key].append(np.mean(
value)) # get mean of all metrics of current epoch metrics dict, to get them ready for storage and output on the terminal.
total_losses['curr_epoch'].append(epoch_idx)
save_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv',
stats_dict=total_losses, current_epoch=i,
continue_from_mode=True if (self.starting_epoch != 0 or i > 0) else False) # save statistics to stats file.
# load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to
out_string = "_".join(
["{}_{:.4f}".format(key, np.mean(value)) for key, value in current_epoch_losses.items()])
# create a string to use to report our epoch metrics
self.state['current_epoch_idx'] = epoch_idx
self.state['best_valid_model_acc'] = self.best_valid_model_acc
self.state['best_valid_model_f1'] = self.best_valid_model_f1
self.state['best_valid_model_idx'] = self.best_valid_model_idx
self.save_model(model_save_dir=self.experiment_saved_models,
# save model and best valid idx and best valid acc, using the model dir, model name and model idx
model_save_name="train_model", model_idx=epoch_idx, state=self.state)
self.save_model(model_save_dir=self.experiment_saved_models,
# save model and best valid idx and best valid acc, using the model dir, model name and model idx
model_save_name="train_model", model_idx='latest', state=self.state)
print("Epoch {}: {}".format(epoch_idx, out_string))
print("Total epoch time is {0:.2f} secs".format(time.time() - epoch_start_time))
valid_losses = self.output_summary(type='valid', epoch_idx=epoch_idx)
test_losses = self.output_summary(type='test', epoch_idx=epoch_idx)
return total_losses, test_losses
def run_experiment(self):
"""
Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch
:return: The summary current_epoch_losses from starting epoch to total_epochs.
"""
total_losses = {
"val_Gender_acc": [],
"val_SER_acc": [],
"train_acc_GENDER": [],
"train_acc_SER": [],
"train_loss": [],
"val_loss": [],
"ua_train": [],
"ua_eval": []
} # initialize a dict to keep the per-epoch metrics
for i, epoch_idx in enumerate(
range(self.starting_epoch, self.num_epochs)):
epoch_start_time = time.time()
current_epoch_losses = {
"train_acc_SER": [],
"train_acc_GENDER": [],
"train_loss": [],
"val_Gender_acc": [],
"val_SER_acc": [],
"val_loss": []
}
self.current_epoch = epoch_idx
# init un_accuracy
self.current_epoch_emo_count = {0: 0, 1: 0, 2: 0, 3: 0}
self.current_epoch_correct_count = {0: 0, 1: 0, 2: 0, 3: 0}
with tqdm.tqdm(
total=len(self.train_data), ascii=True
) as pbar_train: # create a progress bar for training
try:
for idx, (x, y, z) in enumerate(
self.train_data): # get data batches
loss, accuracy1, accuracy2 = self.run_train_iter(
x=x, y=y, z=z) # take a training iter step
current_epoch_losses["train_loss"].append(
loss
) # add current iter loss to the train loss list
current_epoch_losses["train_acc_SER"].append(
accuracy1
) # add current iter acc to the train acc list
current_epoch_losses["train_acc_GENDER"].append(
accuracy2)
pbar_train.update(1)
pbar_train.set_description(
"loss: {:.4f}, accuracy_SER: {:.4f}, accuracy_Gender: {:.4f}"
.format(loss, accuracy1, accuracy2))
torch.cuda.empty_cache()
except KeyboardInterrupt:
pbar_train.close()
raise
pbar_train.close()
# get un_accuracy result
self.eval_current_epoch_emo_count = {0: 0, 1: 0, 2: 0, 3: 0}
self.eval_current_epoch_correct_count = {0: 0, 1: 0, 2: 0, 3: 0}
with tqdm.tqdm(
total=len(self.val_data), ascii=True
) as pbar_val: # create a progress bar for validation
try:
for x, y, z in self.val_data: # get data batches
loss, accuracy1, accuracy2 = self.run_evaluation_iter(
x=x, y=y, z=z) # run a validation iter
current_epoch_losses["val_loss"].append(
loss) # add current iter loss to val loss list.
current_epoch_losses["val_SER_acc"].append(
accuracy1) # add current iter acc to val acc lst.
current_epoch_losses["val_Gender_acc"].append(
accuracy2) # add current iter acc to val acc lst.
pbar_val.update(1) # add 1 step to the progress bar
pbar_val.set_description(
"loss: {:.4f}, accuracy_SER: {:.4f}, accuracy_Gender: {:.4f}"
.format(loss, accuracy1, accuracy2))
torch.cuda.empty_cache()
except KeyboardInterrupt:
pbar_val.close()
raise
pbar_val.close()
val_mean_accuracy = np.mean(current_epoch_losses['val_SER_acc'])
if val_mean_accuracy > self.best_val_model_acc: # if current epoch's mean val acc is greater than the saved best val acc then
self.best_val_model_acc = val_mean_accuracy # set the best val model acc to be current epoch's val accuracy
self.best_val_model_idx = epoch_idx # set the experiment-wise best val idx to be the current epoch's idx
for key, value in current_epoch_losses.items():
total_losses[key].append(
np.mean(value)
) # get mean of all metrics of current epoch metrics dict, to get them ready for storage and output on the terminal.
ua_train = 0
ua_val = 0
for key, value in self.current_epoch_correct_count.items():
ua_train += value / self.current_epoch_emo_count[key] / 4
total_losses["ua_train"].append(ua_train)
for key, value in self.eval_current_epoch_correct_count.items():
ua_val += value / self.eval_current_epoch_emo_count[key] / 4
total_losses["ua_eval"].append(ua_val)
save_statistics(experiment_log_dir=self.experiment_logs,
filename='summary.csv',
stats_dict=total_losses,
current_epoch=i,
continue_from_mode=True if
(self.starting_epoch != 0 or i > 0) else
False) # save statistics to stats file.
# load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to
out_string = ", ".join([
"{}_{:.4f}".format(key, np.mean(value))
for key, value in current_epoch_losses.items()
])
# create a string to use to report our epoch metrics
epoch_elapsed_time = time.time(
) - epoch_start_time # calculate time taken for epoch
epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time)
print("Epoch {}:".format(epoch_idx), out_string, "epoch time",
epoch_elapsed_time, "seconds")
self.state['model_epoch'] = epoch_idx
if (self.best_val_model_idx == epoch_idx):
# replace best model
#remove all model files, THEN save new best one.
for model_filename in os.listdir(self.experiment_saved_models):
current_model = os.path.join(self.experiment_saved_models,
model_filename)
os.remove(current_model)
self.save_model(
model_save_dir=self.experiment_saved_models,
# save model and best val idx and best val acc, using the model dir, model name and model idx
model_save_name="train_model",
model_idx=epoch_idx,
best_validation_model_idx=self.best_val_model_idx,
best_validation_model_acc=self.best_val_model_acc)
# save(update) latest model every epoch
self.save_model(
model_save_dir=self.experiment_saved_models,
# save model and best val idx and best val acc, using the model dir, model name and model idx
model_save_name="train_model",
model_idx='latest',
best_validation_model_idx=self.best_val_model_idx,
best_validation_model_acc=self.best_val_model_acc)
# early stopping
# criteria = int(self.num_epochs * 0.1)
# window_valloss = total_losses['val_loss'][-(criteria+1):]
# sorted_window_valloss = sorted(window_valloss,reverse = False)
#if (len(window_valloss) >= (criteria + 1)):
# if (sorted_window_valloss[0] == window_valloss[0]):
# print("Early Stop at Epoch {}:".format(epoch_idx), "Best val model index", self.best_val_model_idx, "Best val model accuracy", self.best_val_model_acc)
# break
print("Generating test set evaluation metrics")
self.load_model(
model_save_dir=self.experiment_saved_models,
model_idx=self.best_val_model_idx,
# # load best validation model
model_save_name="train_model")
current_epoch_losses = {
"test_Gender_acc": [],
"test_SER_acc": [],
"test_loss": []
} # initialize a statistics dict
self.eval_current_epoch_emo_count = {0: 0, 1: 0, 2: 0, 3: 0}
self.eval_current_epoch_correct_count = {0: 0, 1: 0, 2: 0, 3: 0}
with tqdm.tqdm(total=len(self.test_data),
ascii=True) as pbar_test: # ini a progress bar
try:
for x, y, z in self.test_data: # sample batch
loss, accuracy1, accuracy2 = self.run_evaluation_iter(
x=x, y=y, z=z
) # compute loss and accuracy by running an evaluation step
current_epoch_losses["test_loss"].append(
loss) # save test loss
current_epoch_losses["test_SER_acc"].append(
accuracy1) # save test accuracy
current_epoch_losses["test_Gender_acc"].append(
accuracy2) # save test accuracy
pbar_test.update(1) # update progress bar status
pbar_test.set_description(
"loss: {:.4f}, SER_ accuracy: {:.4f}, Test_ accuracy: {:.4f},"
.format(
loss, accuracy1,
accuracy2)) # update progress bar string output
torch.cuda.empty_cache()
except KeyboardInterrupt:
pbar_test.close()
raise
pbar_test.close()
test_losses = {
key: [np.mean(value)]
for key, value in current_epoch_losses.items()
} # save test set metrics in dict format
ua_test = 0
for key, value in self.eval_current_epoch_correct_count.items():
ua_test += value / self.eval_current_epoch_emo_count[key] / 4
test_losses["ua_test"] = [ua_test]
save_statistics(
experiment_log_dir=self.experiment_logs,
filename='test_summary.csv',
# save test set metrics on disk in .csv format
stats_dict=test_losses,
current_epoch=0,
continue_from_mode=False)
# print('Sorting model files')
# self.delete_model(model_save_dir=self.experiment_saved_models, model_save_name="train_model",
# model_idx=self.best_val_model_idx)
return total_losses, test_losses
def run_experiment(self):
"""
Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch
:return: The summary current_epoch_losses from starting epoch to total_epochs.
"""
total_losses = {
"train_acc": [],
"train_loss": [],
"val_acc": [],
"val_loss": [],
"val_f1": [],
"val_precision": [],
"val_recall": [],
"curr_epoch": []
} # initialize a dict to keep the per-epoch metrics
for i, epoch_idx in enumerate(
range(self.starting_epoch, self.num_epochs)):
epoch_start_time = time.time()
current_epoch_losses = {
"train_acc": [],
"train_loss": [],
"val_acc": [],
"val_loss": [],
"val_f1": [],
"val_precision": [],
"val_recall": []
}
current_epoch_losses = self.run_training_epoch(
current_epoch_losses)
if self.scheduler is not None:
self.scheduler.step()
train_loss_average = np.mean(current_epoch_losses['train_loss'])
if train_loss_average < self.best_train_loss:
print(f'Saving Best Model')
self.best_train_loss = train_loss_average
self.save_model(
model_save_dir=self.experiment_saved_models,
# save model and best val idx and best val acc, using the model dir, model name and model idx
model_save_name="train_model",
model_idx='best',
state=self.state)
for key, value in current_epoch_losses.items():
total_losses[key].append(np.mean(value))
# get mean of all metrics of current epoch metrics dict,
# to get them ready for storage and output on the terminal.
total_losses['curr_epoch'].append(epoch_idx)
save_statistics(experiment_log_dir=self.experiment_logs,
filename='summary.csv',
stats_dict=total_losses,
current_epoch=i,
continue_from_mode=True if
(self.starting_epoch != 0 or i > 0) else
False) # save statistics to stats file.
# load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to
out_string = "_".join([
"{}_{:.4f}".format(key, np.mean(value))
for key, value in current_epoch_losses.items()
])
# create a string to use to report our epoch metrics
epoch_elapsed_time = time.time(
) - epoch_start_time # calculate time taken for epoch
epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time)
print("Epoch {}:".format(epoch_idx), out_string, "epoch time",
epoch_elapsed_time, "seconds")
self.state['current_epoch_idx'] = epoch_idx
return total_losses
def run_experiment(self):
"""
Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch
:return: The summary current_epoch_losses from starting epoch to total_epochs.
"""
total_losses = {"train_acc": [], "train_loss": [], "val_acc": [],
"val_loss": []} # initialize a dict to keep the per-epoch metrics
train_number_batches = int(math.ceil(self.training_instances/self.batch_size))
val_number_batches = int(math.ceil(self.val_instances/self.batch_size))
if self.stack:
train_number_batches = 2*train_number_batches
val_number_batches = 2*val_number_batches
for i, epoch_idx in enumerate(range(self.starting_epoch, self.num_epochs)):
epoch_start_time = time.time()
current_epoch_losses = {"train_acc": [], "train_loss": [], "val_acc": [], "val_loss": []}
print("num batches",train_number_batches)
if self.stack:
total_ = train_number_batches-2
else:
total_ = train_number_batches-1
if self.shuffle:
idx = np.arange(0,self.train_data.inputs.shape[0])
print("before shuffle", self.train_data.inputs.shape)
np.random.shuffle(idx)
self.train_data.inputs = self.train_data.inputs[idx]
self.train_data.targets = self.train_data.targets[idx]
with tqdm.tqdm(total=total_) as pbar_train: # create a progress bar for training
for idx in range(total_):
x,y,manual_verified = self.get_batch(data = self.train_data,
idx = idx, number_batches = total_,train=True)
loss, accuracy = self.run_train_iter(x=x, y=y,manual_verified=manual_verified,epoch_number = epoch_idx) # take a training iter step
current_epoch_losses["train_loss"].append(loss) # add current iter loss to the train loss list
current_epoch_losses["train_acc"].append(accuracy) # add current iter acc to the train acc list
pbar_train.update(1)
pbar_train.set_description("loss: {:.4f}, accuracy: {:.4f}".format(loss, accuracy))
with tqdm.tqdm(total=val_number_batches) as pbar_val: # create a progress bar for validation
for idx in range(val_number_batches):
x,y = self.get_batch(data = self.val_data,
idx = idx, number_batches = val_number_batches)
loss, accuracy = self.run_evaluation_iter(x=x, y=y) # run a validation iter
current_epoch_losses["val_loss"].append(loss) # add current iter loss to val loss list.
current_epoch_losses["val_acc"].append(accuracy) # add current iter acc to val acc lst.
pbar_val.update(1) # add 1 step to the progress bar
pbar_val.set_description("loss: {:.4f}, accuracy: {:.4f}".format(loss, accuracy))
val_mean_accuracy = np.mean(current_epoch_losses['val_acc'])
if val_mean_accuracy > self.best_val_model_acc: # if current epoch's mean val acc is greater than the saved best val acc then
self.best_val_model_acc = val_mean_accuracy # set the best val model acc to be current epoch's val accuracy
self.best_val_model_idx = epoch_idx # set the experiment-wise best val idx to be the current epoch's idx
for key, value in current_epoch_losses.items():
total_losses[key].append(np.mean(
value)) # get mean of all metrics of current epoch metrics dict, to get them ready for storage and output on the terminal.
save_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv',
stats_dict=total_losses, current_epoch=i) # save statistics to stats file.
load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to
out_string = "_".join(
["{}_{:.4f}".format(key, np.mean(value)) for key, value in current_epoch_losses.items()])
# create a string to use to report our epoch metrics
epoch_elapsed_time = time.time() - epoch_start_time # calculate time taken for epoch
epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time)
print("Epoch {}:".format(epoch_idx), out_string, "epoch time", epoch_elapsed_time, "seconds")
self.state['current_epoch_idx']=epoch_idx
self.state['best_val_model_acc']=self.best_val_model_acc
self.state['best_val_model_idx']=self.best_val_model_idx
self.save_model(model_save_dir=self.experiment_saved_models,
# save model and best val idx and best val acc, using the model dir, model name and model idx
model_save_name="train_model", model_idx=epoch_idx,state=self.state)
self.save_model(model_save_dir=self.experiment_saved_models,
model_save_name="train_model",model_idx='latest',state=self.state)
print("Generating test set evaluation metrics")
self.load_model(model_save_dir=self.experiment_saved_models, model_idx=self.best_val_model_idx,
# load best validation model
model_save_name="train_model")
current_epoch_losses = {"test_acc": [], "test_loss": []} # initialize a statistics dict
test_number_batches = int(math.ceil(self.test_instances/self.batch_size))
if self.stack:
test_number_batches = 2*test_number_batches
with tqdm.tqdm(total=test_number_batches) as pbar_test: # ini a progress bar
for idx in range(test_number_batches): # sample batch
x,y = self.get_batch(data = self.test_data,
idx = idx, number_batches = test_number_batches)
loss, accuracy = self.run_evaluation_iter(x=x,
y=y) # compute loss and accuracy by running an evaluation step
current_epoch_losses["test_loss"].append(loss) # save test loss
current_epoch_losses["test_acc"].append(accuracy) # save test accuracy
pbar_test.update(1) # update progress bar status
pbar_test.set_description(
"loss: {:.4f}, accuracy: {:.4f}".format(loss, accuracy)) # update progress bar string output
test_losses = {key: [np.mean(value)] for key, value in
current_epoch_losses.items()} # save test set metrics in dict format
save_statistics(experiment_log_dir=self.experiment_logs, filename='test_summary.csv',
#save test set metrics on disk in .csv format
stats_dict=test_losses, current_epoch=0)
return total_losses, test_losses
def run_experiment(self):
"""
Runs experiment train and evaluation iterations, saving the model and best val model
and val model accuracy after each epoch
:return: The summary current_epoch_losses from starting epoch to total_epochs.
"""
if self.continue_from_epoch == -1:
self.model.reset_parameters()
for i, epoch_idx in enumerate(
range(self.starting_epoch, self.num_epochs)):
epoch_start_time = time.time()
current_epoch_losses = {
"train_loss": [],
"val_loss": [],
"val_nrmse_loss": []
}
with tqdm.tqdm(
total=self.train_data.num_batches
) as pbar_train: # create a progress bar for training
for idx, (x,
y) in enumerate(self.train_data): # get data batches
loss = self.run_train_iter(
x=x, y=y) # take a training iter step
current_epoch_losses["train_loss"].append(
loss) # add current iter loss to the train loss list
pbar_train.update(1)
pbar_train.set_description("loss: {:.4f}".format(loss))
with tqdm.tqdm(
total=self.val_data.num_batches
) as pbar_val: # create a progress bar for validation
for x, y in self.val_data: # get data batches
mse_loss, nrmse_loss = self.run_evaluation_iter(
x=x, y=y) # run a validation iter
current_epoch_losses["val_loss"].append(
mse_loss) # add current iter loss to val loss list.
current_epoch_losses["val_nrmse_loss"].append(nrmse_loss)
pbar_val.update(1) # add 1 step to the progress bar
pbar_val.set_description("loss: {:.4f}".format(loss))
val_mean_loss = np.mean(current_epoch_losses['val_loss'])
# if lowest validation loss was achieved in this epoch
if val_mean_loss < self.best_val_loss:
self.best_val_loss = val_mean_loss
self.best_val_model_idx = epoch_idx
val_nrmse_mean_loss = np.mean(
current_epoch_losses['val_nrmse_loss'])
# if lowest nrmse validation loss was achieved in this epoch
if val_nrmse_mean_loss < self.best_val_nrmse_loss:
self.best_val_nrmse_loss = val_nrmse_mean_loss
self.best_val_nrmse_model_idx = epoch_idx
# get mean of all metrics of current epoch metrics dict,
# to get them ready for storage and output on the terminal.
for key, value in current_epoch_losses.items():
self.metrics[key].append(np.mean(value))
self.metrics['curr_epoch'].append(epoch_idx)
save_statistics(experiment_log_dir=self.experiment_logs,
filename='summary.csv',
stats_dict=self.metrics,
current_epoch=epoch_idx,
continue_from_mode=(self.starting_epoch != 0
or i > 0))
save_best_val_scores(
experiment_log_dir=self.experiment_logs,
filename='best_val_scores.csv',
best_val_loss=self.best_val_loss,
best_val_model_idx=self.best_val_model_idx,
best_val_nrmse_loss=self.best_val_nrmse_loss,
best_val_nrmse_model_idx=self.best_val_nrmse_model_idx)
out_string = "_".join([
"{}_{:.4f}".format(key, np.mean(value))
for key, value in current_epoch_losses.items()
])
# create a string to use to report our epoch metrics
epoch_elapsed_time = time.time(
) - epoch_start_time # calculate time taken for epoch
epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time)
print("Epoch {}:".format(epoch_idx), out_string, "epoch time",
epoch_elapsed_time, "seconds")
# save the current model
self.save_model(model_save_dir=self.experiment_saved_models,
model_save_name="train_model",
epoch_idx=epoch_idx)