def evaluate(self): print("Generating test set evaluation metrics") self.load_model( model_save_dir=self.experiment_saved_models, model_idx=self.best_val_model_idx, # load best validation model model_save_name="train_model") current_epoch_losses = { "test_aps": [], "test_loss": [] } # initialize a statistics dict predictions = {"y_pred": []} with tqdm.tqdm( total=len(self.test_data)) as pbar_test: # ini a progress bar for idx, batch in enumerate(self.test_data): # sample batch desc_1_batch = batch['desc1'].to(self.device) desc_2_batch = batch['desc2'].to(self.device) img_1_batch = batch['image_1'].to(self.device) img_2_batch = batch['image_2'].to(self.device) y = batch['target'] x = [desc_1_batch, img_1_batch, desc_2_batch, img_2_batch] loss, aps, predicted = self.run_evaluation_iter( x=x, y=y ) # compute loss and accuracy by running an evaluation step current_epoch_losses["test_loss"].append( loss) # save test loss current_epoch_losses["test_aps"].append( aps) # save test accuracy predicted = predicted.flatten() for p in predicted: predictions['y_pred'].append(p) pbar_test.update(1) # update progress bar status pbar_test.set_description( "loss: {:.4f}, average precision score: {:.4f}".format( loss, aps)) # update progress bar string output test_losses = { key: [np.mean(value)] for key, value in current_epoch_losses.items() } # save test set metrics in dict format save_statistics( experiment_log_dir=self.experiment_logs, filename='test_summary.csv', # save test set metrics on disk in .csv format stats_dict=test_losses, current_epoch=0, continue_from_mode=False) save_statistics( experiment_log_dir=self.experiment_logs, filename='test_predictions.csv', # save test set metrics on disk in .csv format save_full_dict=True, stats_dict=predictions, current_epoch=0, continue_from_mode=False)
def output_summary(self, type, epoch_idx): print("Generating {} set evaluation metrics".format(type)) self.load_model(model_save_dir=self.experiment_saved_models, model_idx=self.best_valid_model_idx, # load best validation model model_save_name="train_model") current_epoch_losses = {type + "_loss": [], type+"_l_s": [], type+"_l_t": [], type + "_acc": [], type+"_a_s": [], type+"_a_t": [], type + "_f1": [], type+"_f_s": [], type+"_f_t": [], type + "_prec": [], type + "_p_s": [], type + "_p_t": [], type + "_rec": [], type+"_r_s": [], type+"_r_t": []} # initialize a statistics dict current_epoch_losses, y_out, pred_out = self.run_epoch(current_epoch_losses, type, output=True, epoch_idx=epoch_idx) # save test set metrics in dict format losses = {} losses[type+"_loss"] = [np.mean(current_epoch_losses[type+"_loss"])] losses[type + "_l_s"] = [np.mean(current_epoch_losses[type + "_l_s"])] losses[type + "_l_t"] = [np.mean(current_epoch_losses[type + "_l_t"])] losses[type + "_acc"] = [np.mean(current_epoch_losses[type + "_acc"])] losses[type + "_a_s"] = [np.mean(current_epoch_losses[type + "_a_s"])] losses[type + "_a_t"] = [np.mean(current_epoch_losses[type + "_a_t"])] losses[type + "_f1"] = [np.mean(current_epoch_losses[type + "_f1"])] losses[type + "_f_s"] = [np.mean(current_epoch_losses[type + "_f_s"])] losses[type + "_f_t"] = [np.mean(current_epoch_losses[type + "_f_t"])] losses[type + "_prec"] = [np.mean(current_epoch_losses[type + "_prec"])] losses[type + "_p_s"] = [np.mean(current_epoch_losses[type + "_p_s"])] losses[type + "_p_t"] = [np.mean(current_epoch_losses[type + "_p_t"])] losses[type + "_rec"] = [np.mean(current_epoch_losses[type + "_rec"])] losses[type + "_r_s"] = [np.mean(current_epoch_losses[type + "_r_s"])] losses[type + "_r_t"] = [np.mean(current_epoch_losses[type + "_r_t"])] y_out = y_out.detach().cpu().numpy().flatten() # losses[type+'_y'] = [str(y_arr)] pred_out = pred_out.detach().cpu().numpy() #.flatten() # pred_out = np.exp(pred_arr) # pred_arr is log probabilities, make probabilities # losses[type+'_pred'] = [str(self.sigmoid_array(pred_arr))] print(type + " evaluation:") print("shape of y: ", y_out.shape) print("shape of pred:", pred_out.shape) out_path = os.path.join(self.experiment_logs, "{}_out.npz".format(type)) # print("Saving in file {}: y_out {}, and pred_out {}".format(out_path, y_out, pred_out)) np.savez(out_path, y=y_out, pred=pred_out) save_statistics(experiment_log_dir=self.experiment_logs, filename=type+'_summary.csv', # save test set metrics on disk in .csv format stats_dict=losses, current_epoch=0, continue_from_mode=False) return losses
def run_experiment(self): """ Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch :return: The summary current_epoch_losses from starting epoch to total_epochs. """ total_losses = { "train_acc": [], "train_loss": [], "val_acc": [], "val_loss": [], "curr_epoch": [] } # initialize a dict to keep the per-epoch metrics for i, epoch_idx in enumerate( range(self.starting_epoch, self.num_epochs)): epoch_start_time = time.time() current_epoch_losses = { "train_acc": [], "train_loss": [], "val_acc": [], "val_loss": [] } with tqdm.tqdm( total=len(self.train_data) ) as pbar_train: # create a progress bar for training for idx, (x, y) in enumerate(self.train_data): # get data batches loss, accuracy = self.run_train_iter( x=x, y=y) # take a training iter step current_epoch_losses["train_loss"].append( loss) # add current iter loss to the train loss list current_epoch_losses["train_acc"].append( accuracy) # add current iter acc to the train acc list pbar_train.update(1) pbar_train.set_description( "loss: {:.4f}, accuracy: {:.4f}".format( loss, accuracy)) with tqdm.tqdm( total=len(self.val_data) ) as pbar_val: # create a progress bar for validation for x, y in self.val_data: # get data batches loss, accuracy = self.run_evaluation_iter( x=x, y=y) # run a validation iter current_epoch_losses["val_loss"].append( loss) # add current iter loss to val loss list. current_epoch_losses["val_acc"].append( accuracy) # add current iter acc to val acc lst. pbar_val.update(1) # add 1 step to the progress bar pbar_val.set_description( "loss: {:.4f}, accuracy: {:.4f}".format( loss, accuracy)) val_mean_accuracy = np.mean(current_epoch_losses['val_acc']) if val_mean_accuracy > self.best_val_model_acc: # if current epoch's mean val acc is greater than the saved best val acc then self.best_val_model_acc = val_mean_accuracy # set the best val model acc to be current epoch's val accuracy self.best_val_model_idx = epoch_idx # set the experiment-wise best val idx to be the current epoch's idx for key, value in current_epoch_losses.items(): total_losses[key].append( np.mean(value) ) # get mean of all metrics of current epoch metrics dict, to get them ready for storage and output on the terminal. total_losses['curr_epoch'].append(epoch_idx) save_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv', stats_dict=total_losses, current_epoch=i, continue_from_mode=True if (self.starting_epoch != 0 or i > 0) else False) # save statistics to stats file. # load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to out_string = "_".join([ "{}_{:.4f}".format(key, np.mean(value)) for key, value in current_epoch_losses.items() ]) # create a string to use to report our epoch metrics epoch_elapsed_time = time.time( ) - epoch_start_time # calculate time taken for epoch epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time) print("Epoch {}:".format(epoch_idx), out_string, "epoch time", epoch_elapsed_time, "seconds") self.state['current_epoch_idx'] = epoch_idx self.state['best_val_model_acc'] = self.best_val_model_acc self.state['best_val_model_idx'] = self.best_val_model_idx self.save_model( model_save_dir=self.experiment_saved_models, # save model and best val idx and best val acc, using the model dir, model name and model idx model_save_name="train_model", model_idx=epoch_idx, state=self.state) self.save_model( model_save_dir=self.experiment_saved_models, # save model and best val idx and best val acc, using the model dir, model name and model idx model_save_name="train_model", model_idx='latest', state=self.state) print("Generating test set evaluation metrics") self.load_model( model_save_dir=self.experiment_saved_models, model_idx=self.best_val_model_idx, # load best validation model model_save_name="train_model") current_epoch_losses = { "test_acc": [], "test_loss": [] } # initialize a statistics dict with tqdm.tqdm( total=len(self.test_data)) as pbar_test: # ini a progress bar for x, y in self.test_data: # sample batch loss, accuracy = self.run_evaluation_iter( x=x, y=y ) # compute loss and accuracy by running an evaluation step current_epoch_losses["test_loss"].append( loss) # save test loss current_epoch_losses["test_acc"].append( accuracy) # save test accuracy pbar_test.update(1) # update progress bar status pbar_test.set_description( "loss: {:.4f}, accuracy: {:.4f}".format( loss, accuracy)) # update progress bar string output test_losses = { key: [np.mean(value)] for key, value in current_epoch_losses.items() } # save test set metrics in dict format save_statistics( experiment_log_dir=self.experiment_logs, filename='test_summary.csv', # save test set metrics on disk in .csv format stats_dict=test_losses, current_epoch=0, continue_from_mode=False) return total_losses, test_losses
def run_experiment(self): """ Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch :return: The summary current_epoch_losses from starting epoch to total_epochs. """ total_losses = { "train_acc": [], "train_loss": [], "val_acc": [], "val_loss": [], "curr_epoch": [] } # initialize a dict to keep the per-epoch metrics for i, epoch_idx in enumerate( range(self.starting_epoch, self.num_epochs)): epoch_start_time = time.time() current_epoch_losses = { "train_acc": [], "train_loss": [], "val_acc": [], "val_loss": [] } current_epoch_losses = self.run_training_epoch( current_epoch_losses) current_epoch_losses = self.run_validation_epoch( current_epoch_losses) val_mean_accuracy = np.mean(current_epoch_losses['val_acc']) if val_mean_accuracy > self.best_val_model_acc: # if current epoch's mean val acc is greater than the saved best val acc then self.best_val_model_acc = val_mean_accuracy # set the best val model acc to be current epoch's val accuracy self.best_val_model_idx = epoch_idx # set the experiment-wise best val idx to be the current epoch's idx for key, value in current_epoch_losses.items(): total_losses[key].append(np.mean(value)) # get mean of all metrics of current epoch metrics dict, # to get them ready for storage and output on the terminal. total_losses['curr_epoch'].append(epoch_idx) save_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv', stats_dict=total_losses, current_epoch=i, continue_from_mode=True if (self.starting_epoch != 0 or i > 0) else False) # save statistics to stats file. # load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to out_string = "_".join([ "{}_{:.4f}".format(key, np.mean(value)) for key, value in current_epoch_losses.items() ]) # create a string to use to report our epoch metrics epoch_elapsed_time = time.time( ) - epoch_start_time # calculate time taken for epoch epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time) print("Epoch {}:".format(epoch_idx), out_string, "epoch time", epoch_elapsed_time, "seconds") self.state['current_epoch_idx'] = epoch_idx self.state['best_val_model_acc'] = self.best_val_model_acc self.state['best_val_model_idx'] = self.best_val_model_idx self.save_model( model_save_dir=self.experiment_saved_models, # save model and best val idx and best val acc, using the model dir, model name and model idx model_save_name="train_model", model_idx=epoch_idx, state=self.state) self.save_model( model_save_dir=self.experiment_saved_models, # save model and best val idx and best val acc, using the model dir, model name and model idx model_save_name="train_model", model_idx='latest', state=self.state) print("Generating test set evaluation metrics") self.load_model( model_save_dir=self.experiment_saved_models, model_idx=self.best_val_model_idx, # load best validation model model_save_name="train_model") current_epoch_losses = { "test_acc": [], "test_loss": [] } # initialize a statistics dict current_epoch_losses = self.run_testing_epoch( current_epoch_losses=current_epoch_losses) test_losses = { key: [np.mean(value)] for key, value in current_epoch_losses.items() } # save test set metrics in dict format save_statistics( experiment_log_dir=self.experiment_logs, filename='test_summary.csv', # save test set metrics on disk in .csv format stats_dict=test_losses, current_epoch=0, continue_from_mode=False) return total_losses, test_losses
def run_experiment(self): """ Runs experiment train and evaluation iterations, saving the model and best val model and val model roc_auc after each cycle :return: The summary current_cycle_losses from starting cycle to total_cycles. """ total_losses = { "curr_cycle": [], "train_loss": [], "train_acc": [], "val_loss": [], "val_acc": [], "val_roc_auc": [] } # initialize a dict to keep the per-epoch metrics # new_cycle = False current_cycle_losses = { "train_loss": [], "train_acc": [], "train_bs": [], "val_loss": [], "val_acc": [], "val_bs": [] } cycle_start_time = time.time() for i, epoch_idx in enumerate( range(self.starting_epoch, self.num_epochs)): ''' TRAIN ''' with tqdm(total=len(self.train_data), disable=bool(os.environ.get("DISABLE_TQDM", False)) ) as pbar_train: # create a progress bar for training for idx, batch in enumerate( self.train_data): # get data batches new_cycle = self.scheduler.get_lr( self.global_step + 1, self.scheduler.warmup) > self.scheduler.get_lr( self.global_step, self.scheduler.warmup) new_cycle &= self.scheduler.get_lr( self.global_step - 1, self.scheduler.warmup) > self.scheduler.get_lr( self.global_step, self.scheduler.warmup) new_cycle &= (self.global_step != 0) if new_cycle: print('New cycle:', new_cycle) current_cycle_losses = { "train_loss": [], "train_acc": [], "train_bs": [], "val_loss": [], "val_acc": [], "val_bs": [] } cycle_start_time = time.time() self.iteration_step = idx try: loss, accuracy = self.run_train_iter( batch) # take a training iter step except RuntimeError as e: if 'out of memory' in str(e): print('| WARNING: ran out of memory, exiting') for p in self.model.parameters(): del p # free all memory _ = gc.collect() torch.cuda.empty_cache() return else: raise e current_cycle_losses["train_bs"].append(len( batch[0])) # add batch size! current_cycle_losses["train_loss"].append( loss) # add current iter loss to the train loss list current_cycle_losses["train_acc"].append( accuracy) # add current iter acc to the train acc list pbar_train.update(1) if not pbar_train.disable: pbar_train.set_description("loss: {:.4f}".format(loss)) ''' if new cycle start save!''' # self.global_step-1 because it is incremented in the iteration loop before if new_cycle: ''' VALIDATION ''' pred_all = [] y_all = [] with tqdm( total=len(self.val_data), disable=bool( os.environ.get("DISABLE_TQDM", False)) ) as pbar_val: # create a progress bar for validation for batch in self.val_data: # get data batches try: loss, accuracy, target = self.run_evaluation_iter( batch) # run a validation iter except RuntimeError as e: if 'out of memory' in str(e): print( '| WARNING: ran out of memory, exiting' ) for p in self.model.parameters(): del p # free all memory _ = gc.collect() torch.cuda.empty_cache() return else: raise e # keep log pred_all.append(target) y_all.append(batch[-1]) current_cycle_losses["val_bs"].append( len(batch[0])) # add batch size! current_cycle_losses["val_loss"].append( loss ) # add current iter loss to val loss list. current_cycle_losses["val_acc"].append( accuracy ) # add current iter acc to val acc lst. pbar_val.update( 1) # add 1 step to the progress bar if not pbar_val.disable: pbar_val.set_description( "loss: {:.4f}".format(loss)) ''' SAVE LEARNING LOGS''' # val_roc_auc = roc_auc(np.vstack(y_all), np.vstack(pred_all)) # current_cycle_losses["val_roc_auc"] = [val_roc_auc] # REMINDER NEW: keep track of bias roc_auc # val_roc_auc = #.get_final_metric() try: val_roc_auc = self.evaluator.get_final_metric( np.vstack(pred_all)[:, 0]) current_cycle_losses["val_roc_auc"] = [val_roc_auc] except: print('Cannot compute bias val_roc_auc') val_roc_auc = 0 current_cycle_losses["val_roc_auc"] = [val_roc_auc] for key, value in current_cycle_losses.items(): if "bs" in key: continue # get mean of all metrics of current epoch metrics dict, to get them ready for storage and output on the terminal. # weights for incremental mean; or stacked evaluation try: total_losses[key].append( np.average( value, axis=0, weights=total_losses[key.split("_")[0] + "_bs"])) except: total_losses[key].append( np.average(value, axis=0)) if self.val_metrics == 'loss': val_main_metrics = total_losses['val_loss'][-1] if val_main_metrics < self.best_val_model_metrics or self.cycle == 0: # if current epoch's mean val pargsaroc_auc is greater than the saved best val main metrics then self.best_val_model_metrics = val_main_metrics # set the best val model pargsaroc_auc to be current epoch's val main metrics # TODO: epoch_idx is ALREADY OFFSET! self.best_val_model_idx = self.cycle # set the experiment-wise best val idx to be the current epoch's idx else: val_main_metrics = total_losses['val_acc'][ -1] if "acc" in self.val_metrics else total_losses[ 'val_roc_auc'][-1] if val_main_metrics > self.best_val_model_metrics or self.cycle == 0: # if current epoch's mean val pargsaroc_auc is greater than the saved best val main metrics then self.best_val_model_metrics = val_main_metrics # set the best val model pargsaroc_auc to be current epoch's val main metrics self.best_val_model_idx = self.cycle # set the experiment-wise best val idx to be the current epoch's idx total_losses['curr_cycle'].append(self.cycle) # print(total_losses) save_statistics( experiment_log_dir=self.experiment_logs, filename='summary.csv', stats_dict=total_losses, current_epoch=self.cycle, continue_from_mode=True if (self.starting_epoch != 0 or self.cycle > 0) else False) # save statistics to stats file. # load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to out_string = "\n".join([ "{}_{}".format(key, value[-1]) for key, value in total_losses.items() ]) # create a string to use to report our epoch metrics cycle_elapsed_time = time.time( ) - cycle_start_time # calculate time taken for epoch cycle_elapsed_time = "{:.4f}".format( cycle_elapsed_time) print("Cycle {}:".format(self.cycle), out_string, "cycle time", cycle_elapsed_time, "seconds", "best cycle", self.best_val_model_idx) self.state['current_cycle_idx'] = self.cycle self.state[ 'best_val_model_roc_auc'] = self.best_val_model_metrics self.state[ 'best_val_model_idx'] = self.best_val_model_idx # save model and best val idx and best val metrics, using the model dir, model name and model idx if self.best_val_model_idx == self.cycle: print('SAVING BEST', self.best_val_model_idx, self.cycle) self.patience = 0 self.save_model( model_save_dir=self.experiment_saved_models, model_save_name="train_model", model_idx='best', state=self.state) else: self.patience += 1 self.save_model( model_save_dir=self.experiment_saved_models, model_save_name="train_model", model_idx=self.cycle, state=self.state) self.cycle += 1 if self.patience >= self.max_patience: # hard code patience break self.save_model(model_save_dir=self.experiment_saved_models, model_save_name="train_model", model_idx='latest', state=self.state) ''' EXIT TRAINING, DO PREDICTION ''' if self.test_data is None: pass else: return self.run_prediction()
def run_experiment(self): """ Runs experiment train and evaluation iterations, saving the model and best valid model and valid model accuracy after each epoch :return: The summary current_epoch_losses from starting epoch to total_epochs. """ total_losses = {"train_loss": [], "train_l_s": [], "train_l_t": [], "train_acc": [], "train_a_s": [], "train_a_t": [], "train_f1": [], "train_f_s": [], "train_f_t": [], "train_prec": [], "train_p_s": [], "train_p_t": [], "train_rec": [], "train_r_s": [], "train_r_t": [], "valid_loss": [], "valid_l_s": [], "valid_l_t": [], "valid_acc": [], "valid_a_s": [], "valid_a_t": [], "valid_f1": [], "valid_f_s": [], "valid_f_t": [], "valid_prec": [], "valid_p_s": [], "valid_p_t": [], "valid_rec": [], "valid_r_s": [], "valid_r_t": [], "curr_epoch": []} # initialize a dict to keep the per-epoch metrics for i, epoch_idx in enumerate(range(self.starting_epoch, self.starting_epoch + self.num_epochs)): epoch_start_time = time.time() current_epoch_losses = {"train_loss": [], "train_l_s": [], "train_l_t": [], "train_acc": [], "train_a_s": [], "train_a_t": [], "train_f1": [], "train_f_s": [], "train_f_t": [], "train_prec": [], "train_p_s": [], "train_p_t": [], "train_rec": [], "train_r_s": [], "train_r_t": [], "valid_loss": [], "valid_l_s": [], "valid_l_t": [], "valid_acc": [], "valid_a_s": [], "valid_a_t": [], "valid_f1": [], "valid_f_s": [], "valid_f_t": [], "valid_prec": [], "valid_p_s": [], "valid_p_t": [], "valid_rec": [], "valid_r_s": [], "valid_r_t": []} training_start_time = time.time() print("debug_batch_cnt_max={}".format(self.debug_batch_cnt_max)) current_epoch_losses, _, _ = self.run_epoch(current_epoch_losses, "train", output=False, epoch_idx=epoch_idx) print("Total epoch training time is {0:.2f} secs".format(time.time() - training_start_time)) validation_start_time = time.time() current_epoch_losses, _, _ = self.run_epoch(current_epoch_losses, "valid", output=False, epoch_idx=epoch_idx) print("Total epoch validation time is {0:.2f} secs".format(time.time() - validation_start_time)) # valid_mean_accuracy = np.mean(current_epoch_losses['valid_acc']) valid_mean_acc = np.mean(current_epoch_losses['valid_acc']) valid_mean_f1 = np.mean(current_epoch_losses['valid_f1']) if valid_mean_acc > self.best_valid_model_acc: # if current epoch's mean valid acc is greater than the saved best valid acc then self.best_valid_model_acc = valid_mean_acc # set the best valid model acc to be current epoch's valid acc self.best_valid_model_f1 = valid_mean_f1 self.best_valid_model_idx = epoch_idx # set the experiment-wise best valid idx to be the current epoch's idx for key, value in current_epoch_losses.items(): total_losses[key].append(np.mean( value)) # get mean of all metrics of current epoch metrics dict, to get them ready for storage and output on the terminal. total_losses['curr_epoch'].append(epoch_idx) save_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv', stats_dict=total_losses, current_epoch=i, continue_from_mode=True if (self.starting_epoch != 0 or i > 0) else False) # save statistics to stats file. # load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to out_string = "_".join( ["{}_{:.4f}".format(key, np.mean(value)) for key, value in current_epoch_losses.items()]) # create a string to use to report our epoch metrics self.state['current_epoch_idx'] = epoch_idx self.state['best_valid_model_acc'] = self.best_valid_model_acc self.state['best_valid_model_f1'] = self.best_valid_model_f1 self.state['best_valid_model_idx'] = self.best_valid_model_idx self.save_model(model_save_dir=self.experiment_saved_models, # save model and best valid idx and best valid acc, using the model dir, model name and model idx model_save_name="train_model", model_idx=epoch_idx, state=self.state) self.save_model(model_save_dir=self.experiment_saved_models, # save model and best valid idx and best valid acc, using the model dir, model name and model idx model_save_name="train_model", model_idx='latest', state=self.state) print("Epoch {}: {}".format(epoch_idx, out_string)) print("Total epoch time is {0:.2f} secs".format(time.time() - epoch_start_time)) valid_losses = self.output_summary(type='valid', epoch_idx=epoch_idx) test_losses = self.output_summary(type='test', epoch_idx=epoch_idx) return total_losses, test_losses
def run_experiment(self): """ Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch :return: The summary current_epoch_losses from starting epoch to total_epochs. """ total_losses = { "val_Gender_acc": [], "val_SER_acc": [], "train_acc_GENDER": [], "train_acc_SER": [], "train_loss": [], "val_loss": [], "ua_train": [], "ua_eval": [] } # initialize a dict to keep the per-epoch metrics for i, epoch_idx in enumerate( range(self.starting_epoch, self.num_epochs)): epoch_start_time = time.time() current_epoch_losses = { "train_acc_SER": [], "train_acc_GENDER": [], "train_loss": [], "val_Gender_acc": [], "val_SER_acc": [], "val_loss": [] } self.current_epoch = epoch_idx # init un_accuracy self.current_epoch_emo_count = {0: 0, 1: 0, 2: 0, 3: 0} self.current_epoch_correct_count = {0: 0, 1: 0, 2: 0, 3: 0} with tqdm.tqdm( total=len(self.train_data), ascii=True ) as pbar_train: # create a progress bar for training try: for idx, (x, y, z) in enumerate( self.train_data): # get data batches loss, accuracy1, accuracy2 = self.run_train_iter( x=x, y=y, z=z) # take a training iter step current_epoch_losses["train_loss"].append( loss ) # add current iter loss to the train loss list current_epoch_losses["train_acc_SER"].append( accuracy1 ) # add current iter acc to the train acc list current_epoch_losses["train_acc_GENDER"].append( accuracy2) pbar_train.update(1) pbar_train.set_description( "loss: {:.4f}, accuracy_SER: {:.4f}, accuracy_Gender: {:.4f}" .format(loss, accuracy1, accuracy2)) torch.cuda.empty_cache() except KeyboardInterrupt: pbar_train.close() raise pbar_train.close() # get un_accuracy result self.eval_current_epoch_emo_count = {0: 0, 1: 0, 2: 0, 3: 0} self.eval_current_epoch_correct_count = {0: 0, 1: 0, 2: 0, 3: 0} with tqdm.tqdm( total=len(self.val_data), ascii=True ) as pbar_val: # create a progress bar for validation try: for x, y, z in self.val_data: # get data batches loss, accuracy1, accuracy2 = self.run_evaluation_iter( x=x, y=y, z=z) # run a validation iter current_epoch_losses["val_loss"].append( loss) # add current iter loss to val loss list. current_epoch_losses["val_SER_acc"].append( accuracy1) # add current iter acc to val acc lst. current_epoch_losses["val_Gender_acc"].append( accuracy2) # add current iter acc to val acc lst. pbar_val.update(1) # add 1 step to the progress bar pbar_val.set_description( "loss: {:.4f}, accuracy_SER: {:.4f}, accuracy_Gender: {:.4f}" .format(loss, accuracy1, accuracy2)) torch.cuda.empty_cache() except KeyboardInterrupt: pbar_val.close() raise pbar_val.close() val_mean_accuracy = np.mean(current_epoch_losses['val_SER_acc']) if val_mean_accuracy > self.best_val_model_acc: # if current epoch's mean val acc is greater than the saved best val acc then self.best_val_model_acc = val_mean_accuracy # set the best val model acc to be current epoch's val accuracy self.best_val_model_idx = epoch_idx # set the experiment-wise best val idx to be the current epoch's idx for key, value in current_epoch_losses.items(): total_losses[key].append( np.mean(value) ) # get mean of all metrics of current epoch metrics dict, to get them ready for storage and output on the terminal. ua_train = 0 ua_val = 0 for key, value in self.current_epoch_correct_count.items(): ua_train += value / self.current_epoch_emo_count[key] / 4 total_losses["ua_train"].append(ua_train) for key, value in self.eval_current_epoch_correct_count.items(): ua_val += value / self.eval_current_epoch_emo_count[key] / 4 total_losses["ua_eval"].append(ua_val) save_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv', stats_dict=total_losses, current_epoch=i, continue_from_mode=True if (self.starting_epoch != 0 or i > 0) else False) # save statistics to stats file. # load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to out_string = ", ".join([ "{}_{:.4f}".format(key, np.mean(value)) for key, value in current_epoch_losses.items() ]) # create a string to use to report our epoch metrics epoch_elapsed_time = time.time( ) - epoch_start_time # calculate time taken for epoch epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time) print("Epoch {}:".format(epoch_idx), out_string, "epoch time", epoch_elapsed_time, "seconds") self.state['model_epoch'] = epoch_idx if (self.best_val_model_idx == epoch_idx): # replace best model #remove all model files, THEN save new best one. for model_filename in os.listdir(self.experiment_saved_models): current_model = os.path.join(self.experiment_saved_models, model_filename) os.remove(current_model) self.save_model( model_save_dir=self.experiment_saved_models, # save model and best val idx and best val acc, using the model dir, model name and model idx model_save_name="train_model", model_idx=epoch_idx, best_validation_model_idx=self.best_val_model_idx, best_validation_model_acc=self.best_val_model_acc) # save(update) latest model every epoch self.save_model( model_save_dir=self.experiment_saved_models, # save model and best val idx and best val acc, using the model dir, model name and model idx model_save_name="train_model", model_idx='latest', best_validation_model_idx=self.best_val_model_idx, best_validation_model_acc=self.best_val_model_acc) # early stopping # criteria = int(self.num_epochs * 0.1) # window_valloss = total_losses['val_loss'][-(criteria+1):] # sorted_window_valloss = sorted(window_valloss,reverse = False) #if (len(window_valloss) >= (criteria + 1)): # if (sorted_window_valloss[0] == window_valloss[0]): # print("Early Stop at Epoch {}:".format(epoch_idx), "Best val model index", self.best_val_model_idx, "Best val model accuracy", self.best_val_model_acc) # break print("Generating test set evaluation metrics") self.load_model( model_save_dir=self.experiment_saved_models, model_idx=self.best_val_model_idx, # # load best validation model model_save_name="train_model") current_epoch_losses = { "test_Gender_acc": [], "test_SER_acc": [], "test_loss": [] } # initialize a statistics dict self.eval_current_epoch_emo_count = {0: 0, 1: 0, 2: 0, 3: 0} self.eval_current_epoch_correct_count = {0: 0, 1: 0, 2: 0, 3: 0} with tqdm.tqdm(total=len(self.test_data), ascii=True) as pbar_test: # ini a progress bar try: for x, y, z in self.test_data: # sample batch loss, accuracy1, accuracy2 = self.run_evaluation_iter( x=x, y=y, z=z ) # compute loss and accuracy by running an evaluation step current_epoch_losses["test_loss"].append( loss) # save test loss current_epoch_losses["test_SER_acc"].append( accuracy1) # save test accuracy current_epoch_losses["test_Gender_acc"].append( accuracy2) # save test accuracy pbar_test.update(1) # update progress bar status pbar_test.set_description( "loss: {:.4f}, SER_ accuracy: {:.4f}, Test_ accuracy: {:.4f}," .format( loss, accuracy1, accuracy2)) # update progress bar string output torch.cuda.empty_cache() except KeyboardInterrupt: pbar_test.close() raise pbar_test.close() test_losses = { key: [np.mean(value)] for key, value in current_epoch_losses.items() } # save test set metrics in dict format ua_test = 0 for key, value in self.eval_current_epoch_correct_count.items(): ua_test += value / self.eval_current_epoch_emo_count[key] / 4 test_losses["ua_test"] = [ua_test] save_statistics( experiment_log_dir=self.experiment_logs, filename='test_summary.csv', # save test set metrics on disk in .csv format stats_dict=test_losses, current_epoch=0, continue_from_mode=False) # print('Sorting model files') # self.delete_model(model_save_dir=self.experiment_saved_models, model_save_name="train_model", # model_idx=self.best_val_model_idx) return total_losses, test_losses
def run_experiment(self): """ Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch :return: The summary current_epoch_losses from starting epoch to total_epochs. """ total_losses = { "train_acc": [], "train_loss": [], "val_acc": [], "val_loss": [], "val_f1": [], "val_precision": [], "val_recall": [], "curr_epoch": [] } # initialize a dict to keep the per-epoch metrics for i, epoch_idx in enumerate( range(self.starting_epoch, self.num_epochs)): epoch_start_time = time.time() current_epoch_losses = { "train_acc": [], "train_loss": [], "val_acc": [], "val_loss": [], "val_f1": [], "val_precision": [], "val_recall": [] } current_epoch_losses = self.run_training_epoch( current_epoch_losses) if self.scheduler is not None: self.scheduler.step() train_loss_average = np.mean(current_epoch_losses['train_loss']) if train_loss_average < self.best_train_loss: print(f'Saving Best Model') self.best_train_loss = train_loss_average self.save_model( model_save_dir=self.experiment_saved_models, # save model and best val idx and best val acc, using the model dir, model name and model idx model_save_name="train_model", model_idx='best', state=self.state) for key, value in current_epoch_losses.items(): total_losses[key].append(np.mean(value)) # get mean of all metrics of current epoch metrics dict, # to get them ready for storage and output on the terminal. total_losses['curr_epoch'].append(epoch_idx) save_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv', stats_dict=total_losses, current_epoch=i, continue_from_mode=True if (self.starting_epoch != 0 or i > 0) else False) # save statistics to stats file. # load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to out_string = "_".join([ "{}_{:.4f}".format(key, np.mean(value)) for key, value in current_epoch_losses.items() ]) # create a string to use to report our epoch metrics epoch_elapsed_time = time.time( ) - epoch_start_time # calculate time taken for epoch epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time) print("Epoch {}:".format(epoch_idx), out_string, "epoch time", epoch_elapsed_time, "seconds") self.state['current_epoch_idx'] = epoch_idx return total_losses
def run_experiment(self): """ Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch :return: The summary current_epoch_losses from starting epoch to total_epochs. """ total_losses = {"train_acc": [], "train_loss": [], "val_acc": [], "val_loss": []} # initialize a dict to keep the per-epoch metrics train_number_batches = int(math.ceil(self.training_instances/self.batch_size)) val_number_batches = int(math.ceil(self.val_instances/self.batch_size)) if self.stack: train_number_batches = 2*train_number_batches val_number_batches = 2*val_number_batches for i, epoch_idx in enumerate(range(self.starting_epoch, self.num_epochs)): epoch_start_time = time.time() current_epoch_losses = {"train_acc": [], "train_loss": [], "val_acc": [], "val_loss": []} print("num batches",train_number_batches) if self.stack: total_ = train_number_batches-2 else: total_ = train_number_batches-1 if self.shuffle: idx = np.arange(0,self.train_data.inputs.shape[0]) print("before shuffle", self.train_data.inputs.shape) np.random.shuffle(idx) self.train_data.inputs = self.train_data.inputs[idx] self.train_data.targets = self.train_data.targets[idx] with tqdm.tqdm(total=total_) as pbar_train: # create a progress bar for training for idx in range(total_): x,y,manual_verified = self.get_batch(data = self.train_data, idx = idx, number_batches = total_,train=True) loss, accuracy = self.run_train_iter(x=x, y=y,manual_verified=manual_verified,epoch_number = epoch_idx) # take a training iter step current_epoch_losses["train_loss"].append(loss) # add current iter loss to the train loss list current_epoch_losses["train_acc"].append(accuracy) # add current iter acc to the train acc list pbar_train.update(1) pbar_train.set_description("loss: {:.4f}, accuracy: {:.4f}".format(loss, accuracy)) with tqdm.tqdm(total=val_number_batches) as pbar_val: # create a progress bar for validation for idx in range(val_number_batches): x,y = self.get_batch(data = self.val_data, idx = idx, number_batches = val_number_batches) loss, accuracy = self.run_evaluation_iter(x=x, y=y) # run a validation iter current_epoch_losses["val_loss"].append(loss) # add current iter loss to val loss list. current_epoch_losses["val_acc"].append(accuracy) # add current iter acc to val acc lst. pbar_val.update(1) # add 1 step to the progress bar pbar_val.set_description("loss: {:.4f}, accuracy: {:.4f}".format(loss, accuracy)) val_mean_accuracy = np.mean(current_epoch_losses['val_acc']) if val_mean_accuracy > self.best_val_model_acc: # if current epoch's mean val acc is greater than the saved best val acc then self.best_val_model_acc = val_mean_accuracy # set the best val model acc to be current epoch's val accuracy self.best_val_model_idx = epoch_idx # set the experiment-wise best val idx to be the current epoch's idx for key, value in current_epoch_losses.items(): total_losses[key].append(np.mean( value)) # get mean of all metrics of current epoch metrics dict, to get them ready for storage and output on the terminal. save_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv', stats_dict=total_losses, current_epoch=i) # save statistics to stats file. load_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv') # How to load a csv file if you need to out_string = "_".join( ["{}_{:.4f}".format(key, np.mean(value)) for key, value in current_epoch_losses.items()]) # create a string to use to report our epoch metrics epoch_elapsed_time = time.time() - epoch_start_time # calculate time taken for epoch epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time) print("Epoch {}:".format(epoch_idx), out_string, "epoch time", epoch_elapsed_time, "seconds") self.state['current_epoch_idx']=epoch_idx self.state['best_val_model_acc']=self.best_val_model_acc self.state['best_val_model_idx']=self.best_val_model_idx self.save_model(model_save_dir=self.experiment_saved_models, # save model and best val idx and best val acc, using the model dir, model name and model idx model_save_name="train_model", model_idx=epoch_idx,state=self.state) self.save_model(model_save_dir=self.experiment_saved_models, model_save_name="train_model",model_idx='latest',state=self.state) print("Generating test set evaluation metrics") self.load_model(model_save_dir=self.experiment_saved_models, model_idx=self.best_val_model_idx, # load best validation model model_save_name="train_model") current_epoch_losses = {"test_acc": [], "test_loss": []} # initialize a statistics dict test_number_batches = int(math.ceil(self.test_instances/self.batch_size)) if self.stack: test_number_batches = 2*test_number_batches with tqdm.tqdm(total=test_number_batches) as pbar_test: # ini a progress bar for idx in range(test_number_batches): # sample batch x,y = self.get_batch(data = self.test_data, idx = idx, number_batches = test_number_batches) loss, accuracy = self.run_evaluation_iter(x=x, y=y) # compute loss and accuracy by running an evaluation step current_epoch_losses["test_loss"].append(loss) # save test loss current_epoch_losses["test_acc"].append(accuracy) # save test accuracy pbar_test.update(1) # update progress bar status pbar_test.set_description( "loss: {:.4f}, accuracy: {:.4f}".format(loss, accuracy)) # update progress bar string output test_losses = {key: [np.mean(value)] for key, value in current_epoch_losses.items()} # save test set metrics in dict format save_statistics(experiment_log_dir=self.experiment_logs, filename='test_summary.csv', #save test set metrics on disk in .csv format stats_dict=test_losses, current_epoch=0) return total_losses, test_losses
def run_experiment(self): """ Runs experiment train and evaluation iterations, saving the model and best val model and val model accuracy after each epoch :return: The summary current_epoch_losses from starting epoch to total_epochs. """ if self.continue_from_epoch == -1: self.model.reset_parameters() for i, epoch_idx in enumerate( range(self.starting_epoch, self.num_epochs)): epoch_start_time = time.time() current_epoch_losses = { "train_loss": [], "val_loss": [], "val_nrmse_loss": [] } with tqdm.tqdm( total=self.train_data.num_batches ) as pbar_train: # create a progress bar for training for idx, (x, y) in enumerate(self.train_data): # get data batches loss = self.run_train_iter( x=x, y=y) # take a training iter step current_epoch_losses["train_loss"].append( loss) # add current iter loss to the train loss list pbar_train.update(1) pbar_train.set_description("loss: {:.4f}".format(loss)) with tqdm.tqdm( total=self.val_data.num_batches ) as pbar_val: # create a progress bar for validation for x, y in self.val_data: # get data batches mse_loss, nrmse_loss = self.run_evaluation_iter( x=x, y=y) # run a validation iter current_epoch_losses["val_loss"].append( mse_loss) # add current iter loss to val loss list. current_epoch_losses["val_nrmse_loss"].append(nrmse_loss) pbar_val.update(1) # add 1 step to the progress bar pbar_val.set_description("loss: {:.4f}".format(loss)) val_mean_loss = np.mean(current_epoch_losses['val_loss']) # if lowest validation loss was achieved in this epoch if val_mean_loss < self.best_val_loss: self.best_val_loss = val_mean_loss self.best_val_model_idx = epoch_idx val_nrmse_mean_loss = np.mean( current_epoch_losses['val_nrmse_loss']) # if lowest nrmse validation loss was achieved in this epoch if val_nrmse_mean_loss < self.best_val_nrmse_loss: self.best_val_nrmse_loss = val_nrmse_mean_loss self.best_val_nrmse_model_idx = epoch_idx # get mean of all metrics of current epoch metrics dict, # to get them ready for storage and output on the terminal. for key, value in current_epoch_losses.items(): self.metrics[key].append(np.mean(value)) self.metrics['curr_epoch'].append(epoch_idx) save_statistics(experiment_log_dir=self.experiment_logs, filename='summary.csv', stats_dict=self.metrics, current_epoch=epoch_idx, continue_from_mode=(self.starting_epoch != 0 or i > 0)) save_best_val_scores( experiment_log_dir=self.experiment_logs, filename='best_val_scores.csv', best_val_loss=self.best_val_loss, best_val_model_idx=self.best_val_model_idx, best_val_nrmse_loss=self.best_val_nrmse_loss, best_val_nrmse_model_idx=self.best_val_nrmse_model_idx) out_string = "_".join([ "{}_{:.4f}".format(key, np.mean(value)) for key, value in current_epoch_losses.items() ]) # create a string to use to report our epoch metrics epoch_elapsed_time = time.time( ) - epoch_start_time # calculate time taken for epoch epoch_elapsed_time = "{:.4f}".format(epoch_elapsed_time) print("Epoch {}:".format(epoch_idx), out_string, "epoch time", epoch_elapsed_time, "seconds") # save the current model self.save_model(model_save_dir=self.experiment_saved_models, model_save_name="train_model", epoch_idx=epoch_idx)