def train_and_validate(model, criterion, optimizer, train_data, val_data, metric, mtype, ctype=0): """ Defines the training and validation cycle for the input batched data for the conv model Args: model: object, trained model of PitchContourAssessor class criterion: object, of torch.nn.Functional class which defines the loss optimizer: object, of torch.optim class which defines the optimization algorithm train_data: list, batched training data val_data: list, batched validation data metric: int, from 0 to 3, which metric to evaluate against mtype: string, 'conv' for fully convolutional model, 'lstm' for lstm based model ctype: int, 0 for regression, 1 for classification """ # train the network train(model, criterion, optimizer, train_data, metric, mtype, ctype) # evaluate the network on train data train_loss_avg, train_r_sq, train_accu, train_accu2 = eval_utils.eval_model( model, criterion, train_data, metric, mtype, ctype) # evaluate the network on validation data val_loss_avg, val_r_sq, val_accu, val_accu2 = eval_utils.eval_model( model, criterion, val_data, metric, mtype, ctype) # return values return train_loss_avg, train_r_sq, train_accu, train_accu2, val_loss_avg, val_r_sq, val_accu, val_accu2
def train_and_validate(model, criterion, optimizer, train_data, val_data, metric): """ Defines the training and validation cycle for the input batched data for the conv model Args: model: object, trained model of PitchContourAssessor class criterion: object, of torch.nn.Functional class which defines the loss optimizer: object, of torch.optim class which defines the optimization algorithm train_data: list, batched training data val_data: list, batched validation data metric: int, from 0 to 3, which metric to evaluate against """ # train the network train(model, criterion, optimizer, train_data, metric) # evaluate the network on train data train_loss_avg, train_r_sq, train_accu = eval_utils.eval_model( model, criterion, train_data, metric) # evaluate the network on validation data val_loss_avg, val_r_sq, val_accu = eval_utils.eval_model( model, criterion, val_data, metric) # return values return train_loss_avg, train_r_sq, train_accu, val_loss_avg, val_r_sq, val_accu
else: perf_model.load_state_dict( torch.load('saved/' + filename + '.pt', map_location=lambda storage, loc: storage)) # initialize dataset, dataloader and created batched data file_name = BAND + '_' + str(SEGMENT) + '_data' if sys.version_info[0] < 3: data_path = 'dat/' + file_name + '.dill' else: data_path = 'dat/' + file_name + '_3.dill' dataset = PitchContourDataset(data_path) dataloader = PitchContourDataloader(dataset, NUM_DATA_POINTS, NUM_BATCHES) _, _, vef, _, tef = dataloader.create_split_data(1000, 500) # test on full length data test_loss, test_r_sq, test_accu, test_accu2, pred, target = eval_utils.eval_model( perf_model, criterion, tef, METRIC, MTYPE, CTYPE, 1) print('[%s %0.5f, %s %0.5f, %s %0.5f %0.5f]' % ('Testing Loss: ', test_loss, ' R-sq: ', test_r_sq, ' Accu:', test_accu, test_accu2)) # convert to numpy if torch.cuda.is_available(): pred = pred.clone().cpu().numpy() target = target.clone().cpu().numpy() else: pred = pred.clone().numpy() target = target.clone().numpy() a = np.absolute((pred - target)) # compute correlation coefficient R, p = ss.pearsonr(pred, target)
print('[%s %0.5f, %s %0.5f, %s %0.5f]'% ('Valid Loss: ', val_loss, ' R-sq: ', val_r_sq, ' Accu:', val_accu)) # save model if best validation loss if val_loss < best_val_loss: n = file_info + '_best' train_utils.save(n, perf_model) best_val_loss = val_loss print("Saving...") train_utils.save(file_info, perf_model, log_parameters) except KeyboardInterrupt: print("Saving before quit...") train_utils.save(file_info, perf_model, log_parameters) # RUN VALIDATION SET ON THE BEST MODEL # read the best model filename = file_info + '_best' + '_Reg' if torch.cuda.is_available(): perf_model.cuda() perf_model.load_state_dict(torch.load('saved/' + filename + '.pt')) else: perf_model.load_state_dict(torch.load('saved/' + filename + '.pt', map_location=lambda storage, loc: storage)) # run on validation set val_loss, val_r_sq, val_accu = eval_utils.eval_model(perf_model, criterion, validation_data, METRIC) print('[%s %0.5f, %s %0.5f, %s %0.5f]'% ('Best Valid Loss: ', val_loss, ' R-sq: ', val_r_sq, ' Accu:', val_accu)) print('Saving best model with timestamp') train_utils.save(file_info + time.strftime("_%m-%d_%H-%M-%S"), perf_model, log_parameters)
# store the best r-squared value from training if val_r_sq > best_valrsq: best_valrsq = val_r_sq if best_epoch < epoch - 200: break print("Saving...") train_utils.save(NAME, perf_model) except KeyboardInterrupt: print("Saving before quit...") train_utils.save(NAME, perf_model) print('BEST R^2 VALUE: ' + str(best_valrsq)) # test # test of full length data test_loss, test_r_sq, test_accu, test_accu2 = eval_utils.eval_model(perf_model, criterion, testing_data, METRIC, MTYPE, CTYPE) print('[%s %0.5f, %s %0.5f, %s %0.5f %0.5f]'% ('Testing Loss: ', test_loss, ' R-sq: ', test_r_sq, ' Accu:', test_accu, test_accu2)) # validate and test on best validation model # read the model #filename = file_info + '_Reg' filename = NAME + '_best' if torch.cuda.is_available(): perf_model.cuda() #perf_model.load_state_dict(torch.load('/Users/michaelfarren/Desktop/MusicPerfAssessment-master/src/runs/' + filename + '.pt')) perf_model.load_state_dict(torch.load('pc_runs/' + NAME)) else: perf_model.load_state_dict(torch.load('pc_runs/' + filename + '.pt', map_location=lambda storage, loc: storage)) val_loss, val_r_sq, val_accu, val_accu2 = eval_utils.eval_model(perf_model, criterion, vef, METRIC, MTYPE, CTYPE)
# import data from MAST dataset if sys.version_info[0] < 3: mast_path = '/Users/Som/GitHub/Mastmelody_dataset/f0data' else: mast_path = '/home/apati/MASTmelody_dataset/f0data' mast_dataset = MASTDataset(mast_path) mast_len = mast_dataset.__len__() mast_data = [] fail_count = 0 for i in range(mast_len): f0, target = mast_dataset.__getitem__(i) if target == 0: fail_count += 1 if fail_count > 266: continue d = {} pitch_tensor = torch.from_numpy(f0).float() pitch_tensor = pitch_tensor.view(1, -1) if pitch_tensor.size(1) < 1000: pitch_tensor = torch.cat( (pitch_tensor, torch.zeros(1, 1000 - pitch_tensor.size(1))), 1) d['pitch_tensor'] = pitch_tensor d['score_tensor'] = torch.from_numpy(np.ones((1, 4)) * target).float() mast_data.append(d) # evaluate model on MAST dataset test_loss, test_r_sq, test_accu, bin_accu, pred, target = eval_model( perf_model, criterion, mast_data, METRIC, MTYPE, extra_outs=1) print('[%s %0.5f]' % ('MAST Accuracy: ', bin_accu))