def run(args, kwargs): args.model_signature = str(datetime.datetime.now())[0:19] model_name = '' + args.model_name if args.loc_gauss or args.loc_inv_q or args.loc_att: args.loc_info = True if args.att_gauss_abnormal or args.att_inv_q_abnormal or args.att_gauss_spatial or args.att_inv_q_spatial or \ args.att_module or args.laplace_att: args.self_att = True print(args) with open('experiment_log_' + args.operator + '.txt', 'a') as f: print(args, file=f) # IMPORT MODEL====================================================================================================== from models.CNN import CNN as Model # START KFOLDS====================================================================================================== print('\nSTART KFOLDS CROSS VALIDATION\n') # print(f'{args.kfold_test} Test-Train folds each has {args.epochs} epochs for a {1.0/args.kfold_val}/{(args.kfold_val - 1.0)/args.kfold_val} Valid-Train split\n') train_folds, test_folds = kfold_indices_warwick(args.dataset_size, args.kfold_test, seed=args.seed) train_error_folds = [] test_error_folds = [] for current_fold in range(1, args.kfold_test + 1): print('#################### Train-Test fold: {}/{} ####################'.format(current_fold, args.kfold_test)) # DIRECTORY FOR SAVING========================================================================================== snapshots_path = 'snapshots/' dir = snapshots_path + model_name + '_' + args.model_signature + '/' sw = SummaryWriter(f'tensorboard/{model_name}_{args.model_signature}_fold_{current_fold}') if not os.path.exists(dir): os.makedirs(dir) # LOAD DATA===================================================================================================== train_fold, val_fold = kfold_indices_warwick(len(train_folds[current_fold - 1]), args.kfold_val, seed=args.seed) train_fold = [train_folds[current_fold - 1][i] for i in train_fold] val_fold = [train_folds[current_fold - 1][i] for i in val_fold] loc = True if args.loc_info or args.out_loc else False train_set, val_set, test_set = load_breast(train_fold[0], val_fold[0], test_folds[current_fold - 1], loc) # CREATE MODEL================================================================================================== print('\tcreate models') model = Model(args) if args.cuda: model.cuda() # INIT OPTIMIZER================================================================================================ print('\tinit optimizer') if args.optimizer == 'Adam': optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(0.9, 0.999), weight_decay=args.reg) elif args.optimizer == 'SGD': optimizer = optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.reg, momentum=0.9) else: raise Exception('Wrong name of the optimizer!') scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=60, gamma=0.1) # PERFORM EXPERIMENT============================================================================================ print('\tperform experiment\n') train_error, test_error = experiment( args, kwargs, current_fold, train_set, val_set, test_set, model, optimizer, scheduler, dir, sw, ) # APPEND FOLD RESULTS=========================================================================================== train_error_folds.append(train_error) test_error_folds.append(test_error) with open('final_results_' + args.operator + '.txt', 'a') as f: print('RESULT FOR A SINGLE FOLD\n' 'SEED: {}\n' 'OPERATOR: {}\n' 'FOLD: {}\n' 'ERROR (TRAIN): {}\n' 'ERROR (TEST): {}\n\n'.format(args.seed, args.operator, current_fold, train_error, test_error), file=f) # ====================================================================================================================== print('-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-') with open('experiment_log_' + args.operator + '.txt', 'a') as f: print('-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n', file=f) return np.mean(train_error_folds), np.std(train_error_folds), np.mean(test_error_folds), np.std(test_error_folds)
def main(): songs = get_notes() vocab_set = set() for song in songs: for note in song: vocab_set.add(note) n_in, n_out = prep_sequences(songs, sequence_length=100) X_train, X_val, y_train, y_val = train_test_split(n_in, n_out, test_size=0.2) train_ds = MusicDataset(X_train, y_train) val_ds = MusicDataset(X_val, y_val) train_dataloader = DataLoader(train_ds, batch_size=512, shuffle=True, num_workers=0) val_dataloader = DataLoader(val_ds, batch_size=512, shuffle=False, num_workers=0) model = CNN(100, len(vocab_set)) model.cuda() epochs = 25 initial_lr = 0.001 optimizer = optim.Adam(model.parameters(), lr=initial_lr) scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, epochs) loss_fn = CrossEntropyLoss() train_losses = [] val_losses = [] train_accuracies = [] val_accuracies = [] for epoch in tqdm(range(1, epochs + 1)): model.train() train_loss_total = 0.0 num_steps = 0 correct = 0 ### Train for i, batch in enumerate(train_dataloader): X, y = batch[0].cuda(), batch[1].cuda() train_preds = model(X) loss = loss_fn(train_preds, y) train_loss_total += loss.item() optimizer.zero_grad() loss.backward() optimizer.step() num_steps += 1 train_preds = torch.max(train_preds, 1)[1] correct += (train_preds == y).float().sum() train_loss_total_avg = train_loss_total / num_steps train_accuracy = correct / len(train_ds) train_accuracies.append(train_accuracy) train_losses.append(train_loss_total_avg) model.eval() val_loss_total = 0.0 num_steps = 0 correct = 0 for i, batch in enumerate(val_dataloader): with torch.no_grad(): X, y = batch[0].cuda(), batch[1].cuda() val_preds = model(X) loss = loss_fn(val_preds, y) val_loss_total += loss.item() val_preds = torch.max(val_preds, 1)[1] correct += (val_preds == y).float().sum() num_steps += 1 val_loss_total_avg = val_loss_total / num_steps val_accuracy = correct / len(val_ds) val_accuracies.append(val_accuracy) val_losses.append(val_loss_total_avg) scheduler.step() print('\nTrain loss: {:.4f}'.format(train_loss_total_avg)) print('Train accuracy: {:.4f}'.format(train_accuracy)) print('Val loss: {:.4f}'.format(val_loss_total_avg)) print('Val accuracy\n: {:.4f}'.format(val_accuracy)) torch.save(model.state_dict(), "weights/model_params_epoch" + str(epoch)) torch.save(optimizer.state_dict(), "weights/optim_params_epoch" + str(epoch)) plt.xlabel("Epoch") plt.ylabel("Accuracy") plt.plot(range(1, len(train_accuracies) + 1), train_accuracies) plt.plot(range(1, len(val_accuracies) + 1), val_accuracies) plt.savefig("plots/accuracies.png") plt.close() plt.xlabel("Epoch") plt.ylabel("Loss") plt.plot(range(1, len(train_losses) + 1), train_losses) plt.plot(range(1, len(val_losses) + 1), val_losses) plt.savefig("plots/losses.png") plt.close() generate_midi(model, val_ds, vocab_set, output_filename="output.mid")