def main(): train_loader, valid_loader, test_loader = preprocess() model = DenseNet121(out_size=14) save_file = 'densenet.pth' criterion = torch.nn.BCELoss(size_average=True) optimizer = optim.Adam(model.parameters()) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device) criterion.to(device) best_val_acc = 0.0 train_losses, train_accuracies = [], [] valid_losses, valid_accuracies = [], [] for epoch in range(NUM_EPOCHS): train_loss, train_accuracy = train(model, device, train_loader, criterion, optimizer, epoch) valid_loss, valid_accuracy, valid_results = evaluate( model, device, valid_loader, criterion) train_losses.append(train_loss) valid_losses.append(valid_loss) train_accuracies.append(train_accuracy) valid_accuracies.append(valid_accuracy) is_best = valid_accuracy > best_val_acc # let's keep the model that has the best accuracy, but you can also use another metric. if is_best: best_val_acc = valid_accuracy torch.save(model, os.path.join(PATH_OUTPUT, save_file)) plot_learning_curves(train_losses, valid_losses, train_accuracies, valid_accuracies, PATH_OUTPUT) best_model = torch.load(os.path.join(PATH_OUTPUT, save_file)) test_loss, test_accuracy, test_results = evaluate(best_model, device, test_loader, criterion)
train_losses.append(train_loss) valid_losses.append(valid_loss) train_accuracies.append(train_accuracy) valid_accuracies.append(valid_accuracy) is_best = valid_loss < best_val_losses if is_best: print('Found Best') print(valid_loss) best_val_losses = valid_loss torch.save(model, os.path.join(PATH_OUTPUT, save_file)) print('Saved Best Model') plot_learning_curves(train_losses, valid_losses, train_accuracies, valid_accuracies) best_model = torch.load(os.path.join(PATH_OUTPUT, save_file)) best_model.to(device) print('Train Accuracies') print(train_accuracies) print('Valid Accuracies') print(valid_accuracies) print('Train Losses') print(train_losses) print('Valid Losses') print(valid_losses)
testloader = DataLoader(testset, batch_size=128, shuffle=True, num_workers=2) dset_loaders = {'train': trainloader, 'val': testloader} dset_sizes = { 'train': len(trainset.train_labels), 'val': len(testset.test_labels) } # Defining the model model = resnet.ResNet18() if torch.cuda.is_available(): model.cuda() criterion = CrossEntropyLoss() optimizer = Adam(model.parameters(), lr=0.001, betas=(0.9, 0.999), weight_decay=5e-4) # Training the network train_outputs = train_model(model, dset_loaders, dset_sizes, criterion, optimizer, num_epochs=90, verbose=2) plot_learning_curves(train_outputs[0], train_outputs[1], train_outputs[2], train_outputs[3], description='adam')
# Save results and confusion matrix in case of time-out with open( C.OUTPUT_DIR + "{}results_{}.csv".format("full_", model.details), "w") as f: f.write("true,pred\n") for r in valid_results: f.write("{},{}\n".format(r[0], r[1])) f.close() class_names = ['0', '1', '2', '3', '4'] plot_confusion_matrix(valid_results, class_names, "full_", model.details) # plot learning curves plot_learning_curves(train_losses, valid_losses, train_accuracies, valid_accuracies, "full_", model.details) """ HYPERPARAMETER TUNING (COMMENT OUT IF NOT USING) """ # unit_range = [8, 16, 24, 32, 64] # layer_range = [2, 3, 4] # for fold in range(0, 3): # print("FOLD #{}".format(fold)) # # Data loading # print("Train set loading") # train_path = C.SPLITS_DIR + 'cv_train{}.txt'.format(fold) # train_loader = import_to_dataloader(train_path) # # print("Valid set loading") # valid_path = C.SPLITS_DIR + 'cv_valid{}.txt'.format(fold) # valid_loader = import_to_dataloader(valid_path)
avg_f1 += f1 avg_loss += loss test_step += 1 test_writer.add_summary(summary_str, test_step) valid_costs.append(avg_loss/epoch_step) valid_accuracies.append(acc/epoch_step) except tf.errors.OutOfRangeError: print("Average test set accuracy over {} iterations is {:.2f}%".format(test_step,(avg_acc / test_step) * 100)) print("Average epoch precision is {:.2f}%".format((avg_precision / test_step) *100)) print("Average epoch recall is {:.2f}%".format((avg_recall / test_step) *100)) print("Average epoch f1-score is {:.2f}".format((avg_f1 / test_step))) print("Average test set loss over {} iterations is {:.2f}".format(test_step,(avg_loss / test_step))) print("\n") #plot plot_learning_curves(train_costs, valid_costs, train_accuracies, valid_accuracies, loss_fig=disease+"_Loss.png", accuracy_fig=disease+"_accuracy.png") # Test print("Test\n") sess.run(test_init) avg_acc = 0 avg_precision = 0 avg_recall = 0 avg_f1 = 0 avg_loss = 0 test_step=0 try: while True: acc, y_pred, y_true, loss, summary_str = sess.run( [accuracy, predicted, actual, cost, summ],
import numpy as np import plots as p # o = np.array([0.44, 0.49, 0.48, 0.51, 0.515, 0.52, 0.505, 0.51, 0.52, 0.525]) # s_and_p = np.array([0.44, 0.47, 0.49, 0.51, 0.515, 0.52, 0.505, 0.50, 0.53, 0.538]) # p.plot_learning_curves(o, s_and_p) # np.save('original', o) original = np.load('../output/cnn_original.npy') # s_and_p = np.load('../output/cnn_s_and_p.npy') reflection = np.load('../output/cnn_reflection.npy') # rotation = np.load('../output/cnn_rotation.npy') # p.plot_learning_curves(original, s_and_p, rotation, reflection) p.plot_learning_curves(original, reflection, original, original) print("done")
for epoch in range(NUM_EPOCHS): train_loss, train_auc_lb = train(model, device, train_loader, criterion, optimizer, epoch) valid_loss, valid_auc_lb, valid_results = evaluate(model, device, valid_loader, criterion) train_losses.append(train_loss) valid_losses.append(valid_loss) train_auc_lbs.append(train_auc_lb) valid_auc_lbs.append(valid_auc_lb) # train_auc_ubs.append(train_auc_ub) # valid_auc_ubs.append(valid_auc_ub) is_best = np.mean((valid_auc_lb)) > best_mean_auc # let's keep the model that has the best avg. AUC, but you can also use another metric. if is_best: best_mean_auc = np.mean((valid_auc_lb)) torch.save(model, "CheXpert-model.pt") plot_learning_curves(train_losses, valid_losses, train_auc_lbs, valid_auc_lbs) best_model = torch.load("CheXpert-model.pt") test_loss, test_auc_lb, test_results = evaluate(best_model, device, test_loader, criterion, print_freq=1) r = np.array(test_results) pc_f1 = [f1_score(r[:,0,label], r[:,1,label]) for label in np.arange(r.shape[2])] print("Per-class F1: ", np.around(pc_f1, 3)) print("Avg. F1: ", np.around(np.mean(pc_f1, axis = 0), 3)) class_names = ["No Finding", "Enlarged Cardiomediastinum", "Cardiomegaly", "Lung Opacity", "Lung Lesion", "Edema", "Consolidation", "Pneumonia", "Atelectasis", "Pneumothorax", "Pleural Effusion", "Pleural Other", "Fracture", "Support Devices"] plot_confusion_matrix(test_results, class_names)
criterion, optimizer, epochs=p['epochs'], lr_min=p['lr_min'], lr_max=p['lr_max'], mom_min=p['mom_min'], mom_max=p['mom_max'], annealing_pct=p['annealing_pct'], verbose=2) # Plotting the learning curves description = 'lrmin{}_lrmax{}_mmin{}_mmax{}_pct{}_e{}'.format( p['lr_min'], p['lr_max'], p['mom_min'], p['mom_max'], p['annealing_pct'], p['epochs']) plot_learning_curves(curves[0], curves[1], curves[2], curves[3], description=description) plot_learning_rates(curves[4], description=description) plot_momentums(curves[5], description=description) print(description) print('train loss: {}, val loss: {}, train acc: {}, val_acc:{}'.format( curves[0][-1], curves[1][-1], curves[2][-1], curves[3][-1])) # Save all curves to disk for later plotting for i in range(6): np.save('../learning_curves/' + description + '-' + str(i), np.array(curves[i]))
optimizer, epoch) valid_loss, valid_accuracy, valid_results = evaluate( model, device, valid_loader, criterion) train_losses.append(train_loss) valid_losses.append(valid_loss) train_accuracies.append(train_accuracy) valid_accuracies.append(valid_accuracy) is_best = valid_accuracy > best_val_acc # let's keep the model that has the best accuracy, but you can also use another metric. if is_best: best_val_acc = valid_accuracy torch.save(model, os.path.join(PATH_OUTPUT, "MyVariableRNN.pth")) plot_learning_curves(train_losses, valid_losses, train_accuracies, valid_accuracies, PATH_OUTPUT) best_model = torch.load(os.path.join(PATH_OUTPUT, "MyVariableRNN.pth")) # TODO: For your report, try to make plots similar to those in the previous task. # TODO: You may use the validation set in case you cannot use the test set. # TODO: Complete predict_mortality def predict_mortality(model, device, data_loader): model.eval() # TODO: Evaluate the data (from data_loader) using model, # TODO: return a List of probabilities probas = [] with torch.no_grad(): for i, (input, target) in enumerate(data_loader):
import argparse from analog.load import ExperimentLog from dataloader import loader from plots import plot_learning_curves parser = argparse.ArgumentParser() parser.add_argument('--logdir', type=str, required=True) parser.add_argument('--exp_names', nargs='+', type=str, required=True) parser.add_argument('--std_type', type=str, choices=['time', 'run']) parser.add_argument('--min_t', type=float, default=0.) parser.add_argument('--max_t', type=float, default=100.) args = parser.parse_args() start_date = 'last' stop_date = None expdata: ExperimentLog = ExperimentLog() for exp_name in args.exp_names: expdata.extend(loader(args.logdir, exp_name, start_date=start_date, stop_date=stop_date)) # def filter(args) -> bool: # return ('noscale' in args and args['noscale'] and 'value' in args['algo']) \ # or 'advantage' in args['algo'] # expdata = filter(expdata, filter) plot_learning_curves(expdata, ['Return'], args.exp_names[0], mint=args.min_t, maxt=args.max_t, gtype=args.std_type + "_std") expdata.repr_rawlogs("Return", 5)