def main():
train_loader, valid_loader, test_loader = preprocess()
model = DenseNet121(out_size=14)
save_file = 'densenet.pth'
criterion = torch.nn.BCELoss(size_average=True)
optimizer = optim.Adam(model.parameters())
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
criterion.to(device)
best_val_acc = 0.0
train_losses, train_accuracies = [], []
valid_losses, valid_accuracies = [], []
for epoch in range(NUM_EPOCHS):
train_loss, train_accuracy = train(model, device, train_loader,
criterion, optimizer, epoch)
valid_loss, valid_accuracy, valid_results = evaluate(
model, device, valid_loader, criterion)
train_losses.append(train_loss)
valid_losses.append(valid_loss)
train_accuracies.append(train_accuracy)
valid_accuracies.append(valid_accuracy)
is_best = valid_accuracy > best_val_acc # let's keep the model that has the best accuracy, but you can also use another metric.
if is_best:
best_val_acc = valid_accuracy
torch.save(model, os.path.join(PATH_OUTPUT, save_file))
plot_learning_curves(train_losses, valid_losses, train_accuracies,
valid_accuracies, PATH_OUTPUT)
best_model = torch.load(os.path.join(PATH_OUTPUT, save_file))
test_loss, test_accuracy, test_results = evaluate(best_model, device,
test_loader, criterion)
train_losses.append(train_loss)
valid_losses.append(valid_loss)
train_accuracies.append(train_accuracy)
valid_accuracies.append(valid_accuracy)
is_best = valid_loss < best_val_losses
if is_best:
print('Found Best')
print(valid_loss)
best_val_losses = valid_loss
torch.save(model, os.path.join(PATH_OUTPUT, save_file))
print('Saved Best Model')
plot_learning_curves(train_losses, valid_losses, train_accuracies, valid_accuracies)
best_model = torch.load(os.path.join(PATH_OUTPUT, save_file))
best_model.to(device)
print('Train Accuracies')
print(train_accuracies)
print('Valid Accuracies')
print(valid_accuracies)
print('Train Losses')
print(train_losses)
print('Valid Losses')
print(valid_losses)
testloader = DataLoader(testset, batch_size=128, shuffle=True, num_workers=2)
dset_loaders = {'train': trainloader, 'val': testloader}
dset_sizes = {
'train': len(trainset.train_labels),
'val': len(testset.test_labels)
}
# Defining the model
model = resnet.ResNet18()
if torch.cuda.is_available():
model.cuda()
criterion = CrossEntropyLoss()
optimizer = Adam(model.parameters(),
lr=0.001,
betas=(0.9, 0.999),
weight_decay=5e-4)
# Training the network
train_outputs = train_model(model,
dset_loaders,
dset_sizes,
criterion,
optimizer,
num_epochs=90,
verbose=2)
plot_learning_curves(train_outputs[0],
train_outputs[1],
train_outputs[2],
train_outputs[3],
description='adam')
# Save results and confusion matrix in case of time-out
with open(
C.OUTPUT_DIR +
"{}results_{}.csv".format("full_", model.details),
"w") as f:
f.write("true,pred\n")
for r in valid_results:
f.write("{},{}\n".format(r[0], r[1]))
f.close()
class_names = ['0', '1', '2', '3', '4']
plot_confusion_matrix(valid_results, class_names, "full_",
model.details)
# plot learning curves
plot_learning_curves(train_losses, valid_losses, train_accuracies,
valid_accuracies, "full_", model.details)
"""
HYPERPARAMETER TUNING (COMMENT OUT IF NOT USING)
"""
# unit_range = [8, 16, 24, 32, 64]
# layer_range = [2, 3, 4]
# for fold in range(0, 3):
# print("FOLD #{}".format(fold))
# # Data loading
# print("Train set loading")
# train_path = C.SPLITS_DIR + 'cv_train{}.txt'.format(fold)
# train_loader = import_to_dataloader(train_path)
#
# print("Valid set loading")
# valid_path = C.SPLITS_DIR + 'cv_valid{}.txt'.format(fold)
# valid_loader = import_to_dataloader(valid_path)
avg_f1 += f1
avg_loss += loss
test_step += 1
test_writer.add_summary(summary_str, test_step)
valid_costs.append(avg_loss/epoch_step)
valid_accuracies.append(acc/epoch_step)
except tf.errors.OutOfRangeError:
print("Average test set accuracy over {} iterations is {:.2f}%".format(test_step,(avg_acc / test_step) * 100))
print("Average epoch precision is {:.2f}%".format((avg_precision / test_step) *100))
print("Average epoch recall is {:.2f}%".format((avg_recall / test_step) *100))
print("Average epoch f1-score is {:.2f}".format((avg_f1 / test_step)))
print("Average test set loss over {} iterations is {:.2f}".format(test_step,(avg_loss / test_step)))
print("\n")
#plot
plot_learning_curves(train_costs, valid_costs, train_accuracies, valid_accuracies, loss_fig=disease+"_Loss.png", accuracy_fig=disease+"_accuracy.png")
# Test
print("Test\n")
sess.run(test_init)
avg_acc = 0
avg_precision = 0
avg_recall = 0
avg_f1 = 0
avg_loss = 0
test_step=0
try:
while True:
acc, y_pred, y_true, loss, summary_str = sess.run(
[accuracy, predicted, actual, cost, summ],
import numpy as np
import plots as p
# o = np.array([0.44, 0.49, 0.48, 0.51, 0.515, 0.52, 0.505, 0.51, 0.52, 0.525])
# s_and_p = np.array([0.44, 0.47, 0.49, 0.51, 0.515, 0.52, 0.505, 0.50, 0.53, 0.538])
# p.plot_learning_curves(o, s_and_p)
# np.save('original', o)
original = np.load('../output/cnn_original.npy')
# s_and_p = np.load('../output/cnn_s_and_p.npy')
reflection = np.load('../output/cnn_reflection.npy')
# rotation = np.load('../output/cnn_rotation.npy')
# p.plot_learning_curves(original, s_and_p, rotation, reflection)
p.plot_learning_curves(original, reflection, original, original)
print("done")
for epoch in range(NUM_EPOCHS):
train_loss, train_auc_lb = train(model, device, train_loader, criterion, optimizer, epoch)
valid_loss, valid_auc_lb, valid_results = evaluate(model, device, valid_loader, criterion)
train_losses.append(train_loss)
valid_losses.append(valid_loss)
train_auc_lbs.append(train_auc_lb)
valid_auc_lbs.append(valid_auc_lb)
# train_auc_ubs.append(train_auc_ub)
# valid_auc_ubs.append(valid_auc_ub)
is_best = np.mean((valid_auc_lb)) > best_mean_auc # let's keep the model that has the best avg. AUC, but you can also use another metric.
if is_best:
best_mean_auc = np.mean((valid_auc_lb))
torch.save(model, "CheXpert-model.pt")
plot_learning_curves(train_losses, valid_losses, train_auc_lbs, valid_auc_lbs)
best_model = torch.load("CheXpert-model.pt")
test_loss, test_auc_lb, test_results = evaluate(best_model, device, test_loader, criterion, print_freq=1)
r = np.array(test_results)
pc_f1 = [f1_score(r[:,0,label], r[:,1,label]) for label in np.arange(r.shape[2])]
print("Per-class F1: ", np.around(pc_f1, 3))
print("Avg. F1: ", np.around(np.mean(pc_f1, axis = 0), 3))
class_names = ["No Finding", "Enlarged Cardiomediastinum", "Cardiomegaly", "Lung Opacity", "Lung Lesion", "Edema", "Consolidation", "Pneumonia", "Atelectasis", "Pneumothorax", "Pleural Effusion", "Pleural Other", "Fracture", "Support Devices"]
plot_confusion_matrix(test_results, class_names)
criterion,
optimizer,
epochs=p['epochs'],
lr_min=p['lr_min'],
lr_max=p['lr_max'],
mom_min=p['mom_min'],
mom_max=p['mom_max'],
annealing_pct=p['annealing_pct'],
verbose=2)
# Plotting the learning curves
description = 'lrmin{}_lrmax{}_mmin{}_mmax{}_pct{}_e{}'.format(
p['lr_min'], p['lr_max'], p['mom_min'], p['mom_max'],
p['annealing_pct'], p['epochs'])
plot_learning_curves(curves[0],
curves[1],
curves[2],
curves[3],
description=description)
plot_learning_rates(curves[4], description=description)
plot_momentums(curves[5], description=description)
print(description)
print('train loss: {}, val loss: {}, train acc: {}, val_acc:{}'.format(
curves[0][-1], curves[1][-1], curves[2][-1], curves[3][-1]))
# Save all curves to disk for later plotting
for i in range(6):
np.save('../learning_curves/' + description + '-' + str(i),
np.array(curves[i]))
optimizer, epoch)
valid_loss, valid_accuracy, valid_results = evaluate(
model, device, valid_loader, criterion)
train_losses.append(train_loss)
valid_losses.append(valid_loss)
train_accuracies.append(train_accuracy)
valid_accuracies.append(valid_accuracy)
is_best = valid_accuracy > best_val_acc # let's keep the model that has the best accuracy, but you can also use another metric.
if is_best:
best_val_acc = valid_accuracy
torch.save(model, os.path.join(PATH_OUTPUT, "MyVariableRNN.pth"))
plot_learning_curves(train_losses, valid_losses, train_accuracies,
valid_accuracies, PATH_OUTPUT)
best_model = torch.load(os.path.join(PATH_OUTPUT, "MyVariableRNN.pth"))
# TODO: For your report, try to make plots similar to those in the previous task.
# TODO: You may use the validation set in case you cannot use the test set.
# TODO: Complete predict_mortality
def predict_mortality(model, device, data_loader):
model.eval()
# TODO: Evaluate the data (from data_loader) using model,
# TODO: return a List of probabilities
probas = []
with torch.no_grad():
for i, (input, target) in enumerate(data_loader):
import argparse
from analog.load import ExperimentLog
from dataloader import loader
from plots import plot_learning_curves
parser = argparse.ArgumentParser()
parser.add_argument('--logdir', type=str, required=True)
parser.add_argument('--exp_names', nargs='+', type=str, required=True)
parser.add_argument('--std_type', type=str, choices=['time', 'run'])
parser.add_argument('--min_t', type=float, default=0.)
parser.add_argument('--max_t', type=float, default=100.)
args = parser.parse_args()
start_date = 'last'
stop_date = None
expdata: ExperimentLog = ExperimentLog()
for exp_name in args.exp_names:
expdata.extend(loader(args.logdir, exp_name, start_date=start_date, stop_date=stop_date))
# def filter(args) -> bool:
# return ('noscale' in args and args['noscale'] and 'value' in args['algo']) \
# or 'advantage' in args['algo']
# expdata = filter(expdata, filter)
plot_learning_curves(expdata, ['Return'], args.exp_names[0], mint=args.min_t, maxt=args.max_t, gtype=args.std_type + "_std")
expdata.repr_rawlogs("Return", 5)
