def benchmark_dataset(dataset, title, fname, testname, xlabels, ylabels=None):
global benchmark_data
print(f'Benchmarking {title}.. ')
dataloader = torch.utils.data.DataLoader(
dataset, batch_size=10,
shuffle=False, num_workers=4,
)
tracked_metrics = [
metrics.Accuracy(),
metrics.RocAuc(),
metrics.FScore()
]
logs, cm = trainer.test(
model=model, test_dataloader=dataloader,
criterion=criterion, metrics=tracked_metrics, device=device
)
with open(f'logs/{vars.corda_version}/{name}/{fname}-metric.txt', 'w') as f:
f.write(f'{fname}: ' + trainer.summarize_metrics(logs) + '\n')
ax = sns.heatmap(
cm.get(normalized=True), annot=True, fmt=".2f",
xticklabels=xlabels, yticklabels=ylabels or xlabels,
vmin=0., vmax=1.
)
ax.set_title(title)
plt.xlabel('predicted')
plt.ylabel('ground')
hm = ax.get_figure()
hm.savefig(f'logs/{vars.corda_version}/{name}/{fname}.png')
hm.clf()
fpr, tpr, thresholds = tracked_metrics[1].get_curve()
auc = tracked_metrics[1].get()
f = plt.figure()
plt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC curve (auc = {auc:.2f})')
plt.title(f'{title} ROC')
plt.legend(loc='lower right')
plt.savefig(f'logs/{vars.corda_version}/{name}/{fname}-roc.png')
plt.clf()
plt.cla()
plt.close()
specificity, fpr, fnr, sensitivity = cm.get(normalized=True).ravel()
dor = (sensitivity*specificity)/((1-sensitivity)*(1-specificity))
fscore = tracked_metrics[2].get()
ba = (sensitivity+specificity)/2.
data = {
'arch': args.arch, 'pretrain': args.pretrain, 'train': args.train.upper(),
'test': testname, 'accuracy': tracked_metrics[0].get(), 'auc': auc,
'sensitivity': sensitivity, 'specificity': specificity, 'fscore': fscore,
'ba': ba, 'missrate': fnr, 'dor': dor
}
for k,v in data.items():
benchmark_data[k].append(v)
def benchmark_dataset(dataset, title, fname, xlabels, ylabels=None):
print(f'Benchmarking {title}.. ', end='', flush=True)
dataloader = torch.utils.data.DataLoader(
dataset,
batch_size=10,
shuffle=False,
num_workers=4,
)
tracked_metrics = [metrics.Accuracy(), metrics.RocAuc(), metrics.FScore()]
logs, cm = trainer.test(model=model,
test_dataloader=dataloader,
criterion=criterion,
metrics=tracked_metrics,
device=device)
with open(f'logs/{vars.corda_version}/{name}/{fname}-metric.txt',
'w') as f:
f.write(f'{fname}: ' + trainer.summarize_metrics(logs) + '\n')
ax = sns.heatmap(cm.get(normalized=True),
annot=True,
fmt=".2f",
xticklabels=xlabels,
yticklabels=ylabels or xlabels)
ax.set_title(title)
plt.xlabel('predicted')
plt.ylabel('ground')
hm = ax.get_figure()
hm.savefig(f'logs/{vars.corda_version}/{name}/{fname}.png')
hm.clf()
fpr, tpr, thresholds = tracked_metrics[1].get_curve()
auc = tracked_metrics[1].get()
f = plt.figure()
plt.plot(fpr,
tpr,
color='darkorange',
lw=2,
label=f'ROC curve (auc = {auc:.2f})')
plt.title(f'{title} ROC')
plt.legend(loc='lower right')
plt.savefig(f'logs/{vars.corda_version}/{name}/{fname}-roc.png')
plt.clf()
plt.cla()
plt.close()
elif args.arch == 'resnet50':
model = covid_classifier.CovidClassifier50(
encoder=feature_extractor,
pretrained=False,
freeze_conv=False
).to(device)
#model = covid_classifier.LeNet1024NoPoolingDeep().to(device)
print(f'Using lr {lr}')
# TRAINING
# %%
tracked_metrics = [
metrics.Accuracy(),
metrics.RocAuc(),
metrics.FScore()
]
def focal_loss(output, target, gamma=2., weight=None):
bce = F.binary_cross_entropy(output, target, reduction='none', weight=weight)
pt = target*output + (1-target)*(1-output)
return (torch.pow((1-pt), gamma) * bce).mean()
criterion = focal_loss
optimizer = torch.optim.SGD(model.parameters(), lr=lr, weight_decay=1e-3)
lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=15, verbose=True)
best_model = trainer.fit(
model=model, train_dataloader=train_dataloader,
val_dataloader=val_dataloader, test_dataloader=test_dataloader,
if args.arch == 'resnet18':
model = covid_classifier.CovidClassifier(encoder=feature_extractor,
pretrained=False,
freeze_conv=False).to(device)
elif args.arch == 'resnet50':
model = covid_classifier.CovidClassifier50(encoder=feature_extractor,
pretrained=False,
freeze_conv=False).to(device)
#model = covid_classifier.LeNet1024NoPoolingDeep().to(device)
print(f'Using lr {lr}')
# TRAINING
# %%
tracked_metrics = [metrics.Accuracy(), metrics.RocAuc(), metrics.FScore()]
def focal_loss(output, target, gamma=2., weight=None):
bce = F.binary_cross_entropy(output,
target,
reduction='none',
weight=weight)
pt = target * output + (1 - target) * (1 - output)
return (torch.pow((1 - pt), gamma) * bce).mean()
criterion = focal_loss
optimizer = torch.optim.SGD(model.parameters(), lr=lr, weight_decay=1e-3)
lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
patience=15,