def test_cls_tta(model, test_loader):
probs_tta = []
prs = [0, 1]
for p1 in prs:
for p2 in prs:
test_loader.dataset.transforms.transforms[
1].p = p1 # pr(horizontal flip)
test_loader.dataset.transforms.transforms[
2].p = p2 # pr(vertical flip)
# validate one epoch, note no optimizer is passed
with torch.no_grad():
test_preds, test_probs, test_labels = run_one_epoch_cls(
test_loader, model)
probs_tta.append(test_probs)
probs_tta = np.mean(np.array(probs_tta), axis=0)
preds_tta = np.argmax(probs_tta, axis=1)
test_k, test_auc, test_acc = eval_predictions_multi(
test_labels, preds_tta, probs_tta)
print('Test Kappa: {:.4f} -- AUC: {:.4f} -- Balanced Acc: {:.4f}'.format(
test_k, test_auc, test_acc))
del model
torch.cuda.empty_cache()
return probs_tta, preds_tta, test_labels
def test_cls_tta_dihedral(model, test_loader, n=3):
probs_tta = []
prs = [0, 1]
import torchvision
test_loader.dataset.transforms.transforms.insert(-1, torchvision.transforms.RandomRotation(0))
rotations = np.array([i * 360 // n for i in range(n)])
for angle in rotations:
for p2 in prs:
test_loader.dataset.transforms.transforms[2].p = p2 # pr(vertical flip)
test_loader.dataset.transforms.transforms[-2].degrees = [angle, angle]
# validate one epoch, note no optimizer is passed
with torch.no_grad():
test_preds, test_probs, test_labels = run_one_epoch_cls(test_loader, model)
probs_tta.append(test_probs)
probs_tta = np.mean(np.array(probs_tta), axis=0)
preds_tta = np.argmax(probs_tta, axis=1)
try:
test_k, test_auc, test_acc = eval_predictions_multi(test_labels, preds_tta, probs_tta)
print('Test Kappa: {:.4f} -- AUC: {:.4f} -- Balanced Acc: {:.4f}'.format(test_k, test_auc, test_acc))
except:
print('Test Kappa: {:.4f} -- AUC: {:.4f} -- Balanced Acc: {:.4f}'.format(0, 0, 0))
del model
torch.cuda.empty_cache()
return probs_tta, preds_tta, test_labels
def test_cls(model, test_loader):
# validate one epoch, note no optimizer is passed
with torch.no_grad():
preds, probs, labels = run_one_epoch_cls(test_loader, model)
vl_k, vl_auc, vl_acc = eval_predictions_multi(labels, preds, probs)
print('Val. Kappa: {:.4f} -- AUC: {:.4f}'.format(vl_k, vl_auc).rstrip('0'))
del model
torch.cuda.empty_cache()
return probs, preds, labels
def train_cls(model, optimizer, train_criterion, val_criterion, train_loader,
val_loader, oversample, n_epochs, metric, patience, decay_f,
exp_path):
counter_since_checkpoint = 0
tr_losses, tr_aucs, tr_ks, vl_losses, vl_aucs, vl_ks = [], [], [], [], [], []
stats = {}
is_better, best_monitoring_metric = compare_op(metric)
best_kappa, best_auc = 0, 0
for epoch in range(n_epochs):
print('\n EPOCH: {:d}/{:d}'.format(epoch + 1, n_epochs))
if oversample == [1, 1, 1]:
tr_preds, tr_probs, tr_labels, tr_loss = run_one_epoch_cls(
train_loader, model, train_criterion, optimizer)
else:
csv_train_path = train_loader.dataset.csv_path
train_loader_MOD = modify_dataset(train_loader,
csv_train_path=csv_train_path,
keep_samples=oversample)
# train one epoch
tr_preds, tr_probs, tr_labels, tr_loss = run_one_epoch_cls(
train_loader_MOD, model, train_criterion, optimizer)
# validate one epoch, note no optimizer is passed
with torch.no_grad():
vl_preds, vl_probs, vl_labels, vl_loss = run_one_epoch_cls(
val_loader, model, val_criterion)
tr_k, tr_auc, tr_acc = eval_predictions_multi(tr_labels, tr_preds,
tr_probs)
print('\n')
vl_k, vl_auc, vl_acc = eval_predictions_multi(vl_labels, vl_preds,
vl_probs)
print(
'Train/Val. Loss: {:.4f}/{:.4f} -- Kappa: {:.4f}/{:.4f} -- AUC: {:.4f}/{:.4f} -- LR={:.6f}'
.format(tr_loss, vl_loss, tr_k, vl_k, tr_auc, vl_auc,
get_lr(optimizer)).rstrip('0'))
# store performance for this epoch
tr_losses.append(tr_loss)
tr_aucs.append(tr_auc)
tr_ks.append(tr_k)
vl_losses.append(vl_loss)
vl_aucs.append(vl_auc)
vl_ks.append(vl_k)
# smooth val values with a moving average before comparing
vl_auc = ewma(vl_aucs, window=3)[-1]
vl_loss = ewma(vl_losses, window=3)[-1]
vl_k = ewma(vl_ks, window=3)[-1]
# check if performance was better than anyone before and checkpoint if so
if metric == 'auc': monitoring_metric = vl_auc
elif metric == 'loss': monitoring_metric = vl_loss
elif metric == 'kappa': monitoring_metric = vl_k
elif metric == 'kappa_auc_avg':
monitoring_metric = 0.5 * (vl_k + vl_auc)
else:
sys.exit('Not a suitable metric for this task')
if is_better(monitoring_metric, best_monitoring_metric):
print('Best (smoothed) val {} attained. {:.4f} --> {:.4f}'.format(
metric, best_monitoring_metric, monitoring_metric))
best_auc, best_kappa = vl_auc, vl_k
if exp_path != None:
print(15 * '-', ' Checkpointing ', 15 * '-')
write_model(exp_path, model, optimizer, stats)
best_monitoring_metric = monitoring_metric
stats['tr_losses'], stats['vl_losses'] = tr_losses, vl_losses
stats['tr_aucs'], stats['vl_aucs'] = tr_aucs, vl_aucs
stats['tr_ks'], stats['vl_ks'] = tr_aucs, vl_aucs
counter_since_checkpoint = 0 # reset patience
else:
counter_since_checkpoint += 1
if decay_f != 0 and counter_since_checkpoint == 3 * patience // 4:
reduce_lr(optimizer, epoch, factor=decay_f, verbose=False)
print(8 * '-', ' Reducing LR now ', 8 * '-')
# early stopping if no improvement happened for `patience` epochs
if counter_since_checkpoint == patience:
print('\n Early stopping the training, trained for {:d} epochs'.
format(epoch))
del model
torch.cuda.empty_cache()
return best_kappa, best_auc
del model
torch.cuda.empty_cache()
return best_kappa, best_auc
