def benchmark_dataset(dataset, title, fname, testname, xlabels, ylabels=None):
global benchmark_data
print(f'Benchmarking {title}.. ')
dataloader = torch.utils.data.DataLoader(
dataset, batch_size=10,
shuffle=False, num_workers=4,
)
tracked_metrics = [
metrics.Accuracy(),
metrics.RocAuc(),
metrics.FScore()
]
logs, cm = trainer.test(
model=model, test_dataloader=dataloader,
criterion=criterion, metrics=tracked_metrics, device=device
)
with open(f'logs/{vars.corda_version}/{name}/{fname}-metric.txt', 'w') as f:
f.write(f'{fname}: ' + trainer.summarize_metrics(logs) + '\n')
ax = sns.heatmap(
cm.get(normalized=True), annot=True, fmt=".2f",
xticklabels=xlabels, yticklabels=ylabels or xlabels,
vmin=0., vmax=1.
)
ax.set_title(title)
plt.xlabel('predicted')
plt.ylabel('ground')
hm = ax.get_figure()
hm.savefig(f'logs/{vars.corda_version}/{name}/{fname}.png')
hm.clf()
fpr, tpr, thresholds = tracked_metrics[1].get_curve()
auc = tracked_metrics[1].get()
f = plt.figure()
plt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC curve (auc = {auc:.2f})')
plt.title(f'{title} ROC')
plt.legend(loc='lower right')
plt.savefig(f'logs/{vars.corda_version}/{name}/{fname}-roc.png')
plt.clf()
plt.cla()
plt.close()
specificity, fpr, fnr, sensitivity = cm.get(normalized=True).ravel()
dor = (sensitivity*specificity)/((1-sensitivity)*(1-specificity))
fscore = tracked_metrics[2].get()
ba = (sensitivity+specificity)/2.
data = {
'arch': args.arch, 'pretrain': args.pretrain, 'train': args.train.upper(),
'test': testname, 'accuracy': tracked_metrics[0].get(), 'auc': auc,
'sensitivity': sensitivity, 'specificity': specificity, 'fscore': fscore,
'ba': ba, 'missrate': fnr, 'dor': dor
}
for k,v in data.items():
benchmark_data[k].append(v)
def benchmark_dataset(dataset, title, fname, xlabels, ylabels=None):
print(f'Benchmarking {title}.. ', end='', flush=True)
dataloader = torch.utils.data.DataLoader(
dataset,
batch_size=10,
shuffle=False,
num_workers=4,
)
tracked_metrics = [metrics.Accuracy(), metrics.RocAuc(), metrics.FScore()]
logs, cm = trainer.test(model=model,
test_dataloader=dataloader,
criterion=criterion,
metrics=tracked_metrics,
device=device)
with open(f'logs/{vars.corda_version}/{name}/{fname}-metric.txt',
'w') as f:
f.write(f'{fname}: ' + trainer.summarize_metrics(logs) + '\n')
ax = sns.heatmap(cm.get(normalized=True),
annot=True,
fmt=".2f",
xticklabels=xlabels,
yticklabels=ylabels or xlabels)
ax.set_title(title)
plt.xlabel('predicted')
plt.ylabel('ground')
hm = ax.get_figure()
hm.savefig(f'logs/{vars.corda_version}/{name}/{fname}.png')
hm.clf()
fpr, tpr, thresholds = tracked_metrics[1].get_curve()
auc = tracked_metrics[1].get()
f = plt.figure()
plt.plot(fpr,
tpr,
color='darkorange',
lw=2,
label=f'ROC curve (auc = {auc:.2f})')
plt.title(f'{title} ROC')
plt.legend(loc='lower right')
plt.savefig(f'logs/{vars.corda_version}/{name}/{fname}-roc.png')
plt.clf()
plt.cla()
plt.close()
optimizer = torch.optim.SGD(model.parameters(), lr=lr, weight_decay=1e-3)
lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=15, verbose=True)
best_model = trainer.fit(
model=model, train_dataloader=train_dataloader,
val_dataloader=val_dataloader, test_dataloader=test_dataloader,
test_every=10, criterion=criterion,
optimizer=optimizer, scheduler=lr_scheduler,
metrics=tracked_metrics, n_epochs=args.epochs, name=name,
metric_choice=args.metric, mode=args.mode, device=device,
checkpoint_params={'corda-version': vars.corda_version}
)
print(f'Best model: ')
test_logs, test_cm = trainer.test(
model=best_model, test_dataloader=test_dataloader, weight=None,
criterion=criterion, metrics=tracked_metrics, device=device
)
ax = sns.heatmap(test_cm.get(normalized=True), annot=True, fmt=".2f")
ax.set_title(f'{args.train} Best {preprocessed}')
plt.xlabel('predicted')
plt.ylabel('ground')
hm = ax.get_figure()
hm.savefig(f'logs/{vars.corda_version}/{name}/best.png')
hm.clf()
# %%
print(f'Final model:')
test_logs, test_cm = trainer.test(
model=model, test_dataloader=test_dataloader, weight=None,
criterion=criterion, metrics=tracked_metrics, device=device
)
def main():
# Setup.
parser = argparse.ArgumentParser()
parser.add_argument("--config", default="./config/example.yaml")
parser.add_argument("--gpu", default="0", type=str)
# Path to checkpoint (empty string means the latest checkpoint)
# or False (means training from scratch).
parser.add_argument("--resume", default="", type=str)
args = parser.parse_args()
config, inner_dir, config_name = load_config(args.config)
saved_dir = get_saved_dir(config, inner_dir, config_name, args.resume)
storage_dir, ckpt_dir = get_storage_dir(config, inner_dir, config_name,
args.resume)
logger = get_logger(saved_dir, "adv_training.log", args.resume)
# Prepare data.
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
test_transform = transforms.Compose([transforms.ToTensor()])
train_data = cifar.CIFAR10(root=config["dataset_dir"],
transform=train_transform)
test_data = cifar.CIFAR10(root=config["dataset_dir"],
train=False,
transform=test_transform)
train_loader = DataLoader(train_data,
batch_size=config["batch_size"],
shuffle=True,
num_workers=4)
test_loader = DataLoader(test_data,
batch_size=config["batch_size"],
num_workers=4)
# Resume training state.
model = resnet_cifar.ResNet18()
gpu = int(args.gpu)
logger.info("Set GPU to {}".format(args.gpu))
model = model.cuda(gpu)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(),
**config["optimizer"]["SGD"])
scheduler = lr_scheduler.MultiStepLR(
optimizer, **config["lr_scheduler"]["MultiStepLR"])
resumed_epoch = resume_state(model, optimizer, args.resume, ckpt_dir,
scheduler)
# Set attack first and then add a normalized layer.
pgd_config = {}
for k, v in config["pgd_attack"].items():
if k == "eps" or k == "alpha":
pgd_config[k] = eval(v)
else:
pgd_config[k] = v
attacker = PGD(model, **pgd_config)
normalize_net = NormalizeByChannelMeanStd((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010))
normalize_net.cuda(gpu)
model = nn.Sequential(normalize_net, model)
for epoch in range(config["num_epochs"] - resumed_epoch):
logger.info("===Epoch: {}/{}===".format(epoch + resumed_epoch + 1,
config["num_epochs"]))
logger.info("Adversarial training...")
adv_train_result = train(model,
train_loader,
criterion,
optimizer,
logger,
attacker=attacker)
if scheduler is not None:
scheduler.step()
logger.info("Adjust learning rate to {}".format(
optimizer.param_groups[0]["lr"]))
logger.info("Test model on clean data...")
clean_test_result = test(model, test_loader, criterion, logger)
logger.info("Test model on adversarial data...")
adv_test_result = test(model,
test_loader,
criterion,
logger,
attacker=attacker)
result = {
"adv_train": adv_train_result,
"clean_test": clean_test_result,
"adv_test": adv_test_result,
}
# Save checkpoint
saved_dict = {
"epoch": epoch + resumed_epoch + 1,
"result": result,
"model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
}
if scheduler is not None:
saved_dict["scheduler_state_dict"] = scheduler.state_dict()
torch.save(
saved_dict,
os.path.join(ckpt_dir,
"epoch{}.pt".format(epoch + resumed_epoch + 1)),
)
train_dataloader=train_dataloader,
val_dataloader=val_dataloader,
test_dataloader=test_dataloader,
test_every=10,
criterion=criterion,
optimizer=optimizer,
scheduler=lr_scheduler,
metrics=tracked_metrics,
n_epochs=epochs,
name=name,
metric_choice=metric,
mode=mode,
device=device,
multiclass=True)
# %%
print('best model: ')
test_logs, test_cm = trainer.test(model=best_model,
test_dataloader=test_dataloader,
criterion=criterion,
metrics=tracked_metrics,
device=device,
multiclass=True)
ax = sns.heatmap(test_cm.get(normalized=True), annot=True, fmt=".2f")
hm = ax.get_figure()
ax.set_title('Chest X Ray (unprocessed)')
plt.xlabel('predicted')
plt.ylabel('ground')
hm.savefig(f'logs/{vars.corda_version}/{name}/best.png')
hm.clf()
s=30,
weight=per_cls_weights).cuda()
elif args.loss.lower() == 'focal':
loss_cls = FocalLoss(weight=per_cls_weights, gamma=1).cuda()
# TODO: Modifying the train and evaluation functions
# Training
model = train(epoch, model, optim_model, loss_cls, loss_reg, train_loader,
gpu, args)
if args.LR_schedule == True:
scheduler.step()
# Save the stage0 model
if epoch % args.print_epoch == 0:
result = test(epoch, model, loss_cls, test_loader, gpu, args)
if args.earlystop == True:
early(result['loss'], model, result)
if early.early_stop == True:
break
if args.print_test == True:
print('Epoch : %d, Test Acc : %2.2f, Test Loss : %.2f' %
(epoch, result['acc'], result['loss']))
print(result['confusion'])
print(result['f1'])
# Logging
model = early.model
result = early.result
print('train both classifier and discriminator')
print('##########################################')
os.makedirs(output_dir, exist_ok=True)
best_acc = 0.
for epoch in range(num_epochs + 1):
print('Epoch {}/{}'.format(epoch + 1, num_epochs))
print('-------------')
if epoch == 0:
train_loss, train_cls_loss, train_dom_loss, train_cls_acc, train_dom_acc = 0, 0, 0, 0, 0
val_loss, val_cls_loss, val_dom_loss, val_cls_acc, val_dom_acc = val(
epoch, num_epochs, num_steps_val, net, loader_abc_test,
cls_criterion, dom_criterion, device, hp_lambda, gamma)
test_d_loss, test_d_acc = test(net, loader_d_test, cls_criterion,
device)
else:
for phase in ['train', 'val', 'test']:
if phase == 'train':
train_loss, train_cls_loss, train_dom_loss, train_cls_acc, train_dom_acc = train(
epoch, num_epochs, num_steps, net, loader_abc_train,
cls_criterion, dom_criterion, device, optimizer2,
hp_lambda, gamma)
elif phase == 'val':
val_loss, val_cls_loss, val_dom_loss, val_cls_acc, val_dom_acc = val(
epoch, num_epochs, num_steps_val, net, loader_abc_test,
cls_criterion, dom_criterion, device, hp_lambda, gamma)
else:
test_d_loss, test_d_acc = test(net, loader_d_test,
cls_criterion, device)
