# def attack(x,y):
# eps = 0.3
# delta = torch.zeros_like(x, requires_grad=True)
# L = loss_fn(model(x+delta),y)
# dx = torch.autograd.grad(L, delta, create_graph=False, retain_graph=False, only_inputs=True)[0]
# return (x+eps*dx.sign()).clamp(0,1).detach()
if args.mode=="train":
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.alpha['wdecay'])
scheduler = utils.optim.EvenReductionLR(optimizer, lr_reduction=0.05, gamma=0.5, epochs=args.epochs, last_epoch=-1)
# scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=15, threshold=1.e-4, threshold_mode='rel', cooldown=5, min_lr=0, eps=1e-08, verbose=False)
checkpoints = None
# checkpoints = {'dir': paths['checkpoints'], 'each_nth': 5, 'name': file_name}
train_model = utils.TrainingLoop(model, loss_fn, dataset, args.batch, optimizer, data_augmentation=None, val_dataset=val_dataset, val_batch_size=args.batch, scheduler=scheduler, accuracy_fn=accuracy_fn, val_freq=1, stat_freq=1, checkpoints=checkpoints, tol=1.e-4)
# with profiler.profile(record_shapes=True,use_cuda=True) as prof:
writer = SummaryWriter(Path("logs",file_name))
# save initial model and train
torch.save( model.state_dict(), Path(paths['chkp_init'],file_name) )
try:
alpha = args.alpha.copy()
epoch = 0
i = 0
while epoch < args.epochs:
if i%2==0:
epoch = epoch + 30
i += 1
converged = train_model(30, writer=writer)
if args.mode == "train":
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.alpha['wdecay'])
# scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.7, patience=10, verbose=True, threshold=1.e-4, threshold_mode='rel', cooldown=10, min_lr=1.e-6, eps=1.e-8)
scheduler = utils.optim.EvenReductionLR(optimizer,
lr_reduction=0.2,
gamma=0.8,
epochs=args.epochs,
last_epoch=-1)
train_model = utils.TrainingLoop(model,
loss_fn,
dataset,
args.batch,
optimizer,
val_dataset=val_dataset,
scheduler=scheduler,
val_freq=1,
stat_freq=1)
writer = SummaryWriter(Path("logs", file_name))
# save initial model and train
torch.save(model.state_dict(), Path(paths['chkp_init'], file_name))
try:
train_model(args.epochs, writer=writer)
except:
raise
finally:
torch.save(model.state_dict(), Path(paths['chkp_final'],