# writer
writer = SummaryWriter(args.logdir)
# get number of total param
n_param = 0
for name, param in model.named_parameters():
n_param += len(param.view(-1))
logger.save(str(n_param), 'n_param')
# optimization
torch.backends.cudnn.benchmark = True
for i in range(start_iter, args.maxiter):
# eval noise
if i % args.update_noise == 0:
eval_list = dataset.getTrainBatchList(args.batch_size, 100, True)
std_dict, diag2 = evalCovDiag(eval_list, model, criterion, optimizer)
# train
model.train()
optimizer.zero_grad()
loss_train, acc_train = 0, 0
for x,y in train_list:
out = model(x)
loss = criterion(out, y)
loss.backward()
acc_train += accuracy(out, y).item()
loss_train += loss.detach().item()
for name, param in model.named_parameters():
param.grad.data /= len(train_list)
for name, param in model.named_parameters():
n_param += len(param.view(-1))
logger.save(str(n_param), 'n_param')
torch.backends.cudnn.benchmark = True
for i in range(start_iter, args.iters):
# decay lr
if i in args.schedule:
lr *= 0.1
logger.save('update lr: %f'%(lr))
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# eval noise
if i % args.update_noise == 0:
eval_list = dataset.getTrainBatchList(args.ghostsize, args.numghost, True)
diagF_dict, diag2 = evalFisherDiag(eval_list, model, criterion, optimizer)
resume_noise = True if args.resume is not None else False
if resume_noise:
eval_list = dataset.getTrainBatchList(args.ghostsize, args.numghost, True)
diagF_dict, diag2 = evalFisherDiag(eval_list, model, criterion, optimizer)
resume_noise = False
# train
model.train()
optimizer.zero_grad()
train_loss, train_acc = 0, 0
for x,y in train_list:
out = model(x)
loss = criterion(out, y)