def run(args):
tag = 'bamos.smooth-topk.seed={}.{}'.format(args.seed, args.dataset)
if args.dataset == 'cifar100':
tag += '.noise={}'.format(args.noise_labels)
elif args.dataset == 'imagenet':
tag += '-{}'.format(args.train_size)
setproctitle(tag)
set_seed(args.seed)
xp = create_experiment(args)
train_loader, val_loader, test_loader = get_loaders(args)
loss = get_loss(xp, args)
model = get_model(args)
if args.load_model:
load_model(model, args.load_model)
if args.cuda:
if args.parallel_gpu:
model = torch.nn.DataParallel(model).cuda()
else:
torch.cuda.set_device(args.device)
model.cuda()
loss.cuda()
optimizer = get_optimizer(model, args.mu, args.lr_0, xp)
if args.load_optimizer:
load_optimizer(optimizer, args.load_optimizer, args.lr_0)
with logger.stdout_to("{}_log.txt".format(args.out_name)):
clock = -time.time()
for _ in range(args.epochs):
xp.Epoch.update(1).log()
optimizer = update_optimizer(args.lr_schedule, optimizer,
model, loss, xp)
xp.Learning_Rate.update().log()
xp.Mu.update().log()
xp.Temperature.update().log()
train(model, loss, optimizer, train_loader, xp, args)
test(model, loss, val_loader, xp, args)
test(model, loss, test_loader, xp, args)
clock += time.time()
print("\nEvaluation time: \t {0:.2g} min".format(clock * 1. / 60))
def run(args):
set_seed(args.seed)
xp = create_experiment(args)
train_loader, val_loader, test_loader = get_loaders(args)
loss = get_loss(xp, args)
model = get_model(args)
if args.load_model:
load_model(model, args.load_model)
if args.cuda:
if args.parallel_gpu:
model = torch.nn.DataParallel(model).cuda()
else:
torch.cuda.set_device(args.device)
model.cuda()
loss.cuda()
optimizer = get_optimizer(model, args.mu, args.lr_0, xp)
if args.load_optimizer:
load_optimizer(optimizer, args.load_optimizer, args.lr_0)
with logger.stdout_to("{}_log.txt".format(args.out_name)):
clock = -time.time()
for _ in range(args.epochs):
xp.Epoch.update(1).log()
optimizer = update_optimizer(args.lr_schedule, optimizer, model,
loss, xp)
xp.Learning_Rate.update().log()
xp.Mu.update().log()
xp.Temperature.update().log()
train(model, loss, optimizer, train_loader, xp, args)
test(model, loss, val_loader, xp, args)
test(model, loss, test_loader, xp, args)
clock += time.time()
print("\nEvaluation time: \t {0:.2g} min".format(clock * 1. / 60))
def main():
plt.ioff()
model = torch.nn.DataParallel(resnet50(pretrained=False)).cuda()
if args.checkpoint is not None:
model = copy_pretrained_model(model, args.checkpoint)
if args.center_only:
model = imagenet_wrapper(
model, std=[1.0, 1.0,
1.0]) #to do the normalization inside the forward pass
else:
model = imagenet_wrapper(model)
radius_train = torch.zeros(trainset_size)
radius_test = torch.zeros(testset_size)
sigma_train = torch.ones(trainset_size) * args.sig0_tr
sigma_test = torch.ones(testset_size) * args.sig0_ts
radius_train, radius_test = radius_train.to(device), radius_test.to(device)
sigma_train, sigma_test = sigma_train.to(device), sigma_test.to(device)
if args.path_sigma is not None: #loading test sigma to optimize for it
sigma_test = torch.load(args.path_sigma)
base_bath_save = f'scratch_exps/ImageNet/exps/{args.output_path}/sig0_tr_{args.sig0_tr}/sig0_ts_{args.sig0_tr}/lr_sig_{args.lr_sig}/iter_sig_tr_{args.iter_sig_tr}/iter_sig_ts_{args.iter_sig_ts}'
tensorboard_path = f'scratch_exps/ImageNet/tensorboard/{args.output_path}/sig0_tr_{args.sig0_tr}/sig0_ts_{args.sig0_tr}/lr_sig_{args.lr_sig}/iter_sig_tr_{args.iter_sig_tr}/iter_sig_ts_{args.iter_sig_ts}'
if not path.exists(base_bath_save):
os.makedirs(base_bath_save)
writer = SummaryWriter(tensorboard_path, flush_secs=10)
optimizer = torch.optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=args.step_sz,
gamma=args.gamma)
attacker = PGD_L2(steps=args.num_steps,
device='cuda',
max_norm=args.epsilon)
if args.checkpoint is not None:
optimizer = load_optimizer(optimizer, args.checkpoint)
best_acc = 0.0
for epoch in range(args.epochs):
#the +90 down there because both RS and ARS trained for 90 epochs already and this is the model that I am starting with
attacker.max_norm = np.min(
[args.epsilon, (epoch + 90 + 1) * args.epsilon / args.warmup])
attacker.init_norm = np.min(
[args.epsilon, (epoch + 90 + 1) * args.epsilon / args.warmup])
writer.add_scalar('learning rate', optimizer.param_groups[0]['lr'],
epoch)
if epoch >= args.epoch_switch:
print(
'Switched to DS training, sigma train is {} and sigma test is {}'
.format(sigma_train.mean().item(),
sigma_test.mean().item()))
sigma_train = train(epoch, model, train_loader, optimizer, writer,
sigma_train, args.lr_sig, args.iter_sig_tr,
attacker, args.num_noise_vec)
test_acc, sigma_test = test(epoch, model, test_loader, writer,
sigma_test, args.lr_sig,
args.iter_sig_ts)
else:
print('Training with RS')
sigma_train = train(epoch, model, train_loader, optimizer, writer,
sigma_train, args.lr_sig, 0, attacker,
args.num_noise_vec)
test_acc, sigma_test = test(epoch, model, test_loader, writer,
sigma_test, args.lr_sig, 0)
scheduler.step()
print('Learing Rate: {}.'.format(optimizer.param_groups[0]['lr']))
if test_acc > best_acc:
best_flag = True
best_acc = test_acc
else:
best_flag = False
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'optimizer_param': optimizer.state_dict(),
},
base_bath_save,
best=best_flag)
#Optimizing sigma afterwards
print(
'Training has finished now we optimize for the sigmas. So far, sigma train is {} and sigma test is {}'
.format(sigma_train.mean().item(),
sigma_test.mean().item()))
for i in range(1, args.iter_sig_after + 1):
print('Optimizing sigmas for the iteration {} out of {}'.format(
i, args.iter_sig_after))
sigma_test = optimize_sigma(model,
test_loader,
writer,
sigma_test,
args.lr_sig,
flag='test',
radius=radius_test,
gaussian_num_ds=args.gaussian_num_ds,
epoch=i)
# Saving the sigmas
torch.save(sigma_test,
base_bath_save + '/sigma_test_' + str(i) + '.pth')
print('everything is done, you should be happy now !')
val_dataloader = DataLoader(val_dataset,
batch_size=batch_size,
shuffle=True,
drop_last=True)
train_losses = []
val_losses = []
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
factor=0.1,
patience=5,
verbose=False,
min_lr=LR / 100)
if LOAD_THEN_TRAIN:
load_optimizer(model_dir, optimizer)
load_models(model_dir, encoder_rnn)
for EPOCH in range(EPOCHS):
for i, (history_Us, future_Us) in enumerate(train_dataloader):
optimizer.zero_grad()
# encode history into 3D initial condition vector (out)
history_Xs = history_Us[:, :, 0].view(-1, k, 1)
inferred_U = predict_state(history_Xs)
loss = torch.mean(torch.abs(inferred_U - future_Us)**2)
loss.backward()
checkpoint_model = f'../model/fasterrcnn_resnet50_fpn_bb5_p3.pt'
new_model = f'../model/fasterrcnn_resnet50_fpn_bb5_p4.pt'
utils.load_model(model, checkpoint_model)
# Loading optimizer
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(
params,
lr=0.001, # 0.005
momentum=0.9,
weight_decay=0.0005)
# and a learning rate scheduler
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=3,
gamma=0.1)
utils.load_optimizer(optimizer, new_model)
# Loading last losses and epoch
last_epoch, last_train_loss, last_val_loss = utils.load_loss(new_model)
# Loading dataset
dataset = ColruytDataset(
json_file=f'../data/train/train_info.json',
img_dir=f'../data/train/images',
# img_size=(180, 240),
transforms=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]))
# Split dataset into training and validation sets