def check_hyper(args):
device, dtype = args.device, args.dtype
add_args = {'num_classes': args.num_classes}
if args.cpni:
add_args = {
'weight_noise': args.weight_noise,
'act_noise_a': args.act_noise_a,
'act_noise_b': args.act_noise_b,
'rank': args.noise_rank,
'num_classes': args.num_classes
}
print("Optimizinig over noise and number of Monte-Carlo iterations")
results = [[] for iterations in args.iterations_list]
results_adv = [[] for iterations in args.iterations_list]
max_adv_accuracy = -1
max_accuracy = -1
max_accuracy_noise = 0
max_accuracy_iteration = 0
for iterations_idx, iterations in enumerate(args.iterations_list):
for noise in args.noise_list:
print("starting run attack with noise=" + str(noise) +
" and iterations=" + str(iterations))
smoothing_args = {
'noise_sd': noise,
'm_test': iterations,
'm_train': args.m_train
}
# smoothing_args = {'noise_sd': noise, 'm_test': iterations, 'm_train': args.m_train, 'logits': args.logits}
model = args.net(width=args.width, **smoothing_args, **add_args)
if args.smootht:
smoothing_args_t = smoothing_args
else:
smoothing_args_t = {'noise_sd': 0, 'm_test': 1, 'm_train': 1}
modelt = args.net(width=args.width, **smoothing_args, **add_args)
# if args.noise_sd != 0.0:
# model = SmoothModel(base_model=model, noise_sd=noise, m_train=args.m_train, m_test=iterations)
num_parameters = sum([l.nelement() for l in model.parameters()])
print(model)
print("Number of parameters {}".format(num_parameters))
train_loader, val_loader, _ = args.get_loaders(
args.dataset, args.data, args.batch_size, args.batch_size,
args.workers)
# define loss function (criterion) and optimizer
criterion = torch.nn.CrossEntropyLoss()
model, modelt, criterion = model.to(
device=device, dtype=dtype), modelt.to(
device=device, dtype=dtype), criterion.to(device=device,
dtype=dtype)
if args.opt == 'sgd':
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.decay,
nesterov=args.nesterov_momentum)
elif args.opt == 'adam':
optimizer = torch.optim.Adam(model.parameters(),
args.learning_rate,
weight_decay=args.decay)
else:
raise ValueError('Wrong optimzier!')
scheduler = MultiStepLR(optimizer,
milestones=args.schedule,
gamma=args.gamma)
best_test = 0
# optionally resume from a checkpoint
data = None
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
elif os.path.isdir(args.resume):
checkpoint_path = os.path.join(args.resume,
'checkpoint.pth.tar')
csv_path = os.path.join(args.resume, 'results.csv')
print("=> loading checkpoint '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path,
map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
checkpoint_path, checkpoint['epoch']))
data = []
with open(csv_path) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
data.append(row)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
if args.trans:
if os.path.isfile(args.trans):
print("=> loading checkpoint '{}'".format(args.trans))
checkpoint = torch.load(args.trans, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
modelt.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.trans, checkpoint['epoch']))
elif os.path.isdir(args.trans):
checkpoint_path = os.path.join(args.trans,
'checkpoint.pth.tar')
csv_path = os.path.join(args.trans, 'results.csv')
print("=> loading checkpoint '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path,
map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
modelt.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
checkpoint_path, checkpoint['epoch']))
data = []
with open(csv_path) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
data.append(row)
else:
print("=> no checkpoint found at '{}'".format(args.trans))
normalize = {
'mean': np.array([0.491, 0.482, 0.447]),
'std': np.array([0.247, 0.243, 0.262])
}
if args.gen_adv:
adv_s = gen_attack(model, train_loader, criterion, args.attack,
args.eps, normalize, device, dtype)
torch.save(adv_s,
os.path.join(args.save_path, 'adv_samples.tar'))
else:
for layer in model.modules():
from layers import NoisedConv2D, NoisedLinear, NoisedConv2DColored
if isinstance(layer, NoisedConv2D) or isinstance(
layer, NoisedLinear):
print("Mean of alphas is {}".format(
torch.mean(layer.alpha)))
if isinstance(layer, NoisedConv2DColored):
try:
print(
"Mean of alphas_diag_w is {}+-{} ({}) ".format(
torch.mean(torch.abs(layer.alphad_w)),
torch.std(torch.abs(layer.alphad_w)),
torch.max(torch.abs(layer.alphad_w))))
print("Mean of alphas_factor_w is {}+-{} ({}) ".
format(torch.mean(torch.abs(layer.alphaf_w)),
torch.std(layer.alphaf_w),
torch.max(torch.abs(layer.alphaf_w))))
except:
pass
try:
print("Mean of alphas_diag_a is {}+-{} ({}) ".
format(torch.mean(torch.abs(layer.alphad_i)),
torch.std(torch.abs(layer.alphad_i)),
torch.max(torch.abs(layer.alphad_i))))
print("Mean of alphas_factor_a is {}+-{} ({}) ".
format(torch.mean(torch.abs(layer.alphaf_i)),
torch.std(layer.alphaf_i),
torch.max(torch.abs(layer.alphaf_i))))
except:
pass
if args.attack == Transfer:
a = smoothing_args
a.update(add_args)
a['width'] = args.width
if args.tepgd:
att_object = args.attack(model, criterion, modelt,
EPGD_FGSMk)
else:
att_object = args.attack(model, criterion, modelt,
PGD_FGSMk)
else:
att_object = args.attack(model, criterion)
test_loss, accuracy1, accuracy5, test_loss_a, accuracy1_a, accuracy5_a = \
attack(model, val_loader, criterion, att_object, args.eps, normalize, device, dtype)
results[iterations_idx].append(accuracy1)
results_adv[iterations_idx].append(accuracy1_a)
print("finished run attack with noise=" + str(noise) +
" and iterations=" + str(iterations))
print("args.iterations_list")
print(args.iterations_list)
print("args.noise_list")
print(args.noise_list)
for idx in range(iterations_idx + 1):
current_argmax = np.argmax(results_adv[idx])
# print("current_argmax")
# print(current_argmax)
# current_argmax = int(current_argmax)
current_max_adv_accuracy = results_adv[idx][current_argmax]
current_max_accuracy = results[idx][current_argmax]
current_max_accuracy_noise = args.noise_list[
current_argmax]
current_iteration = args.iterations_list[idx]
if current_max_adv_accuracy > max_adv_accuracy:
max_adv_accuracy = current_max_adv_accuracy
max_accuracy = current_max_accuracy
max_accuracy_noise = current_max_accuracy_noise
max_accuracy_iteration = current_iteration
# print("results for iterations_num =" + str(current_iteration))
print("results[iterations_num=" + str(current_iteration) +
"]")
print(results[idx])
print("results_adv[iterations_num=" +
str(current_iteration) + "]")
print(results_adv[idx])
print(
"best result for this iterations_num is adverserial_accuracy="
+ str(current_max_adv_accuracy) + " accuracy=" +
str(current_max_accuracy) +
" and was recived for iterations_num=" +
str(current_iteration) + " and noise=" +
str(current_max_accuracy_noise))
print("best result so far is adverserial_accuracy=" +
str(max_adv_accuracy) + " accuracy=" +
str(max_accuracy) +
" and was recived for iterations_num=" +
str(max_accuracy_iteration) + " and noise=" +
str(max_accuracy_noise))
return max_accuracy_iteration, max_accuracy_noise, max_adv_accuracy, max_accuracy
def main():
args = get_args()
device, dtype = args.device, args.dtype
noise_sd = args.noise_sd
m_test = args.m_test
m_train = args.m_train
if args.check_hyper_params:
max_accuracy_iteration, max_accuracy_noise, max_adv_accuracy, max_accuracy = check_hyper(
args)
noise_sd = max_accuracy_noise
m_test = max_accuracy_iteration
if args.optimize_hyper_params:
max_accuracy_iteration, max_accuracy_noise, max_adv_accuracy, max_accuracy = optimize_hyper(
args)
noise_sd = max_accuracy_noise
m_test = max_accuracy_iteration
add_args = {'num_classes': args.num_classes}
if args.cpni:
add_args = {
'weight_noise': args.weight_noise,
'act_noise_a': args.act_noise_a,
'act_noise_b': args.act_noise_b,
'rank': args.noise_rank,
'num_classes': args.num_classes
}
if args.dataset == torchvision.datasets.ImageNet:
add_args['pretrained'] = True
else:
add_args['width'] = args.width
add_args['num_classes'] = args.num_classes
smoothing_args = {}
if args.smoothing:
# smoothing_args = {'noise_sd': noise_sd, 'm_test': m_test, 'm_train': m_train, 'logits': args.logits}
smoothing_args = {
'noise_sd': noise_sd,
'm_test': m_test,
'm_train': m_train
}
model = args.net(**smoothing_args, **add_args)
modelt = args.net(**smoothing_args, **add_args)
num_parameters = sum([l.nelement() for l in model.parameters()])
print(model)
print("Number of parameters {}".format(num_parameters))
train_loader, val_loader, _ = args.get_loaders(args.dataset, args.data,
args.batch_size,
args.batch_size,
args.workers)
# define loss function (criterion) and optimizer
criterion = torch.nn.CrossEntropyLoss()
model, modelt, criterion = model.to(device=device, dtype=dtype), modelt.to(
device=device, dtype=dtype), criterion.to(device=device, dtype=dtype)
if args.opt == 'sgd':
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.decay,
nesterov=args.nesterov_momentum)
elif args.opt == 'adam':
optimizer = torch.optim.Adam(model.parameters(),
args.learning_rate,
weight_decay=args.decay)
else:
raise ValueError('Wrong optimzier!')
scheduler = MultiStepLR(optimizer,
milestones=args.schedule,
gamma=args.gamma)
best_test = 0
# optionally resume from a checkpoint
data = None
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
elif os.path.isdir(args.resume):
checkpoint_path = os.path.join(args.resume, 'checkpoint.pth.tar')
csv_path = os.path.join(args.resume, 'results.csv')
print("=> loading checkpoint '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
checkpoint_path, checkpoint['epoch']))
data = []
with open(csv_path) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
data.append(row)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
if args.trans:
if os.path.isfile(args.trans):
print("=> loading checkpoint '{}'".format(args.trans))
checkpoint = torch.load(args.trans, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
modelt.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.trans, checkpoint['epoch']))
elif os.path.isdir(args.trans):
checkpoint_path = os.path.join(args.trans, 'checkpoint.pth.tar')
csv_path = os.path.join(args.trans, 'results.csv')
print("=> loading checkpoint '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
modelt.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
checkpoint_path, checkpoint['epoch']))
data = []
with open(csv_path) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
data.append(row)
else:
print("=> no checkpoint found at '{}'".format(args.trans))
normalize = {
'mean': np.array([0.491, 0.482, 0.447]),
'std': np.array([0.247, 0.243, 0.262])
}
if args.gen_adv:
adv_s = gen_attack(model, train_loader, criterion, args.attack,
args.eps, normalize, device, dtype)
torch.save(adv_s, os.path.join(args.save_path, 'adv_samples.tar'))
else:
for layer in model.modules():
from layers import NoisedConv2D, NoisedLinear, NoisedConv2DColored
if isinstance(layer, NoisedConv2D) or isinstance(
layer, NoisedLinear):
print("Mean of alphas is {}".format(torch.mean(layer.alpha)))
if isinstance(layer, NoisedConv2DColored):
try:
print("Mean of alphas_diag_w is {}+-{} ({}) ".format(
torch.mean(torch.abs(layer.alphad_w)),
torch.std(torch.abs(layer.alphad_w)),
torch.max(torch.abs(layer.alphad_w))))
print("Mean of alphas_factor_w is {}+-{} ({}) ".format(
torch.mean(torch.abs(layer.alphaf_w)),
torch.std(layer.alphaf_w),
torch.max(torch.abs(layer.alphaf_w))))
except:
pass
try:
print("Mean of alphas_diag_a is {}+-{} ({}) ".format(
torch.mean(torch.abs(layer.alphad_i)),
torch.std(torch.abs(layer.alphad_i)),
torch.max(torch.abs(layer.alphad_i))))
print("Mean of alphas_factor_a is {}+-{} ({}) ".format(
torch.mean(torch.abs(layer.alphaf_i)),
torch.std(layer.alphaf_i),
torch.max(torch.abs(layer.alphaf_i))))
except:
pass
if args.attack == Transfer:
a = smoothing_args
a.update(add_args)
a['width'] = args.width
att_object = args.attack(model, criterion, modelt, PGD_FGSMk)
else:
att_object = args.attack(model, criterion)
attack(model, val_loader, criterion, att_object, args.eps, normalize,
device, dtype)
def optimize_hyper(args):
device, dtype = args.device, args.dtype
add_args = {'num_classes': args.num_classes}
if args.cpni:
add_args = {
'weight_noise': args.weight_noise,
'act_noise_a': args.act_noise_a,
'act_noise_b': args.act_noise_b,
'rank': args.noise_rank,
'num_classes': args.num_classes
}
print("Optimizinig over noise and number of Monte-Carlo iterations")
results = [[] for iterations in args.iterations_list]
results_adv = [[] for iterations in args.iterations_list]
noise_list = [[] for iterations in args.iterations_list]
max_adv_accuracy = -1
max_accuracy = -1
max_accuracy_noise = 0
max_accuracy_iteration = 0
for iterations_idx, iterations in enumerate(args.iterations_list):
# smoothing_args = {'noise_sd': args.noise_sd, 'm_test': iterations, 'm_train': args.m_train,
# 'logits': args.logits, 'learn_noise': True}
smoothing_args = {
'noise_sd': args.noise_sd,
'm_test': iterations,
'm_train': args.m_train,
'learn_noise': True
}
model = args.net(width=args.width, **smoothing_args, **add_args)
num_parameters = sum([l.nelement() for l in model.parameters()])
print(model)
print("Number of parameters {}".format(num_parameters))
train_loader, val_loader, _ = args.get_loaders(args.dataset, args.data,
args.batch_size,
args.batch_size,
args.workers)
# define loss function (criterion) and optimizer
criterion = torch.nn.CrossEntropyLoss()
model, criterion = model.to(device=device,
dtype=dtype), criterion.to(device=device,
dtype=dtype)
model_noise_parameter = [
# {'params': model.parameters()},
{
'params': [model.noise_sd]
}
]
if args.opt == 'sgd':
optimizer = torch.optim.SGD(model_noise_parameter,
args.learning_rate,
momentum=args.momentum,
weight_decay=args.decay,
nesterov=args.nesterov_momentum)
elif args.opt == 'adam':
optimizer = torch.optim.Adam(model_noise_parameter,
args.learning_rate,
weight_decay=args.decay)
else:
raise ValueError('Wrong optimzier!')
scheduler = MultiStepLR(optimizer,
milestones=args.schedule,
gamma=args.gamma)
best_test = 0
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
elif os.path.isdir(args.resume):
checkpoint_path = os.path.join(args.resume,
'checkpoint.pth.tar')
csv_path = os.path.join(args.resume, 'results.csv')
print("=> loading checkpoint '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
checkpoint_path, checkpoint['epoch']))
data = []
with open(csv_path) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
data.append(row)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
normalize = {
'mean': np.array([0.491, 0.482, 0.447]),
'std': np.array([0.247, 0.243, 0.262])
}
att_object = args.attack(model, criterion)
for epoch in trange(args.start_epoch, args.epochs + 1):
adv_train(model, train_loader, 0, optimizer, criterion, device,
dtype, args.batch_size, args.log_interval, args.attack,
args.eps, 0.5, normalize)
test_loss, accuracy1, accuracy5, test_loss_a, accuracy1_a, accuracy5_a = attack(
model, val_loader, criterion, att_object, args.eps, normalize,
device, dtype)
results[iterations_idx].append(accuracy1)
results_adv[iterations_idx].append(accuracy1_a)
noise = model.noise_sd.item()
noise_list[iterations_idx].append(noise)
print("finished run attack with noise=" + str(noise) +
" and iterations=" + str(iterations))
print("args.iterations_list")
print(args.iterations_list)
for idx in range(iterations_idx + 1):
current_argmax = np.argmax(results_adv[idx])
# print("current_argmax")
# print(current_argmax)
# current_argmax = int(current_argmax)
current_max_adv_accuracy = results_adv[idx][current_argmax]
current_max_accuracy = results[idx][current_argmax]
current_max_accuracy_noise = noise_list[idx][current_argmax]
current_iteration = args.iterations_list[idx]
if current_max_adv_accuracy > max_adv_accuracy:
max_adv_accuracy = current_max_adv_accuracy
max_accuracy = current_max_accuracy
max_accuracy_noise = current_max_accuracy_noise
max_accuracy_iteration = current_iteration
# print("results for iterations_num =" + str(current_iteration))
print("noise_list[iterations_num=" + str(current_iteration) +
"]")
print(noise_list[idx])
print("results[iterations_num=" + str(current_iteration) + "]")
print(results[idx])
print("results_adv[iterations_num=" + str(current_iteration) +
"]")
print(results_adv[idx])
print(
"best result for this iterations_num is adverserial_accuracy="
+ str(current_max_adv_accuracy) + " accuracy=" +
str(current_max_accuracy) +
" and was recived for iterations_num=" +
str(current_iteration) + " and noise=" +
str(current_max_accuracy_noise))
print("best result so far is adverserial_accuracy=" +
str(max_adv_accuracy) + " accuracy=" + str(max_accuracy) +
" and was recived for iterations_num=" +
str(max_accuracy_iteration) + " and noise=" +
str(max_accuracy_noise))
scheduler.step()
return max_accuracy_iteration, max_accuracy_noise, max_adv_accuracy, max_accuracy