def auto(model, device, val_loader, criterion, args, epoch=0):
"""
Evaluate on atuo-attack adversarial validation set inputs.
"""
batch_time = AverageMeter("Time", ":6.3f")
losses = AverageMeter("Loss", ":.4f")
adv_losses = AverageMeter("Adv_Loss", ":.4f")
top1 = AverageMeter("Acc_1", ":6.2f")
top5 = AverageMeter("Acc_5", ":6.2f")
top1_adv = AverageMeter("Adv-Acc_1", ":6.2f")
top5_adv = AverageMeter("Adv-Acc_5", ":6.2f")
progress = ProgressMeter(
len(val_loader),
[batch_time, losses, adv_losses, top1, top5, top1_adv, top5_adv],
prefix="Test: ",
)
# switch to evaluation mode
model.eval()
assert args.distance in ["linf", "l2"]
print(
"USING ONLY APGD_CE & APGD-T attacks. Rest of them don't change robust accuracy much but take super long to finish."
)
adversary = AutoAttack(model,
norm="Linf" if args.distance == "linf" else "L2",
eps=args.epsilon)
adversary.attacks_to_run = ['apgd-ce']
with torch.no_grad():
end = time.time()
for i, data in enumerate(val_loader):
images, target = data[0].to(device), data[1].to(device)
# clean images
output = model(images)
loss = criterion(output, target)
acc1, acc5 = accuracy(output, target, topk=(1, 2))
losses.update(loss.item(), images.size(0))
top1.update(acc1[0], images.size(0))
top5.update(acc5[0], images.size(0))
images = adversary.run_standard_evaluation(images,
target,
bs=len(images))
# compute output
output = model(images)
loss = criterion(output, target)
# measure accuracy and record loss
acc1, acc5 = accuracy(output, target, topk=(1, 2))
adv_losses.update(loss.item(), images.size(0))
top1_adv.update(acc1[0], images.size(0))
top5_adv.update(acc5[0], images.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % args.print_freq == 0 and args.local_rank == 0:
progress.display(i)
if args.local_rank == 0:
progress.display(i) # print final results
result = {
"top1": top1.avg,
"top5": top5.avg,
"loss": losses.avg,
"top1_adv": top1_adv.avg,
"top5_adv": top5_adv.avg,
"adv_loss": adv_losses.avg
}
return result
def robust_test(model,
loss_fn,
loader,
args,
att_dir,
epoch=0,
training_output_dir_name=None,
figure_dir_name=None,
n_batches=0,
train_data=False,
training_time=False):
"""
n_batches (int): Number of batches for evaluation.
"""
model.eval()
num_correct, num_correct_adv, num_samples = 0, 0, 0
steps = 1
losses_adv = []
losses_ben = []
prob_dict = {}
if args.track_hard:
loss_dict = collections.OrderedDict()
pred_dict = {}
if training_time and args.track_hard:
f_name = training_output_dir_name + 'losses.json'
f = open(f_name, 'a+')
loss_dict['epoch'] = epoch
if 'PGD' in att_dir['attack']:
for t, (x, y, idx, ez, m) in enumerate(loader):
x = x.cuda()
y = y.cuda()
x_var = Variable(x, requires_grad=True)
y_var = Variable(y, requires_grad=False)
if att_dir['targeted']:
y_target = generate_target_label_tensor(y_var.cpu(),
args).cuda()
else:
y_target = y_var
if 'PGD_linf' in att_dir['attack']:
adv_x = pgd_attack(model, x, x_var, y_target,
att_dir['attack_iter'], att_dir['epsilon'],
att_dir['eps_step'], att_dir['clip_min'],
att_dir['clip_max'], att_dir['targeted'],
att_dir['rand_init'])
elif 'PGD_l2' in att_dir['attack']:
adv_x = pgd_l2_attack(model, x, x_var, y_target,
att_dir['attack_iter'],
att_dir['epsilon'], att_dir['eps_step'],
att_dir['clip_min'], att_dir['clip_max'],
att_dir['targeted'],
att_dir['rand_init'],
att_dir['num_restarts'])
# Predictions
scores = model(x.cuda())
_, preds = scores.data.max(1)
scores_adv = model(adv_x)
_, preds_adv = scores_adv.data.max(1)
# Losses
batch_loss_adv = loss_fn(scores_adv, y)
loss_adv = torch.mean(batch_loss_adv)
losses_adv.append(loss_adv.data.cpu().numpy())
batch_loss_ben = loss_fn(scores, y)
loss_ben = torch.mean(batch_loss_ben)
losses_ben.append(loss_ben.data.cpu().numpy())
# Correct count
num_correct += (preds == y).sum()
num_correct_adv += (preds_adv == y).sum()
num_samples += len(preds)
# Adding probs to dict
count = 0
for i in idx.numpy():
score_curr = scores_adv[count].cpu().detach().numpy()
prob_dict[str(i)] = softmax(score_curr)
# print(count)
count += 1
if args.track_hard:
idx_matched = idx[~ez].numpy()
m_matched = m[~ez].numpy()
preds_adv_matched = preds_adv[~ez].cpu().numpy()
loss_adv_matched = batch_loss_adv[~ez].cpu().detach().numpy()
y_np_matched = y[~ez].cpu().numpy()
ez_np = ez.data.cpu().numpy()
for i in range(len(y_np_matched)):
pred_dict[str(idx_matched[i])] = [
m_matched[i], y_np_matched[i], preds_adv_matched[i]
]
loss_dict = track_hard_losses(ez_np, batch_loss_adv,
batch_loss_ben, loss_dict, t)
if not training_time:
if args.viz and steps == 1:
if not os.path.exists(figure_dir_name):
os.makedirs(figure_dir_name)
custom_save_image(adv_x, preds_adv, y, args,
figure_dir_name, train_data)
if n_batches > 0 and steps == n_batches:
break
steps += 1
elif 'AA' in att_dir['attack']:
x_test = []
y_test = []
idx_all = []
for t, (x, y, idx, ez, m) in enumerate(loader):
x_test.append(x)
y_test.append(y)
idx_all.extend(idx.numpy())
x_test = torch.cat(x_test, 0)
y_test = torch.cat(y_test, 0)
print(x_test.size())
print(y_test.size())
adversary = AutoAttack(model,
norm='L2',
eps=att_dir['epsilon'],
version='standard')
adversary.attacks_to_run = ['apgd-ce', 'fab']
adv_x = adversary.run_standard_evaluation(x_test, y_test)
loss_adv, loss_ben, num_correct, num_correct_adv, num_samples, scores_adv = compute_scores_and_losses(
model, loss_fn, x_test, y_test, adv_x)
losses_adv.append(loss_adv.data.cpu().numpy())
losses_ben.append(loss_ben.data.cpu().numpy())
# Adding probs to dict
count = 0
for i in idx_all:
score_curr = scores_adv[count].cpu().detach().numpy()
prob_dict[str(i)] = softmax(score_curr)
# print(count)
count += 1
acc = float(num_correct) / num_samples
acc_adv = float(num_correct_adv) / num_samples
print('Clean accuracy: {:.2f}% ({}/{})'.format(
100. * acc,
num_correct,
num_samples,
))
print('Adversarial accuracy: {:.2f}% ({}/{})'.format(
100. * acc_adv,
num_correct_adv,
num_samples,
))
if args.track_hard:
if not training_time:
print('Counting hard points')
hard_point_class_count(pred_dict)
if training_time:
json.dump(loss_dict, f)
f.write('\n')
if len(loss_dict['batch_losses_hard']) > 0:
print('Adv loss easy: %.8f' %
np.mean(loss_dict['batch_losses_easy']))
print('Adv loss hard: %.8f' %
np.mean(loss_dict['batch_losses_hard']))
print('Ben loss easy: %.8f' %
np.mean(loss_dict['batch_losses_ben_easy']))
print('Ben loss hard: %.8f' %
np.mean(loss_dict['batch_losses_ben_hard']))
return 100. * acc, 100. * acc_adv, np.mean(losses_ben), np.mean(
losses_adv), prob_dict
# load attack
from autoattack import AutoAttack
adversary = AutoAttack(model,
norm=args.norm,
eps=args.epsilon,
log_path=args.log_path,
version=args.version)
l = [x for (x, y) in test_loader]
x_test = torch.cat(l, 0)
l = [y for (x, y) in test_loader]
y_test = torch.cat(l, 0)
# example of custom version
if args.version == 'custom':
adversary.attacks_to_run = ['apgd-ce', 'fab']
adversary.apgd.n_restarts = 2
adversary.fab.n_restarts = 2
# run attack and save images
with torch.no_grad():
if not args.individual:
adv_complete = adversary.run_standard_evaluation(
x_test[:args.n_ex], y_test[:args.n_ex], bs=args.batch_size)
torch.save({'adv_complete': adv_complete},
'{}/{}_{}_1_{}_eps_{:.5f}.pth'.format(
args.save_dir, 'aa', args.version,
adv_complete.shape[0], args.epsilon))
else:
## AA EVAL ##
test_loader = data.DataLoader(item,
batch_size=128,
shuffle=False,
num_workers=0)
from autoattack import AutoAttack
log = 'store/' + log_folder + '/AA_eval-new.txt'
model = model.cuda()
adversary = AutoAttack(model,
norm='Linf',
eps=8 / 255,
log_path=log,
version='standard')
adversary.attacks_to_run = ['apgd-t']
l = [x for (x, y) in test_loader]
x_test = torch.cat(l, 0).cuda()
l = [y for (x, y) in test_loader]
y_test = torch.cat(l, 0).cuda()
adv_complete = adversary.run_standard_evaluation(x_test, y_test, bs=128)
save_dir = 'store/' + log_folder
print(adv_complete)
torch.save({'adv_complete': adv_complete},
'{}/{}_{}_1_{}_eps_{:.5f}.pth'.format(save_dir, 'aa', 'standard',
adv_complete.shape[0],
8 / 255))
## PGD20 EVAL ##
out_store = cox.store.Store('store', exp_id=log_folder)