def main(_):
# Load training and test data
data = ld_cifar10()
# Instantiate model, loss, and optimizer for training
net = CNN(in_channels=3)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if device == 'cuda':
net = net.cuda()
loss_fn = torch.nn.CrossEntropyLoss(reduction='mean')
optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)
# Train vanilla model
net.train()
for epoch in range(1, FLAGS.nb_epochs + 1):
train_loss = 0.
for x, y in data.train:
x, y = x.to(device), y.to(device)
if FLAGS.adv_train:
# Replace clean example with adversarial example for adversarial training
x = projected_gradient_descent(net, x, FLAGS.eps, 0.01, 40,
np.inf)
# Stop backward from entering the graph that created the adv example
x = x.clone().detach()
optimizer.zero_grad()
loss = loss_fn(net(x), y)
loss.backward()
optimizer.step()
train_loss += loss.item()
print('epoch: {}/{}, train loss: {:.3f}'.format(
epoch, FLAGS.nb_epochs, train_loss))
# Evaluate on clean and adversarial data
net.eval()
report = EasyDict(nb_test=0, correct=0, correct_fgm=0, correct_pgd=0)
for x, y in data.test:
x, y = x.to(device), y.to(device)
x_fgm = fast_gradient_method(net, x, FLAGS.eps, np.inf)
x_pgd = projected_gradient_descent(net, x, FLAGS.eps, 0.01, 40, np.inf)
_, y_pred = net(x).max(1) # model prediction on clean examples
_, y_pred_fgm = net(x_fgm).max(
1) # model prediction on FGM adversarial examples
_, y_pred_pgd = net(x_pgd).max(
1) # model prediction on PGD adversarial examples
report.nb_test += y.size(0)
report.correct += y_pred.eq(y).sum().item()
report.correct_fgm += y_pred_fgm.eq(y).sum().item()
report.correct_pgd += y_pred_pgd.eq(y).sum().item()
print('test acc on clean examples (%): {:.3f}'.format(
report.correct / report.nb_test * 100.))
print('test acc on FGM adversarial examples (%): {:.3f}'.format(
report.correct_fgm / report.nb_test * 100.))
print('test acc on PGD adversarial examples (%): {:.3f}'.format(
report.correct_pgd / report.nb_test * 100.))
def test(opt, net, loader):
correct = 0
if opt.adversarial:
correct_fgm, correct_pgd = 0, 0
total = 0
net.eval()
logger.info('Starting testing...')
with torch.set_grad_enabled(opt.adversarial):
# adversarial attacks need grad computations
n_batches = len(loader)
for i, (input, target) in enumerate(loader):
logger.info('batch: {}/{}'.format(i + 1, n_batches))
input, target = \
input.to(device, non_blocking=True), target.to(
device, non_blocking=True)
output = net(input, update_centers=False)
_, predicted = torch.max(output.data, 1)
total += target.size(0)
correct += (predicted == target).sum().item()
if opt.adversarial:
def net_fn(input):
return net(input, update_centers=False)
input_fgm = fast_gradient_method(net_fn, input,
opt.adversary_eps,
opt.adversary_norm)
input_pgd = projected_gradient_descent(net_fn, input,
opt.adversary_eps,
opt.pgd_step_eps,
opt.pgd_n_steps,
opt.adversary_norm)
output_fgm = net(input_fgm, update_centers=False)
output_pgd = net(input_pgd, update_centers=False)
_, predicted_fgm = torch.max(output_fgm.data, 1)
_, predicted_pgd = torch.max(output_pgd.data, 1)
correct_fgm += (predicted_fgm == target).sum().item()
correct_pgd += (predicted_pgd == target).sum().item()
acc = 100 * correct / total
results = {'accuracy (%)': acc}
logger.info('Accuracy (%): {:.3f}'.format(acc))
if opt.adversarial:
acc_fgm = 100 * correct_fgm / total
logger.info('Accuracy under FGM (%): {:.3f}'.format(acc_fgm))
acc_pgd = 100 * correct_pgd / total
logger.info('Accuracy under PGD (%): {:.3f}'.format(acc_pgd))
results['accuracy under FGM (%)'] = acc_fgm
results['accuracy under PGD (%)'] = acc_pgd
with open(os.path.join(opt.save_dir, 'test.json'), 'w') as out:
json.dump(results, out, indent=2)
logger.info('Testing finished!')
def test(opt, net, loader):
correct = 0
if opt.adversarial:
correct_fgm, correct_pgd = 0, 0
total = 0
net.eval()
logger.info('Starting testing...')
with torch.set_grad_enabled(opt.adversarial):
# adversarial attacks need grad computations
for i, (input, target) in enumerate(loader):
logger.info('batch: {}/{}'.format(i, len(loader)))
input, target = \
input.to(device, non_blocking=True), target.to(device, non_blocking=True)
output = net(input, update_centers=False)
# save tensors for visualization
raw_data = input if i == 0 else torch.cat((raw_data, input))
labels = target if i == 0 else torch.cat((labels, target))
activations = net_head(input)
all_activations = activations if i == 0 else torch.cat((all_activations, activations))
_, predicted = torch.max(output.data, 1)
total += target.size(0)
correct += (predicted == target).sum().item()
if opt.adversarial:
net_fn = lambda input: net(input, update_centers=False)
input_fgm = fast_gradient_method(net_fn, input, opt.adversary_eps, opt.adversary_norm)
input_pgd = projected_gradient_descent(net_fn, input, opt.adversary_eps,
opt.pgd_step_eps, opt.pgd_n_steps, opt.adversary_norm)
output_fgm = net(input_fgm, update_centers=False)
output_pgd = net(input_pgd, update_centers=False)
_, predicted_fgm = torch.max(output_fgm.data, 1)
_, predicted_pgd = torch.max(output_pgd.data, 1)
correct_fgm += (predicted_fgm == target).sum().item()
correct_pgd += (predicted_pgd == target).sum().item()
logger.info('Accuracy (%): {:.3f}'.format(100 * correct / total))
if opt.adversarial:
logger.info('Accuracy under FGM (%): {:.3f}'.format(100 * correct_fgm / total))
logger.info('Accuracy under PGD (%): {:.3f}'.format(100 * correct_pgd / total))
logger.info('Testing finished!')
return raw_data.cpu().numpy(), labels.cpu().numpy(), all_activations.cpu().numpy()
def run_attacks_cleverhans(res_path, ncl=False):
if ncl:
MODEL_DIR = '/mnt/md0/orville/Miriam/modular-loss-experiments-morph/results_ncl/CIFAR-10/densenet-82-8-8'
rel_dirs = ['alpha_0.0_gamma_0.02_n_models_2_1583114412120',
'alpha_0.0_gamma_0.05_n_models_2_1583114439810']
alpha = ['0.02', '0.05']
res_path = res_path + '_ncl'
else:
MODEL_DIR = '/mnt/md0/orville/Miriam/modular-loss-experiments-morph/results/CIFAR-10/densenet-82-8-8'
# UNCORR_MODEL_DIR = 'alpha_0.0_gamma_0.0_n_models_2_1581641733617'
# CORR_MODEL_DIR = 'alpha_0.1_gamma_0.0_n_models_2_1581641746832'
# CORR_MODEL_DIR_2 = 'alpha_0.2_gamma_0.0_n_models_2_1581641777871'
# UNCORR_MODEL_DIR = 'alpha_0.0_gamma_0.0_n_models_3_1585505819121'
# CORR_MODEL_DIR = 'alpha_0.1_gamma_0.0_n_models_3_1585505685528'
# CORR_MODEL_DIR_2 = 'alpha_0.2_gamma_0.0_n_models_3_1585505042819'
# rel_dirs = [UNCORR_MODEL_DIR, CORR_MODEL_DIR, CORR_MODEL_DIR_2]
rel_dirs = ['alpha_0.0_gamma_0.0_n_models_3_1585505819121',
'alpha_0.1_gamma_0.0_n_models_3_1589795142450',
'alpha_0.2_gamma_0.0_n_models_3_1589794987034',
'alpha_0.3_gamma_0.0_n_models_3_1589795486214',
'alpha_0.4_gamma_0.0_n_models_3_1589796192038',
'alpha_0.5_gamma_0.0_n_models_3_1589796200262',
'alpha_0.6_gamma_0.0_n_models_3_1589796218204',
'alpha_0.7_gamma_0.0_n_models_3_1589796234665']
alpha = list(map(lambda x: format(x, '2.1f'), np.arange(0.0, 0.8, 0.1)))
batch_size = 256 # 128 # 516
n_workers = 20
dataset = 'CIFAR-10'
network = 'densenet-82-8-8'
loaders, _ = get_dataloaders_(batch_size, 0, dataset, False, early_stop=False, n_workers=n_workers)
n_models = 2 if ncl else 3
params = {}
params['densenet-82-8-8'] = {'num_modules': n_models, 'bottleneck': True, 'reduction': 0.5, 'depth': 82, 'growth_rate': 8,
'input_shape': (3, 32, 32), 'output_dim': 10}
network = 'densenet-82-8-8'
model = DenseNet(input_shape=params[network]['input_shape'],
output_dim=params[network]['output_dim'],
growth_rate=params[network]['growth_rate'],
depth=params[network]['depth'],
reduction=params[network]['reduction'],
bottleneck=params[network]['bottleneck'],
num_modules=n_models)
device = torch.device("cuda")
reports = dict.fromkeys(alpha)
for model_path, curr_alpha in tqdm(zip(rel_dirs, alpha), total=len(alpha)):
if ncl:
weight_path = path.join(MODEL_DIR, model_path, 'trial_0/' + curr_alpha + '/weights/final_weights.pt')
else:
weight_path = path.join(MODEL_DIR, model_path, 'trial_0/0.0/weights/final_weights.pt')
model.reset_parameters()
model.load_state_dict(torch.load(weight_path))
model.eval() # model.train(mode=False)
net = ModelMeanEP(model).to(device)
report = dict()
for x, y in tqdm(loaders['test'], total=len(loaders['test'])):
x, y = x.to(device), y.to(device)
report['nb_test'] = report.get('nb_test', 0) + y.size(0)
_, y_pred = net(x).max(1) # model prediction on clean examples
report['acc'] = report.get('acc', 0) + y_pred.eq(y).sum().item()
# model prediction on FGM adversarial examples
x_adv = fast_gradient_method(net, x, 0.02, np.inf)
_, y_pred = net(x_adv).max(1) # model prediction on FGM adversarial examples
report['FGM_0.02'] = report.get('FGM_0.02', 0) + y_pred.eq(y).sum().item()
x_adv = fast_gradient_method(net, x, 0.04, np.inf)
_, y_pred = net(x_adv).max(1) # model prediction on FGM adversarial examples
report['FGM_0.04'] = report.get('FGM_0.04', 0) + y_pred.eq(y).sum().item()
# model prediction on BIM adversarial examples
x_adv = projected_gradient_descent(net, x, eps=0.01, eps_iter=0.01 / 10, nb_iter=10, norm=np.inf, rand_init=0)
_, y_pred = net(x_adv).max(1)
report['BIM_0.01'] = report.get('BIM_0.01', 0) + y_pred.eq(y).sum().item()
x_adv = projected_gradient_descent(net, x, eps=0.02, eps_iter=0.02 / 10, nb_iter=10, norm=np.inf, rand_init=0)
_, y_pred = net(x_adv).max(1)
report['BIM_0.02'] = report.get('BIM_0.02', 0) + y_pred.eq(y).sum().item()
# model prediction on PGD adversarial examples
x_adv = projected_gradient_descent(net, x, eps=0.01, eps_iter=0.01 / 10, nb_iter=10, norm=np.inf)
_, y_pred = net(x_adv).max(1)
report['PGD_0.01'] = report.get('PGD_0.01', 0) + y_pred.eq(y).sum().item()
x_adv = projected_gradient_descent(net, x, eps=0.02, eps_iter=0.02 / 10, nb_iter=10, norm=np.inf)
_, y_pred = net(x_adv).max(1)
report['PGD_0.02'] = report.get('PGD_0.02', 0) + y_pred.eq(y).sum().item()
for key in ['acc', 'FGM_0.02', 'FGM_0.04', 'BIM_0.01', 'BIM_0.02', 'PGD_0.01', 'PGD_0.02']:
report[key] = (report[key] / report['nb_test']) * 100.
reports[curr_alpha] = report
pickle.dump(reports, open(res_path, 'wb'))
pickle.dump(reports, open(res_path, 'wb'))
def main(_):
# Load training and test data
data = ld_cifar10()
# Instantiate model, loss, and optimizer for training
# net = CNN(in_channels=3)
net = resnet.ResNet18()
device = 'cuda' if torch.cuda.is_available() else 'cpu'
logging.info('Using GPU' if device == 'cuda' else 'Using CPU')
if device == 'cuda':
net = net.cuda()
loss_fn = torch.nn.CrossEntropyLoss(reduction='mean')
optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)
# load checkpoint if exists
if os.path.exists(FLAGS.checkpoint):
ckpt = torch.load(FLAGS.checkpoint)
net.load_state_dict(ckpt['net'])
logging.info('Loaded model %s with accuracy %.3f', FLAGS.checkpoint,
ckpt['acc'])
else: # Train vanilla model
if input('No checkpoint found, continue? y/[n]') != 'y':
return -1
net.train()
with trange(1, FLAGS.nb_epochs + 1, desc='Training',
unit='Epoch') as t:
for epoch in t:
train_loss = 0.
for x, y in data.train:
x, y = x.to(device), y.to(device)
if FLAGS.adv_train:
# Replace clean example with adversarial example for adversarial training
x = projected_gradient_descent(net, x, FLAGS.eps, 0.01,
40, np.inf)
optimizer.zero_grad()
loss = loss_fn(net(x), y)
loss.backward()
optimizer.step()
train_loss += loss.item()
t.set_description('Train Loss=%.3f' % train_loss)
# Evaluate on clean and adversarial data
net.eval()
report = EasyDict(nb_test=0, correct=0, correct_fgm=0, correct_pgd=0)
for x, y in tqdm(data.test, unit='Samples', desc='Testing'):
x, y = x.to(device), y.to(device)
x_fgm = fast_gradient_method(net, x, FLAGS.eps, np.inf)
x_pgd = projected_gradient_descent(net, x, FLAGS.eps, 0.01, 40, np.inf)
_, y_pred = net(x).max(1) # model prediction on clean examples
_, y_pred_fgm = net(x_fgm).max(
1) # model prediction on FGM adversarial examples
_, y_pred_pgd = net(x_pgd).max(
1) # model prediction on PGD adversarial examples
report.nb_test += y.size(0)
report.correct += y_pred.eq(y).sum().item()
report.correct_fgm += y_pred_fgm.eq(y).sum().item()
report.correct_pgd += y_pred_pgd.eq(y).sum().item()
print('test acc on clean examples (%): {:.3f}'.format(
report.correct / report.nb_test * 100.))
print('test acc on FGM adversarial examples (%): {:.3f}'.format(
report.correct_fgm / report.nb_test * 100.))
print('test acc on PGD adversarial examples (%): {:.3f}'.format(
report.correct_pgd / report.nb_test * 100.))
return 0