def get_theoretic_lowerbound(model, eps_range, config, pretrained_config,
output_root):
n_examples = config['num_examples']
n_batches = int(
math.ceil((n_examples * 1.0) / pretrained_config.optim.batch_size))
pretrained_config.cuda = pretrained_config.cuda
data = load_data(pretrained_config)
# Perform the attack.
eps_acc = []
for sample in tqdm(islice(data['validation'], n_batches), total=n_batches):
x = to_cuda(sample[0], cuda=pretrained_config.cuda)
y = to_cuda(sample[1].type(torch.LongTensor),
cuda=pretrained_config.cuda)
eps_acc.append(
theoretical_adversary(
model, x, y, pretrained_config.model.l_constant * eps_range))
avg_eps_acc = np.concatenate(eps_acc, 0).mean(0)
results = {'eps': eps_range.tolist(), 'acc': avg_eps_acc.tolist()}
print(results)
with open(os.path.join(output_root, 'results.json'), 'w') as f:
json.dump(results, f, sort_keys=True, indent=4)
def main(opt):
exp_dir = opt['model']['exp_path']
model_path = os.path.join(exp_dir, 'checkpoints', 'best', 'best_model.pt')
with open(os.path.join(exp_dir, 'logs', 'config.json'), 'r') as f:
model_config = Munch.fromDict(json.load(f))
# Weird required hack to fix groupings (None is added to start during model training)
if 'groupings' in model_config.model and model_config.model.groupings[0] is -1:
model_config.model.groupings = model_config.model.groupings[1:]
model_config.cuda = opt['cuda']
model_config.data.cuda = opt['cuda']
model = get_model(model_config)
model.load_state_dict(torch.load(model_path))
if opt['cuda']:
print('Using CUDA')
model.cuda()
def on_sample(state):
if opt['cuda']:
state['sample'] = [x.cuda() for x in state['sample']]
def on_forward(state):
state['model'].add_to_meters(state)
data = load_data(model_config)
# Change the model to use ortho layers by copying the base weights
model = convert_model_from_bjorck(model, model_config)
# model.model.project_network_weights(Munch.fromDict({'type': 'l_inf_projected'}))
# Instantiate the trainer.
trainer = Trainer()
trainer.hooks['on_sample'] = on_sample
trainer.hooks['on_forward'] = on_forward
print('TESTING')
state = trainer.test(model, data['test'])
for tag, meter in state['model'].meters.items():
print(tag, meter.value())
logit_margins = check_logit_margins(model, data['test'])
print(logit_margins.min().item(), logit_margins.max().item(), logit_margins.mean().item())
plt.hist(logit_margins.detach().cpu().numpy())
plt.show()
def main(opt):
# if not os.path.isdir(opt['output_root']):
# os.makedirs(opt['output_root'])
exp_dir = opt['model']['exp_path']
model_path = os.path.join(exp_dir, 'checkpoints', 'best', 'best_model.pt')
with open(os.path.join(exp_dir, 'logs', 'config.json'), 'r') as f:
model_config = Munch.fromDict(json.load(f))
# Weird required hack to fix groupings (None is added to start during model training)
if 'groupings' in model_config.model and model_config.model.groupings[
0] is -1:
model_config.model.groupings = model_config.model.groupings[1:]
model_config.model.linear.bjorck_iters = 20
model_config.cuda = opt['cuda']
model = get_model(model_config)
model.load_state_dict(torch.load(model_path))
if opt['cuda']:
print('Using CUDA')
model.cuda()
data = load_data(model_config)
# Change the model to use ortho layers by copying the base weights
model = convert_model_from_bjorck(model, model_config)
model.eval()
rates = []
thresholds = np.linspace(0.0, 1.0, 50, endpoint=True)
for t in thresholds:
undead_rate = get_undead_rate(model,
data['test'],
threshold=t,
cuda=opt['cuda'])
rates.append(undead_rate)
plt.plot(thresholds, np.array(rates))
plt.show()
np.save('undead_rates', rates)
print(undead_rate)
def main(opt):
if not os.path.isdir(opt['output_root']):
os.makedirs(opt['output_root'])
exp_dir = opt['model']['exp_path']
model_path = os.path.join(exp_dir, 'checkpoints', 'best', 'best_model.pt')
with open(os.path.join(exp_dir, 'logs', 'config.json'), 'r') as f:
model_config = Munch.fromDict(json.load(f))
# Weird required hack to fix groupings (None is added to start during model training)
if 'groupings' in model_config.model and model_config.model.groupings[
0] is -1:
model_config.model.groupings = model_config.model.groupings[1:]
model = get_model(model_config)
model.load_state_dict(torch.load(model_path))
if opt['data']['cuda']:
print('Using CUDA')
model.cuda()
model_config.data.cuda = opt['data']['cuda']
data = load_data(model_config)
# Change the model to use ortho layers by copying the base weights
bjorck_iters = 50
model = convert_model_to_bjorck(model, model_config)
for m in model.modules():
if isinstance(m, BjorckLinear):
m.config.model.linear.bjorck_iter = bjorck_iters
model_config.output_root = opt['output_root']
model_config.optim.lr_schedule.lr_init = 1e-5
model_config.optim.epochs = 5
model = train(model, data, model_config)
'wait': 0
})
# Enter the training loop.
trainer.train(model,
loaders['train'],
maxepoch=config.optim.epochs,
optimizer=optimizer)
# Pick the best model according to validation score and test it.
model.reset_meters()
best_model_path = os.path.join(dirs.best_path, "best_model.pt")
if os.path.exists(best_model_path):
model.load_state_dict(torch.load(best_model_path))
if loaders['test'] is not None:
print("Testing the best model. ")
logger.log_meters('test', trainer.test(model, loaders['test']))
return model
if __name__ == '__main__':
# Get the config, initialize the model and construct the data loader.
cfg = process_config()
model_initialization = get_model(cfg)
print(model_initialization)
data_loaders = load_data(cfg)
# Train.
trained_model = train(model_initialization, data_loaders, cfg)
def generate_examples(model, config, pretrained_config, output_root):
adv_example_filepath = os.path.join(output_root, 'examples')
adv_targets_filepath = os.path.join(output_root, 'targets')
# Set up adversarial attack.
adv_model = PyTorchModel(model, (0, 1),
pretrained_config.data.class_count,
cuda=config.cuda)
criterion = Misclassification()
attack = getattr(foolbox.attacks, config.name)(adv_model, criterion)
# Get data.
pretrained_config.cuda = config.cuda
pretrained_config.optim.batch_size = config.data.batch_size
data = load_data(pretrained_config)
# print('Test Accuracy:{}'.format(loader_accuracy(model, data['test'])))
n_examples = config['num_examples']
n_batches = int(
math.ceil((n_examples * 1.0) / pretrained_config.optim.batch_size))
# Save the results of the computations in the following variable.
adv_ex = torch.Tensor()
adv_targets = torch.LongTensor()
adv_mse = torch.Tensor()
adv_inf = torch.Tensor()
success = torch.Tensor()
# Set up distance for the adversarial attack.
distance_name = config.get('distance')
distance = getattr(foolbox.distances, distance_name) if distance_name is not None \
else foolbox.distances.MeanSquaredDistance
# Perform the attack.
for sample in tqdm(islice(data['validation'], n_batches), total=n_batches):
x = sample[0]
y = sample[1].type(torch.LongTensor)
x = to_cuda(x, cuda=config.cuda)
adv, adv_t, batch_success, batch_adv_mse, batch_adv_inf = batch_attack(
attack, adv_model, criterion, x,
y.cpu().numpy(), config['attack_kwargs'], distance)
adv_ex = torch.cat([adv_ex, adv], 0)
adv_targets = torch.cat([adv_targets, adv_t], 0)
success = torch.cat([success, batch_success], 0)
adv_mse = torch.cat([adv_mse, batch_adv_mse], 0)
adv_inf = torch.cat([adv_inf, batch_adv_inf], 0)
# evaluate_adv_grad_norms(model, adv_ex, adv_targets, config.cuda)
# Summarize the results.
results = {
"success_rate":
success.mean().item(),
"defense_rate":
1 - success.mean().item(),
"mean_mse": ((adv_mse * success).sum() / success.sum()).item(),
"mean_inf": ((adv_inf * success).sum() / success.sum()).item(),
"mse_quartiles":
list(
np.percentile(adv_mse[success == 1.0].numpy(),
[0, 25, 50, 75, 100]))
}
results["median_mse"] = results["mse_quartiles"][2]
print("success rate: {}".format(results["success_rate"]))
print("defense rate: {}".format(results["defense_rate"]))
print("mean MSE for successful attacks: {}".format(results["mean_mse"]))
print("mean L_inf for successful attacks: {}".format(results["mean_inf"]))
print("MSE quartiles for successful attacks: {}".format(
results["mse_quartiles"]))
with open(os.path.join(config['output_root'], 'results.json'), 'w') as f:
json.dump(results, f, sort_keys=True, indent=4)
np.save(adv_example_filepath, adv_ex)
np.save(adv_targets_filepath, adv_targets)
print(
accuracy(model, to_cuda(adv_ex, cuda=config.cuda),
to_cuda(adv_targets, cuda=config.cuda)))
def visualize_saliency(config):
# Create the output directory.
output_root = config.output_root
if not os.path.isdir(output_root):
os.makedirs(output_root)
# Load a pretrained model.
pretrained_path = config.pretrained_path
model, pretrained_config = load_model_from_config(pretrained_path)
# Push model to GPU if available.
if config.cuda:
print('Using cuda: Yes')
model.cuda()
model.eval()
# Get data.
pretrained_config.data.cuda = config.cuda
pretrained_config.data.batch_size = config.data.batch_size
data = load_data(pretrained_config)
# Compute adversarial gradients and save their visualizations.
for i, (x, y) in enumerate(data['test']):
x = to_cuda(x, cuda=config.cuda)
# Save the input image.
save_path = os.path.join(output_root, 'x{}.png'.format(i))
save_image(x, save_path)
# Save the adversarial gradients.
for j in range(pretrained_config.data.class_count):
# Compute and save the adversarial gradients.
x_grad = get_saliency_map(model, x, j)
save_image(x_grad,
os.path.join(output_root,
'x_{}_grad_{}.png'.format(i, j)),
normalize=True,
scale_each=True)
break
# Produce joint image.
nrow = config.visualization.num_rows
x_sub = to_cuda(
torch.zeros(nrow,
*x.size()[1:]).copy_(x[:nrow]).detach(), config.cuda)
print("Size of visualization: ", x_sub.size(), "Maximum pixel value: ",
x_sub.max())
tensors = []
c = 0
for i, (x, y) in enumerate(data['test']):
for (k, t) in enumerate(y):
if t == c:
c += 1
tensors.append(x[k])
if len(tensors) == pretrained_config.data.class_count:
break
if len(tensors) == pretrained_config.data.class_count:
break
# Collect tensors from each class
x_sub = to_cuda(torch.stack(tensors, 0), cuda=config.cuda)
tensors = [x_sub]
for j in range(pretrained_config.data.class_count):
# Compute and visualize the adversarial gradients.
model.zero_grad()
x_grad = get_saliency_map(model, x_sub, j).clone().detach()
tensors.append(x_grad)
# Concatenate and visualize.
joint_tensor = torch.cat(tensors, dim=0)
save_image(joint_tensor,
os.path.join(output_root, 'x_joint.png'),
nrow=pretrained_config.data.class_count,
normalize=True,
colormap='seismic')
def generate_examples(model, config, pretrained_config, output_root):
adv_example_filepath = os.path.join(output_root, 'examples')
adv_targets_filepath = os.path.join(output_root, 'adv_targets')
targets_filepath = os.path.join(output_root, 'targets')
n_examples = config['num_examples']
n_batches = int(
math.ceil((n_examples * 1.0) / pretrained_config.optim.batch_size))
pretrained_config.cuda = pretrained_config.cuda
data = load_data(pretrained_config)
# Save the results of the computations in the following variable.
adv_ex = torch.Tensor()
adv_targets = torch.LongTensor()
true_targets = torch.LongTensor()
adv_mse = torch.Tensor()
adv_inf = torch.Tensor()
success = torch.Tensor()
margins = torch.Tensor()
# Perform the attack.
i = 0
for sample in tqdm(islice(data['validation'], n_batches), total=n_batches):
i += 1
model.zero_grad()
x = to_cuda(sample[0], cuda=pretrained_config.cuda)
y = to_cuda(sample[1].type(torch.LongTensor),
cuda=pretrained_config.cuda)
true_targets = torch.cat([true_targets, y.detach().cpu()], 0)
if config.fgs:
adv, adv_t, original_pred = manual_fgs(model,
x,
y,
config.eps,
clamp=False)
elif config.pgd:
adv, adv_t, original_pred = manual_pgd(model,
x,
y,
config.eps,
config.eps,
rand_start=False,
clamp=False)
adv_ex = torch.cat([adv_ex, adv.cpu().detach()], 0)
adv_targets = torch.cat([adv_targets, adv_t.cpu().detach()], 0)
# import pdb; pdb.set_trace()
original_top_2 = model(x).topk(2, 1)[0]
original_margin = original_top_2[:, 0] - original_top_2[:, 1]
margins = torch.cat([margins, original_margin.cpu().detach()], 0)
batch_success = ((original_pred == y) & (adv_t != y)).float()
success = torch.cat([success, batch_success.cpu().detach()], 0)
adv_mse = torch.cat([
adv_mse, ((adv.view(adv.size(0), -1) - x.view(adv.size(0), -1))**
2).mean(-1).cpu().detach()
], 0)
adv_inf = torch.cat([
adv_inf, (adv.view(adv.size(0), -1) -
x.view(adv.size(0), -1)).abs().max(-1)[0].cpu().detach()
], 0)
total_accuracy = accuracy(model, to_cuda(adv_ex, cuda=config.cuda),
to_cuda(true_targets, cuda=config.cuda)).item()
# Summarize the results.
results = {
"eps": config.eps,
"success_rate": success.mean().item(),
"defense_rate": 1 - success.mean().item(),
"total_acc": total_accuracy,
"all_margins_mean": margins.mean().item(),
"successful_margins": ((margins * success).sum() / success.sum()).item(),
"mean_mse": ((adv_mse * success).sum() / success.sum()).item(),
"mean_inf": ((adv_inf * success).sum() / success.sum()).item(),
"mse_quartiles": list(np.percentile(adv_mse[success == 1.0].numpy(), [0, 25, 50, 75, 100])) \
if len(adv_mse[success == 1.0]) > 0 else [0, 0, 0, 0, 0]
}
results["median_mse"] = results["mse_quartiles"][2]
print("success rate: {}".format(results["success_rate"]))
print("defense rate: {}".format(results["defense_rate"]))
print("total accuracy: {}".format(results["total_acc"]))
print("Avg Margin: {}".format(results['all_margins_mean']))
print("Avg Success Margin: {}".format(results['successful_margins']))
print("mean MSE for successful attacks: {}".format(results["mean_mse"]))
print("mean L_inf for successful attacks: {}".format(results["mean_inf"]))
print("MSE quartiles for successful attacks: {}".format(
results["mse_quartiles"]))
with open(os.path.join(output_root, 'results.json'), 'w') as f:
json.dump(results, f, sort_keys=True, indent=4)
np.save(adv_example_filepath, adv_ex)
np.save(adv_targets_filepath, adv_targets)
np.save(targets_filepath, true_targets)
return adv_ex