def sample_fmodel(hypernet, arch):
w_batch = utils.sample_hypernet(hypernet)
rand = np.random.randint(32)
sample_w = (w_batch[0][rand], w_batch[1][rand], w_batch[2][rand])
model = utils.weights_to_clf(sample_w, arch, args.stat['layer_names'])
model.eval()
fmodel = attacks.load_model(model)
return model, fmodel
def run_adv_model(args, models):
fmodels = [attacks.load_model(model) for model in models]
criterion = Misclassification()
fgs = foolbox.attacks.HyperBIM(fmodels[0])
_, test_loader = datagen.load_mnist(args)
adv, y, inter = [], [], []
acc, _accs = [], []
total_adv, total_correct = 0, 0
missed = 0
for eps in [0.01, 0.03, 0.08, 0.1, .2, .3, 1]:
total_adv = 0
_accs, _vars, _stds = [], [], []
pred_labels = []
for data, target in test_loader:
data, target = data.cuda(), target.cuda()
adv_batch, target_batch, _ = attack_batch_ensemble(data, target, eps, fgs, fmodels)
if adv_batch is None:
continue
n_adv = 0.
acc, pred_labels = [], []
output = ensemble_prediction(models, adv_batch)
for i in range(5):
pred = output[i].data.max(1, keepdim=True)[1]
correct = pred.eq(target_batch.data.view_as(pred)).long().cpu().sum()
n_adv += len(target_batch)-correct.item()
pred_labels.append(pred.view(pred.numel()))
ens_pred = output.mean(0).data.max(1, keepdim=True)[1]
ens_correct = ens_pred.eq(target_batch.data.view_as(pred)).long().cpu().sum()
total_adv += len(target_batch) - ens_correct.item()
p_labels = torch.stack(pred_labels).float().transpose(0, 1)
_vars.append(p_labels.var(1).mean())
_stds.append(p_labels.std(1).mean())
acc, adv, y = [], [], []
print ('Eps: {}, Adv: {}/{}, var: {}, std: {}'.format(eps,
total_adv, len(test_loader.dataset), torch.tensor(_vars).mean(),
torch.tensor(_stds).mean()))
def run_adv_model(args, models):
for model in models:
model.eval()
model = FusedNet(models)
fmodel = attacks.load_model(model)
criterion = Misclassification()
fgs = foolbox.attacks.BIM(fmodel)
_, test_loader = datagen.load_mnist(args)
for eps in [.08, .1, .3, .5, 1.0]:
total_adv = 0
_soft, _logs, _vars = [], [], []
for idx, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
adv_batch, target_batch, _ = sample_adv_batch(
data, target, fmodel, eps, fgs)
# get intial prediction of ensemble, sure
output = model(adv_batch)
pred = output.data.max(1, keepdim=True)[1]
correct = pred.eq(
target_batch.data.view_as(pred)).long().cpu().sum()
n_adv = len(target_batch) - correct.item()
# set up to sample from individual models
soft_out, pred_out, logits = [], [], []
for i in range(len(models)):
output = models[i](adv_batch)
soft_out.append(F.softmax(output, dim=1))
pred_out.append(output.data.max(1, keepdim=True)[1])
logits.append(output)
softs = torch.stack(soft_out)
preds = torch.stack(pred_out).float()
logs = torch.stack(logits)
# Measure variance of individual logits across models.
# HyperGAN ensemble has lower variance across 10 class predictions
# But a single logit has high variance acorss models
units_softmax = softs.var(
0).mean().item() # var across models across images
units_logprob = logs.var(0).mean().item()
ensemble_var = softs.mean(0).var(1).mean().item()
""" Core Debug """
# print ('softmax var: ', units_softmax)
# print ('logprob var: ', units_logprob)
# print ('ensemble var: ', ensemble_var)
# build lists
_soft.append(units_softmax)
_logs.append(units_logprob)
_vars.append(ensemble_var)
total_adv += n_adv
if idx % 200 == 0:
print(
'Eps: {}, Iter: {}, Log var: {}, Softmax var: {}, Ens var: {}'
.format(eps, idx,
torch.tensor(_logs).mean(),
torch.tensor(_soft).mean(),
torch.tensor(_vars).mean()))
print(
'[Final] - Eps: {}, Adv: {}/{}, Log var: {}, Softmax var: {}, Ens var: {}'
.format(eps, total_adv, len(test_loader.dataset),
torch.tensor(_logs).mean(),
torch.tensor(_soft).mean(),
torch.tensor(_vars).mean()))
def run_adv_model(args, models):
for model in models:
model.eval()
print ('models loaded')
#models = models[:5]
model = FusedNet(models)
print ('made fusednet')
fmodel = attacks.load_model(model)
criterion = Misclassification()
fgs = foolbox.attacks.FGSM(fmodel)
print ('created attack')
_, test_loader = datagen.load_mnist(args)
print ('loaded dataset')
for eps in [0.01, 0.03, 0.08, .1, .3, .5, 1.0]:
total_adv = 0
_soft, _logs, _vars, _ents, _lsoft = [], [], [], [], []
_soft_adv, _logs_adv, _vars_adv, _ents_adv, _lsoft_adv = [], [], [], [], []
_kl_real, _kl_adv = [], []
for idx, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
adv_batch, target_batch, _ = sample_adv_batch(data, target, fmodel, eps, fgs)
if adv_batch is None:
continue
# get intial prediction of ensemble, sure
output = model(adv_batch)
pred = output.data.max(1, keepdim=True)[1]
correct = pred.eq(target_batch.data.view_as(pred)).long().cpu().sum()
n_adv = len(target_batch)-correct.item()
# set up to sample from individual models
soft_out, pred_out, logits, lsoft_out = [], [], [], []
soft_out_adv, pred_out_adv, logits_adv, lsoft_out_adv = [], [], [], []
for i in range(len(models)):
output = models[i](data)
soft_out.append(F.softmax(output, dim=1))
pred_out.append(output.data.max(1, keepdim=True)[1])
lsoft_out.append(F.log_softmax(output, dim=1))
logits.append(output)
output = models[i](adv_batch)
soft_out_adv.append(F.softmax(output, dim=1))
lsoft_out_adv.append(F.log_softmax(output, dim=1))
softs = torch.stack(soft_out).float()
preds = torch.stack(pred_out).float()
lsoft = torch.stack(lsoft_out).float()
logs = torch.stack(logits).float()
softs_adv = torch.stack(soft_out_adv).float()
lsoft_adv = torch.stack(lsoft_out_adv).float()
# Measure variance of individual logits across models.
# HyperGAN ensemble has lower variance across 10 class predictions
# But a single logit has high variance acorss models
units_softmax = softs.var(0).mean().item() # var across models across images
units_logprob = logs.var(0).mean().item()
ensemble_var = softs.mean(0).var(1).mean().item()
ent = float(entropy(softs.mean(0).transpose(0, 1).detach()).mean())
#units_logprob = logs.var(0).mean().item()
ent_adv = float(entropy(softs_adv.mean(0).transpose(0, 1).detach()).mean())
units_softmax_adv = softs_adv.var(0).mean().item() # var across models - images
ensemble_var_adv = softs_adv.mean(0).var(1).mean().item()
""" Core Debug """
# print ('softmax var: ', units_softmax)
# print ('logprob var: ', units_logprob)
# print ('ensemble var: ', ensemble_var)
# build lists
_soft.append(units_softmax)
_soft_adv.append(units_softmax_adv)
_logs.append(units_logprob)
# log variance
_ents.append(ent)
_ents_adv.append(ent_adv)
total_adv += n_adv
if idx > 10:
print ('REAL: ent: {}'.format(torch.tensor(_ents).mean()))
print ('ADV Eps: {}, ent: {}'.format(
eps, torch.tensor(_ents_adv).mean()))
break;
soft_out = []
pred_out = []
logits = []
paths = [
'mnist_model_small2_0.pt', 'mnist_model_small2_1.pt',
'mnist_model_small2_2.pt', 'mnist_model_small2_3.pt',
'mnist_model_small2_4.pt', 'mnist_model_small2_5.pt',
'mnist_model_small2_6.pt', 'mnist_model_small2_7.pt',
'mnist_model_small2_8.pt', 'mnist_model_small2_9.pt'
]
models = []
for path in paths:
model = adv.get_network(args)
model.load_state_dict(torch.load(path))
models.append(model.eval())
model = adv.FusedNet(models)
import attacks
fmodel = attacks.load_model(model)
fgs = foolbox.attacks.BIM(fmodel, criterion)
adv_batch, target_batch, _ = adv.sample_adv_batch(stack_data[0], stack_y[0],
fmodel, 0.08, fgs)
soft_out = []
pred_out = []
logits = []
for i in range(5):
output = models[i](adv_batch)
soft_out.append(F.softmax(output, dim=1))
pred_out.append(output.data.max(1, keepdim=True)[1])
logits.append(output)
