def sample_fmodel(hypernet, arch): w_batch = utils.sample_hypernet(hypernet) rand = np.random.randint(32) sample_w = (w_batch[0][rand], w_batch[1][rand], w_batch[2][rand]) model = utils.weights_to_clf(sample_w, arch, args.stat['layer_names']) model.eval() fmodel = attacks.load_model(model) return model, fmodel
def run_adv_model(args, models): fmodels = [attacks.load_model(model) for model in models] criterion = Misclassification() fgs = foolbox.attacks.HyperBIM(fmodels[0]) _, test_loader = datagen.load_mnist(args) adv, y, inter = [], [], [] acc, _accs = [], [] total_adv, total_correct = 0, 0 missed = 0 for eps in [0.01, 0.03, 0.08, 0.1, .2, .3, 1]: total_adv = 0 _accs, _vars, _stds = [], [], [] pred_labels = [] for data, target in test_loader: data, target = data.cuda(), target.cuda() adv_batch, target_batch, _ = attack_batch_ensemble(data, target, eps, fgs, fmodels) if adv_batch is None: continue n_adv = 0. acc, pred_labels = [], [] output = ensemble_prediction(models, adv_batch) for i in range(5): pred = output[i].data.max(1, keepdim=True)[1] correct = pred.eq(target_batch.data.view_as(pred)).long().cpu().sum() n_adv += len(target_batch)-correct.item() pred_labels.append(pred.view(pred.numel())) ens_pred = output.mean(0).data.max(1, keepdim=True)[1] ens_correct = ens_pred.eq(target_batch.data.view_as(pred)).long().cpu().sum() total_adv += len(target_batch) - ens_correct.item() p_labels = torch.stack(pred_labels).float().transpose(0, 1) _vars.append(p_labels.var(1).mean()) _stds.append(p_labels.std(1).mean()) acc, adv, y = [], [], [] print ('Eps: {}, Adv: {}/{}, var: {}, std: {}'.format(eps, total_adv, len(test_loader.dataset), torch.tensor(_vars).mean(), torch.tensor(_stds).mean()))
def run_adv_model(args, models): for model in models: model.eval() model = FusedNet(models) fmodel = attacks.load_model(model) criterion = Misclassification() fgs = foolbox.attacks.BIM(fmodel) _, test_loader = datagen.load_mnist(args) for eps in [.08, .1, .3, .5, 1.0]: total_adv = 0 _soft, _logs, _vars = [], [], [] for idx, (data, target) in enumerate(test_loader): data, target = data.cuda(), target.cuda() adv_batch, target_batch, _ = sample_adv_batch( data, target, fmodel, eps, fgs) # get intial prediction of ensemble, sure output = model(adv_batch) pred = output.data.max(1, keepdim=True)[1] correct = pred.eq( target_batch.data.view_as(pred)).long().cpu().sum() n_adv = len(target_batch) - correct.item() # set up to sample from individual models soft_out, pred_out, logits = [], [], [] for i in range(len(models)): output = models[i](adv_batch) soft_out.append(F.softmax(output, dim=1)) pred_out.append(output.data.max(1, keepdim=True)[1]) logits.append(output) softs = torch.stack(soft_out) preds = torch.stack(pred_out).float() logs = torch.stack(logits) # Measure variance of individual logits across models. # HyperGAN ensemble has lower variance across 10 class predictions # But a single logit has high variance acorss models units_softmax = softs.var( 0).mean().item() # var across models across images units_logprob = logs.var(0).mean().item() ensemble_var = softs.mean(0).var(1).mean().item() """ Core Debug """ # print ('softmax var: ', units_softmax) # print ('logprob var: ', units_logprob) # print ('ensemble var: ', ensemble_var) # build lists _soft.append(units_softmax) _logs.append(units_logprob) _vars.append(ensemble_var) total_adv += n_adv if idx % 200 == 0: print( 'Eps: {}, Iter: {}, Log var: {}, Softmax var: {}, Ens var: {}' .format(eps, idx, torch.tensor(_logs).mean(), torch.tensor(_soft).mean(), torch.tensor(_vars).mean())) print( '[Final] - Eps: {}, Adv: {}/{}, Log var: {}, Softmax var: {}, Ens var: {}' .format(eps, total_adv, len(test_loader.dataset), torch.tensor(_logs).mean(), torch.tensor(_soft).mean(), torch.tensor(_vars).mean()))
def run_adv_model(args, models): for model in models: model.eval() print ('models loaded') #models = models[:5] model = FusedNet(models) print ('made fusednet') fmodel = attacks.load_model(model) criterion = Misclassification() fgs = foolbox.attacks.FGSM(fmodel) print ('created attack') _, test_loader = datagen.load_mnist(args) print ('loaded dataset') for eps in [0.01, 0.03, 0.08, .1, .3, .5, 1.0]: total_adv = 0 _soft, _logs, _vars, _ents, _lsoft = [], [], [], [], [] _soft_adv, _logs_adv, _vars_adv, _ents_adv, _lsoft_adv = [], [], [], [], [] _kl_real, _kl_adv = [], [] for idx, (data, target) in enumerate(test_loader): data, target = data.cuda(), target.cuda() adv_batch, target_batch, _ = sample_adv_batch(data, target, fmodel, eps, fgs) if adv_batch is None: continue # get intial prediction of ensemble, sure output = model(adv_batch) pred = output.data.max(1, keepdim=True)[1] correct = pred.eq(target_batch.data.view_as(pred)).long().cpu().sum() n_adv = len(target_batch)-correct.item() # set up to sample from individual models soft_out, pred_out, logits, lsoft_out = [], [], [], [] soft_out_adv, pred_out_adv, logits_adv, lsoft_out_adv = [], [], [], [] for i in range(len(models)): output = models[i](data) soft_out.append(F.softmax(output, dim=1)) pred_out.append(output.data.max(1, keepdim=True)[1]) lsoft_out.append(F.log_softmax(output, dim=1)) logits.append(output) output = models[i](adv_batch) soft_out_adv.append(F.softmax(output, dim=1)) lsoft_out_adv.append(F.log_softmax(output, dim=1)) softs = torch.stack(soft_out).float() preds = torch.stack(pred_out).float() lsoft = torch.stack(lsoft_out).float() logs = torch.stack(logits).float() softs_adv = torch.stack(soft_out_adv).float() lsoft_adv = torch.stack(lsoft_out_adv).float() # Measure variance of individual logits across models. # HyperGAN ensemble has lower variance across 10 class predictions # But a single logit has high variance acorss models units_softmax = softs.var(0).mean().item() # var across models across images units_logprob = logs.var(0).mean().item() ensemble_var = softs.mean(0).var(1).mean().item() ent = float(entropy(softs.mean(0).transpose(0, 1).detach()).mean()) #units_logprob = logs.var(0).mean().item() ent_adv = float(entropy(softs_adv.mean(0).transpose(0, 1).detach()).mean()) units_softmax_adv = softs_adv.var(0).mean().item() # var across models - images ensemble_var_adv = softs_adv.mean(0).var(1).mean().item() """ Core Debug """ # print ('softmax var: ', units_softmax) # print ('logprob var: ', units_logprob) # print ('ensemble var: ', ensemble_var) # build lists _soft.append(units_softmax) _soft_adv.append(units_softmax_adv) _logs.append(units_logprob) # log variance _ents.append(ent) _ents_adv.append(ent_adv) total_adv += n_adv if idx > 10: print ('REAL: ent: {}'.format(torch.tensor(_ents).mean())) print ('ADV Eps: {}, ent: {}'.format( eps, torch.tensor(_ents_adv).mean())) break;
soft_out = [] pred_out = [] logits = [] paths = [ 'mnist_model_small2_0.pt', 'mnist_model_small2_1.pt', 'mnist_model_small2_2.pt', 'mnist_model_small2_3.pt', 'mnist_model_small2_4.pt', 'mnist_model_small2_5.pt', 'mnist_model_small2_6.pt', 'mnist_model_small2_7.pt', 'mnist_model_small2_8.pt', 'mnist_model_small2_9.pt' ] models = [] for path in paths: model = adv.get_network(args) model.load_state_dict(torch.load(path)) models.append(model.eval()) model = adv.FusedNet(models) import attacks fmodel = attacks.load_model(model) fgs = foolbox.attacks.BIM(fmodel, criterion) adv_batch, target_batch, _ = adv.sample_adv_batch(stack_data[0], stack_y[0], fmodel, 0.08, fgs) soft_out = [] pred_out = [] logits = [] for i in range(5): output = models[i](adv_batch) soft_out.append(F.softmax(output, dim=1)) pred_out.append(output.data.max(1, keepdim=True)[1]) logits.append(output)