def get_images(self, n_samples=10, fraction_to_dismiss=0.1, weighted=False, sample_sigma=1): if n_samples not in self.images.keys(): self.init_grid(n_samples, fraction_to_dismiss=fraction_to_dismiss, sample_sigma=sample_sigma) self.images[n_samples] = np.empty( (self.n_classes, n_samples, self.nc, self.nx, self.ny)) for c, AE in enumerate(self.AEs[:self.n_classes]): AE.eval() images = F.sigmoid(AE.Decoder.forward( self.l_v[n_samples])).cpu().data.numpy() if weighted: images = images[:, 0, None] self.images[n_samples][c, ...] = images self.l_v[n_samples] assert n_samples not in self.th_images self.th_images[n_samples] = tensor(self.images[n_samples]).type( torch.FloatTensor).to(u.dev()) print('done creating samples') return self.images[n_samples]
def init_models(): strides = [1, 2, 2, 1] latent_act_fct = u.LinearActFct kernelE = [5, 4, 3, 5] feat_mapsE = [32, 32, 64, nd] encoder = { 'feat_maps': feat_mapsE, 'kernels': kernelE, 'strides': strides } kernelD = [4, 5, 5, 4] feat_mapsD = [32, 16, 16, 1] decoder = { 'feat_maps': feat_mapsD, 'kernels': kernelD, 'strides': strides } AEs = [] for i in range(n_classes): AE = nets.VariationalAutoEncoder(encoder, decoder, latent_act_fct=latent_act_fct) AE.eval() AE.to(u.dev()) AEs.append(AE) return AEs
def evalAdvAttack(model=None, test_loader=None): print("Evaluating single model results on adv data") total = 0 correct = 0 model.eval() for xs, ys in test_loader: if torch.cuda.is_available(): xs, ys = xs.cuda(), ys.cuda() # pytorch fast gradient method model.eval() fmodel = foolbox.models.PyTorchModel( fgsm_model, # return logits in shape (bs, n_classes) bounds=(0., 1.), # num_classes=10, device=u.dev()) attack = fa.LinfProjectedGradientDescentAttack( rel_stepsize=0.01 / 0.3, steps=100, random_start=True, ) xs, _, success = attack(fmodel, xs, ys, epsilons=[0.3]) xs, ys = Variable(xs[0]), Variable(ys) preds1 = model(xs) preds_np1 = preds1.cpu().detach().numpy() finalPred = np.argmax(preds_np1, axis=1) correct += (finalPred == ys.cpu().detach().numpy()).sum() total += test_loader.batch_size acc = float(correct) / total print('Adv accuracy: %.2f%%' % (acc * 100))
def forward(self, input_batch, return_more=True): assert len(input_batch.size()) == 4 assert input_batch.size()[-1] == self.samples.size()[-1] assert input_batch.size()[-2] == self.samples.size()[-2] assert input_batch.size()[-3] == self.samples.size()[-3] bs = input_batch.shape[0] input_batch = input_batch[:, None, ...].to(u.dev()) # (bs, 1, nch, x, y) def calc_dist(input_batch): dists = u.L2(self.samples, input_batch, axes=[2, 3, 4]) l2, best_ind_classes = torch.min(dists, 1) return l2, best_ind_classes l2s, best_ind_classes = u.auto_batch(self.max_bs, calc_dist, input_batch) # boring bookkeeping pred = self.get_classes(bs, input_batch, best_ind_classes) imgs = self.samples[0, best_ind_classes] # print(pred, imgs, l2s)\ if return_more: return pred, imgs, l2s else: return pred
def get_transfer_model(load_path=DEFAULT_PATH): # new arch shape = (1, 1, 28, 28) strides = [1, 2, 2, 1] kernelE = [5, 4, 3, 5] feat_mapsE = [32, 32, 64, 10] # (32, 32, 16, 2) model = nets.NN(feat_mapsE, shape[1:], kernels=kernelE, strides=strides) model.load_state_dict( torch.load(load_path + 'transfer_cnn.net', map_location=str(u.dev()))) model.to(u.dev()) if load_path is not None: model.load_state_dict(torch.load(load_path, map_location=str(u.dev()))) model.eval() model.code_base = 'pytorch' return model
def get_NearestNeighbor(): n_classes = 10 mnist_train = datasets.MNIST('./../data', train=True, download=True, transform=transforms.Compose( [transforms.ToTensor()])) NN = NearestNeighbor( mnist_train.train_data[:, None, ...].type(torch.float32).to(u.dev()) / 255, mnist_train.train_labels.to(u.dev()), n_classes=n_classes) print('model initialized') NN.eval() # does nothing but avoids warnings NN.code_base = 'pytorch' NN.has_grad = False return NN
def generate_data(self, model, loader, cuda=False): for i, (test_data, test_label) in enumerate(loader): if i == int(np.ceil(400 / loader.batch_size)): break x = test_data yt = test_label x = x.to(u.dev()) model.forward(x) latent = model.latent.cpu().data.numpy().swapaxes(0, 1).squeeze() self.xl += latent[0].tolist() self.yl += latent[1].tolist() self.cl += yt.data.numpy().tolist()
def gd_inference_b(l_v_best, x_inp, AEs, n_classes=10, clip=5, lr=0.01, n_iter=20, beta=1, dist_fct=loss_functions.squared_L2_loss): bs, n_ch, nx, ny = x_inp.shape with torch.enable_grad(): l_v_best = tensor(l_v_best.data.clone().to(u.dev()), requires_grad=True) opti = optim.Adam([l_v_best], lr=lr) for i in range(n_iter): ELBOs = [] all_recs = [] for j in range(n_classes): if i == n_iter - 1: l_v_best = l_v_best.detach( ) # no gradients in last run AEs[j].eval() rec = F.sigmoid(AEs[j].Decoder.forward(l_v_best[:, j])) ELBOs.append( loss_functions.ELBOs( rec, # (bs, n_ch, nx, ny) l_v_best[:, j], # (bs, n_latent, 1, 1) x_inp, # (bs, n_ch, nx, ny) beta=beta, dist_fct=dist_fct, auto_batch_size=None)) if i == n_iter - 1: all_recs.append(rec.view(bs, 1, n_ch, nx, ny).detach()) ELBOs = torch.cat(ELBOs, dim=1) if i < n_iter - 1: loss = ( torch.sum(ELBOs)) - 8. / 784. / 2 # historic reasons # backward opti.zero_grad() loss.backward() opti.step() l_v_best.data = u.clip_to_sphere(l_v_best.data, clip, channel_dim=2) else: opti.zero_grad() all_recs = torch.cat(all_recs, dim=1) return ELBOs.detach(), l_v_best.detach(), all_recs
def get_CNN(load_path=DEFAULT_PATH): load_path = join(DEFAULT_PATH, '../exp/mnist_cnn/nets/') # network shape = (1, 1, 28, 28) kernelE = [5, 4, 3, 5] strides = [1, 2, 2, 1] feat_mapsE = [20, 70, 256, 10] # (32, 32, 16, 2) model = nets.NN(feat_mapsE, shape[1:], kernels=kernelE, strides=strides) # load net print('path', load_path + '/vanilla_cnn.net') model.load_state_dict( torch.load(load_path + '/vanilla_cnn.net', map_location=str(u.dev()))) print('model loaded') NN = CNN(model) NN.eval() NN.to(u.dev()) NN.code_base = 'pytorch' return NN
def init_grid(self, n_samples, fraction_to_dismiss=None, sample_sigma=None): n_grid = self.n_samples_to_n_grid(n_samples) print('init new grid', n_samples, n_grid) limit = 0.99 if self.limit is not None: limit = self.limit grids = [(np.linspace(-limit, limit, n_grid)) for i in range(self.nd)] xys = np.array(np.meshgrid(*grids)) xys = np.moveaxis(xys, 0, -1).reshape(n_grid**self.nd, self.nd) self.samples[n_samples] = xys self.l_v[n_samples] = \ torch.from_numpy(xys[:, :, None, None].astype(np.float32)).to(u.dev())
def init_grid(self, n_samples, fraction_to_dismiss=0.1, mus=None, sample_sigma=1): if mus is None: mus = np.zeros(self.nd) samples = get_gaussian_samples(n_samples, self.nd, mus, fraction_to_dismiss=fraction_to_dismiss, sample_sigma=sample_sigma) self.samples[n_samples] = samples self.l_v[n_samples] = \ torch.from_numpy(samples[:, :, None, None].astype( np.float32)).to(u.dev())
def __call__(self, x, l, n_coarse_steps=3, n_ft_steps=10): x, l = u.n2t(x), u.n2t(l) x, l = x.to(u.dev()), l.to(u.dev()) bs = x.shape[0] best_other = 0 best_advs = [{ 'original_label': -1, 'adversarial_label': None, 'distance': np.inf, 'img': torch.zeros(x.shape[1:]).to(u.dev()) } for _ in range(bs)] coarse_steps = torch.zeros(bs).to(u.dev()) n_adv_found = 0 for i, coarse_step in enumerate( torch.linspace(0, 1., n_coarse_steps).to(u.dev())): current_adv = (1 - coarse_step) * x + coarse_step * best_other best_other, current_label = self.get_best_prototypes( current_adv, l) for j, (current_adv_i, pred_l_i, l_i) in enumerate(zip(current_adv, current_label, l)): if best_advs[j]['original_label'] == -1 and pred_l_i != l_i: self.update_adv(best_advs[j], current_adv_i, pred_l_i, l_i, x[j]) coarse_steps[i] = coarse_step n_adv_found += 1 if n_adv_found == bs: break best_advs_imgs = torch.cat([a['img'][None] for a in best_advs]) coarse_steps_old = coarse_steps[:, None, None, None] # binary search best_advs_imgs_old = best_advs_imgs.clone() sign, step = -torch.ones(bs, 1, 1, 1).to(u.dev()), 0.5 for i in range(n_ft_steps): coarse_steps = coarse_steps_old + step * sign current_adv = ( 1 - coarse_steps) * x + coarse_steps * best_advs_imgs_old _, current_label = self.get_best_prototypes(current_adv, l) for j, (pred_l_i, l_i) in enumerate(zip(current_label, l)): if pred_l_i == l_i: sign[j] = 1 else: self.update_adv(best_advs[j], current_adv[j], pred_l_i, l_i, x[j]) sign[j] = -1 step /= 2 return best_advs
def __init__(self, AEs, n_samples, n_iter, beta, GM, fraction_to_dismiss=0.1, clip=5, lr=0.05): super().__init__(AEs, n_samples, n_iter, beta, GM, fraction_to_dismiss=fraction_to_dismiss, clip=clip, lr=lr) self.name_check = 'ABS' self.rescale_b = True self.discriminative_scalings = torch.tensor( [1., 0.96, 1.001, 1.06, 0.98, 0.96, 1.03, 1., 1., 1.]).to(u.dev())
# model = mz.get_NearestNeighbor() # Nearest Neighbor, "nearest L2 dist to each class"=logits # model = mz.get_madry() # Robust network from Madry et al. in tf # - # code is agnostic of pytorch/ tensorflow model --> foolbox model if model.code_base == 'tensorflow': fmodel = foolbox.models.TensorFlowModel(model.x_input, model.pre_softmax, (0., 1.), channel_axis=3) elif model.code_base == 'pytorch': model.eval() fmodel = foolbox.models.PyTorchModel( model, # return logits in shape (bs, n_classes) bounds=(0., 1.), num_classes=10, device=u.dev()) else: print('not implemented') # test model b, l = u.get_batch(bs=1000) # returns random batch as np.array pred_label = np.argmax(fmodel.batch_predictions(b), axis=1) print('score', float(np.sum(pred_label == l)) / b.shape[0]) # # Decision based attacks # Note that this is only demo code. All experiments were optimized to our compute architecture. # + att = fa.PointwiseAttack(fmodel) metric = foolbox.distances.L0 criterion = foolbox.criteria.Misclassification()
def get_classes(self, bs, input_batch, best_ind_classes): pred = torch.zeros(bs, self.n_classes).to(u.dev()) pred[range(bs), self.classes[best_ind_classes]] = 1. return pred
def get_VAE(n_samples=8000, n_iter=50, beta=1, clip=5, fraction_to_dismiss=0.1, lr=0.05, load=True, binary=False, load_path=DEFAULT_PATH): """Creates the ABS model. If binary is True, returns the full ABS model including binarization and scalar, otherwise returns the base ABS model without binarization and without scalar.""" load_path = join(DEFAULT_PATH, '../exp/VAE_swarm_MSE/nets/') print('ABS model') n_classes = 10 nd = 8 nx, ny = 28, 28 def init_models(): strides = [1, 2, 2, 1] latent_act_fct = u.LinearActFct kernelE = [5, 4, 3, 5] feat_mapsE = [32, 32, 64, nd] encoder = { 'feat_maps': feat_mapsE, 'kernels': kernelE, 'strides': strides } kernelD = [4, 5, 5, 4] feat_mapsD = [32, 16, 16, 1] decoder = { 'feat_maps': feat_mapsD, 'kernels': kernelD, 'strides': strides } AEs = [] for i in range(n_classes): AE = nets.VariationalAutoEncoder(encoder, decoder, latent_act_fct=latent_act_fct) AE.eval() AE.to(u.dev()) AEs.append(AE) return AEs AEs = init_models() if load: for i in range(n_classes): path = load_path + f'/ABS_{i}.net' AEs[i].iters = 29000 AEs[i].load_state_dict(torch.load(path, map_location=str(u.dev()))) print('model loaded') GM = sampling.GaussianSamples(AEs, nd, n_classes, nx=nx, ny=ny) if binary: model = ELBOVAE_binary else: model = ELBOVAE model = model(AEs, n_samples, n_iter, beta, GM, fraction_to_dismiss, clip, lr=lr) model.eval() model.code_base = 'pytorch' model.has_grad = False return model