def get_images(self,
n_samples=10,
fraction_to_dismiss=0.1,
weighted=False,
sample_sigma=1):
if n_samples not in self.images.keys():
self.init_grid(n_samples,
fraction_to_dismiss=fraction_to_dismiss,
sample_sigma=sample_sigma)
self.images[n_samples] = np.empty(
(self.n_classes, n_samples, self.nc, self.nx, self.ny))
for c, AE in enumerate(self.AEs[:self.n_classes]):
AE.eval()
images = F.sigmoid(AE.Decoder.forward(
self.l_v[n_samples])).cpu().data.numpy()
if weighted:
images = images[:, 0, None]
self.images[n_samples][c, ...] = images
self.l_v[n_samples]
assert n_samples not in self.th_images
self.th_images[n_samples] = tensor(self.images[n_samples]).type(
torch.FloatTensor).to(u.dev())
print('done creating samples')
return self.images[n_samples]
def init_models():
strides = [1, 2, 2, 1]
latent_act_fct = u.LinearActFct
kernelE = [5, 4, 3, 5]
feat_mapsE = [32, 32, 64, nd]
encoder = {
'feat_maps': feat_mapsE,
'kernels': kernelE,
'strides': strides
}
kernelD = [4, 5, 5, 4]
feat_mapsD = [32, 16, 16, 1]
decoder = {
'feat_maps': feat_mapsD,
'kernels': kernelD,
'strides': strides
}
AEs = []
for i in range(n_classes):
AE = nets.VariationalAutoEncoder(encoder,
decoder,
latent_act_fct=latent_act_fct)
AE.eval()
AE.to(u.dev())
AEs.append(AE)
return AEs
def evalAdvAttack(model=None, test_loader=None):
print("Evaluating single model results on adv data")
total = 0
correct = 0
model.eval()
for xs, ys in test_loader:
if torch.cuda.is_available():
xs, ys = xs.cuda(), ys.cuda()
# pytorch fast gradient method
model.eval()
fmodel = foolbox.models.PyTorchModel(
fgsm_model, # return logits in shape (bs, n_classes)
bounds=(0., 1.), # num_classes=10,
device=u.dev())
attack = fa.LinfProjectedGradientDescentAttack(
rel_stepsize=0.01 / 0.3,
steps=100,
random_start=True,
)
xs, _, success = attack(fmodel, xs, ys, epsilons=[0.3])
xs, ys = Variable(xs[0]), Variable(ys)
preds1 = model(xs)
preds_np1 = preds1.cpu().detach().numpy()
finalPred = np.argmax(preds_np1, axis=1)
correct += (finalPred == ys.cpu().detach().numpy()).sum()
total += test_loader.batch_size
acc = float(correct) / total
print('Adv accuracy: %.2f%%' % (acc * 100))
def forward(self, input_batch, return_more=True):
assert len(input_batch.size()) == 4
assert input_batch.size()[-1] == self.samples.size()[-1]
assert input_batch.size()[-2] == self.samples.size()[-2]
assert input_batch.size()[-3] == self.samples.size()[-3]
bs = input_batch.shape[0]
input_batch = input_batch[:, None,
...].to(u.dev()) # (bs, 1, nch, x, y)
def calc_dist(input_batch):
dists = u.L2(self.samples, input_batch, axes=[2, 3, 4])
l2, best_ind_classes = torch.min(dists, 1)
return l2, best_ind_classes
l2s, best_ind_classes = u.auto_batch(self.max_bs, calc_dist,
input_batch)
# boring bookkeeping
pred = self.get_classes(bs, input_batch, best_ind_classes)
imgs = self.samples[0, best_ind_classes]
# print(pred, imgs, l2s)\
if return_more:
return pred, imgs, l2s
else:
return pred
def get_transfer_model(load_path=DEFAULT_PATH):
# new arch
shape = (1, 1, 28, 28)
strides = [1, 2, 2, 1]
kernelE = [5, 4, 3, 5]
feat_mapsE = [32, 32, 64, 10] # (32, 32, 16, 2)
model = nets.NN(feat_mapsE, shape[1:], kernels=kernelE, strides=strides)
model.load_state_dict(
torch.load(load_path + 'transfer_cnn.net', map_location=str(u.dev())))
model.to(u.dev())
if load_path is not None:
model.load_state_dict(torch.load(load_path, map_location=str(u.dev())))
model.eval()
model.code_base = 'pytorch'
return model
def get_NearestNeighbor():
n_classes = 10
mnist_train = datasets.MNIST('./../data',
train=True,
download=True,
transform=transforms.Compose(
[transforms.ToTensor()]))
NN = NearestNeighbor(
mnist_train.train_data[:, None, ...].type(torch.float32).to(u.dev()) /
255,
mnist_train.train_labels.to(u.dev()),
n_classes=n_classes)
print('model initialized')
NN.eval() # does nothing but avoids warnings
NN.code_base = 'pytorch'
NN.has_grad = False
return NN
def generate_data(self, model, loader, cuda=False):
for i, (test_data, test_label) in enumerate(loader):
if i == int(np.ceil(400 / loader.batch_size)):
break
x = test_data
yt = test_label
x = x.to(u.dev())
model.forward(x)
latent = model.latent.cpu().data.numpy().swapaxes(0, 1).squeeze()
self.xl += latent[0].tolist()
self.yl += latent[1].tolist()
self.cl += yt.data.numpy().tolist()
def gd_inference_b(l_v_best,
x_inp,
AEs,
n_classes=10,
clip=5,
lr=0.01,
n_iter=20,
beta=1,
dist_fct=loss_functions.squared_L2_loss):
bs, n_ch, nx, ny = x_inp.shape
with torch.enable_grad():
l_v_best = tensor(l_v_best.data.clone().to(u.dev()),
requires_grad=True)
opti = optim.Adam([l_v_best], lr=lr)
for i in range(n_iter):
ELBOs = []
all_recs = []
for j in range(n_classes):
if i == n_iter - 1:
l_v_best = l_v_best.detach(
) # no gradients in last run
AEs[j].eval()
rec = F.sigmoid(AEs[j].Decoder.forward(l_v_best[:, j]))
ELBOs.append(
loss_functions.ELBOs(
rec, # (bs, n_ch, nx, ny)
l_v_best[:, j], # (bs, n_latent, 1, 1)
x_inp, # (bs, n_ch, nx, ny)
beta=beta,
dist_fct=dist_fct,
auto_batch_size=None))
if i == n_iter - 1:
all_recs.append(rec.view(bs, 1, n_ch, nx, ny).detach())
ELBOs = torch.cat(ELBOs, dim=1)
if i < n_iter - 1:
loss = (
torch.sum(ELBOs)) - 8. / 784. / 2 # historic reasons
# backward
opti.zero_grad()
loss.backward()
opti.step()
l_v_best.data = u.clip_to_sphere(l_v_best.data,
clip,
channel_dim=2)
else:
opti.zero_grad()
all_recs = torch.cat(all_recs, dim=1)
return ELBOs.detach(), l_v_best.detach(), all_recs
def get_CNN(load_path=DEFAULT_PATH):
load_path = join(DEFAULT_PATH, '../exp/mnist_cnn/nets/')
# network
shape = (1, 1, 28, 28)
kernelE = [5, 4, 3, 5]
strides = [1, 2, 2, 1]
feat_mapsE = [20, 70, 256, 10] # (32, 32, 16, 2)
model = nets.NN(feat_mapsE, shape[1:], kernels=kernelE, strides=strides)
# load net
print('path', load_path + '/vanilla_cnn.net')
model.load_state_dict(
torch.load(load_path + '/vanilla_cnn.net', map_location=str(u.dev())))
print('model loaded')
NN = CNN(model)
NN.eval()
NN.to(u.dev())
NN.code_base = 'pytorch'
return NN
def init_grid(self,
n_samples,
fraction_to_dismiss=None,
sample_sigma=None):
n_grid = self.n_samples_to_n_grid(n_samples)
print('init new grid', n_samples, n_grid)
limit = 0.99
if self.limit is not None:
limit = self.limit
grids = [(np.linspace(-limit, limit, n_grid)) for i in range(self.nd)]
xys = np.array(np.meshgrid(*grids))
xys = np.moveaxis(xys, 0, -1).reshape(n_grid**self.nd, self.nd)
self.samples[n_samples] = xys
self.l_v[n_samples] = \
torch.from_numpy(xys[:, :, None, None].astype(np.float32)).to(u.dev())
def init_grid(self,
n_samples,
fraction_to_dismiss=0.1,
mus=None,
sample_sigma=1):
if mus is None:
mus = np.zeros(self.nd)
samples = get_gaussian_samples(n_samples,
self.nd,
mus,
fraction_to_dismiss=fraction_to_dismiss,
sample_sigma=sample_sigma)
self.samples[n_samples] = samples
self.l_v[n_samples] = \
torch.from_numpy(samples[:, :, None, None].astype(
np.float32)).to(u.dev())
def __call__(self, x, l, n_coarse_steps=3, n_ft_steps=10):
x, l = u.n2t(x), u.n2t(l)
x, l = x.to(u.dev()), l.to(u.dev())
bs = x.shape[0]
best_other = 0
best_advs = [{
'original_label': -1,
'adversarial_label': None,
'distance': np.inf,
'img': torch.zeros(x.shape[1:]).to(u.dev())
} for _ in range(bs)]
coarse_steps = torch.zeros(bs).to(u.dev())
n_adv_found = 0
for i, coarse_step in enumerate(
torch.linspace(0, 1., n_coarse_steps).to(u.dev())):
current_adv = (1 - coarse_step) * x + coarse_step * best_other
best_other, current_label = self.get_best_prototypes(
current_adv, l)
for j, (current_adv_i, pred_l_i,
l_i) in enumerate(zip(current_adv, current_label, l)):
if best_advs[j]['original_label'] == -1 and pred_l_i != l_i:
self.update_adv(best_advs[j], current_adv_i, pred_l_i, l_i,
x[j])
coarse_steps[i] = coarse_step
n_adv_found += 1
if n_adv_found == bs:
break
best_advs_imgs = torch.cat([a['img'][None] for a in best_advs])
coarse_steps_old = coarse_steps[:, None, None, None]
# binary search
best_advs_imgs_old = best_advs_imgs.clone()
sign, step = -torch.ones(bs, 1, 1, 1).to(u.dev()), 0.5
for i in range(n_ft_steps):
coarse_steps = coarse_steps_old + step * sign
current_adv = (
1 - coarse_steps) * x + coarse_steps * best_advs_imgs_old
_, current_label = self.get_best_prototypes(current_adv, l)
for j, (pred_l_i, l_i) in enumerate(zip(current_label, l)):
if pred_l_i == l_i:
sign[j] = 1
else:
self.update_adv(best_advs[j], current_adv[j], pred_l_i,
l_i, x[j])
sign[j] = -1
step /= 2
return best_advs
def __init__(self,
AEs,
n_samples,
n_iter,
beta,
GM,
fraction_to_dismiss=0.1,
clip=5,
lr=0.05):
super().__init__(AEs,
n_samples,
n_iter,
beta,
GM,
fraction_to_dismiss=fraction_to_dismiss,
clip=clip,
lr=lr)
self.name_check = 'ABS'
self.rescale_b = True
self.discriminative_scalings = torch.tensor(
[1., 0.96, 1.001, 1.06, 0.98, 0.96, 1.03, 1., 1., 1.]).to(u.dev())
# model = mz.get_NearestNeighbor() # Nearest Neighbor, "nearest L2 dist to each class"=logits
# model = mz.get_madry() # Robust network from Madry et al. in tf
# -
# code is agnostic of pytorch/ tensorflow model --> foolbox model
if model.code_base == 'tensorflow':
fmodel = foolbox.models.TensorFlowModel(model.x_input,
model.pre_softmax, (0., 1.),
channel_axis=3)
elif model.code_base == 'pytorch':
model.eval()
fmodel = foolbox.models.PyTorchModel(
model, # return logits in shape (bs, n_classes)
bounds=(0., 1.),
num_classes=10,
device=u.dev())
else:
print('not implemented')
# test model
b, l = u.get_batch(bs=1000) # returns random batch as np.array
pred_label = np.argmax(fmodel.batch_predictions(b), axis=1)
print('score', float(np.sum(pred_label == l)) / b.shape[0])
# # Decision based attacks
# Note that this is only demo code. All experiments were optimized to our compute architecture.
# +
att = fa.PointwiseAttack(fmodel)
metric = foolbox.distances.L0
criterion = foolbox.criteria.Misclassification()
def get_classes(self, bs, input_batch, best_ind_classes):
pred = torch.zeros(bs, self.n_classes).to(u.dev())
pred[range(bs), self.classes[best_ind_classes]] = 1.
return pred
def get_VAE(n_samples=8000,
n_iter=50,
beta=1,
clip=5,
fraction_to_dismiss=0.1,
lr=0.05,
load=True,
binary=False,
load_path=DEFAULT_PATH):
"""Creates the ABS model. If binary is True, returns the full
ABS model including binarization and scalar, otherwise returns
the base ABS model without binarization and without scalar."""
load_path = join(DEFAULT_PATH, '../exp/VAE_swarm_MSE/nets/')
print('ABS model')
n_classes = 10
nd = 8
nx, ny = 28, 28
def init_models():
strides = [1, 2, 2, 1]
latent_act_fct = u.LinearActFct
kernelE = [5, 4, 3, 5]
feat_mapsE = [32, 32, 64, nd]
encoder = {
'feat_maps': feat_mapsE,
'kernels': kernelE,
'strides': strides
}
kernelD = [4, 5, 5, 4]
feat_mapsD = [32, 16, 16, 1]
decoder = {
'feat_maps': feat_mapsD,
'kernels': kernelD,
'strides': strides
}
AEs = []
for i in range(n_classes):
AE = nets.VariationalAutoEncoder(encoder,
decoder,
latent_act_fct=latent_act_fct)
AE.eval()
AE.to(u.dev())
AEs.append(AE)
return AEs
AEs = init_models()
if load:
for i in range(n_classes):
path = load_path + f'/ABS_{i}.net'
AEs[i].iters = 29000
AEs[i].load_state_dict(torch.load(path, map_location=str(u.dev())))
print('model loaded')
GM = sampling.GaussianSamples(AEs, nd, n_classes, nx=nx, ny=ny)
if binary:
model = ELBOVAE_binary
else:
model = ELBOVAE
model = model(AEs,
n_samples,
n_iter,
beta,
GM,
fraction_to_dismiss,
clip,
lr=lr)
model.eval()
model.code_base = 'pytorch'
model.has_grad = False
return model
