def get_attack(attack, fmodel):
args = []
kwargs = {}
# L0
if attack == 'SAPA':
metric = foolbox.distances.L0
A = fa.SaltAndPepperNoiseAttack(fmodel, distance = metric)
elif attack == 'PA':
metric = foolbox.distances.L0
A = fa.PointwiseAttack(fmodel, distance = metric)
# L2
elif 'IGD' in attack:
metric = foolbox.distances.MSE
A = fa.L2BasicIterativeAttack(fmodel, distance = metric)
# kwargs['epsilons'] = 1.5
elif attack == 'AGNA':
metric = foolbox.distances.MSE
kwargs['epsilons'] = np.linspace(0.5, 1, 50)
A = fa.AdditiveGaussianNoiseAttack(fmodel, distance = metric)
elif attack == 'BA':
metric = foolbox.distances.MSE
A = fa.BoundaryAttack(fmodel, distance = metric)
kwargs['log_every_n_steps'] = 500001
elif 'DeepFool' in attack:
metric = foolbox.distances.MSE
A = fa.DeepFoolL2Attack(fmodel, distance = metric)
elif attack == 'PAL2':
metric = foolbox.distances.MSE
A = fa.PointwiseAttack(fmodel, distance = metric)
elif attack == "CWL2":
metric = foolbox.distances.MSE
A = fa.CarliniWagnerL2Attack(fmodel, distance = metric)
# L inf
elif 'FGSM' in attack and not 'IFGSM' in attack:
metric = foolbox.distances.Linf
A = fa.FGSM(fmodel, distance = metric)
kwargs['epsilons'] = 20
elif 'PGD' in attack:
metric = foolbox.distances.Linf
A = fa.LinfinityBasicIterativeAttack(fmodel, distance = metric)
elif 'IGM' in attack:
metric = foolbox.distances.Linf
A = fa.MomentumIterativeAttack(fmodel, distance = metric)
else:
raise Exception('Not implemented')
return A, metric, args, kwargs
def get_adv_perf(args):
sample_num, IM_SIZE, latent_size= int(args.sample_num), int(args.IM_SIZE), int(args.latent_size)
model_loc = Path(args.model_loc)
RESULT_PATH = Path(args.model_loc).parent
device = torch.device(args.device)
distance = str(args.distance)
num_workers = 0 ## ??? what?
batch_size = 1
num_classes = 10
labels = list(range(num_classes))
with open(args.files_df_loc, 'rb') as f:
files_df = pickle.load(f)
files_df['val'] = files_df['val'].sample(n=sample_num)
# Make generators:
if args.dataset == 'CIFAR10':
dataloaders = make_generators_DF_cifar(files_df, batch_size, num_workers, size=IM_SIZE,
path_colname='path', adv_path_colname=None, return_loc=True, normalize=False)
elif args.dataset == 'MNIST':
dataloaders = make_generators_DF_MNIST(files_df, batch_size, num_workers, size=IM_SIZE,
path_colname='path', adv_path_colname=None, return_loc=True, normalize=False)
mean = 0.1307
std = 0.3081
transform = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize(IM_SIZE),
transforms.ToTensor(),
transforms.Normalize((mean,), (std,))])
else:
print("Incorrect Dataset")
# if args.model_type == 'vae': # then the model_loc isn't the real location
# model_dict = {}
# model_file = Path(model_loc).name
# model_name = model_file.split('-')[0]
# for label in labels:
# model_file_cr = model_file + '_label_'+str(label)+'_model_best.pth.tar'
# model_loc_cr = str(model_loc.parent / model_file_cr)
# model_dict[label] = load_net(model_loc_cr).to(device).eval()
# gen_model = GenerativeVAE(model_dict=model_dict, labels=labels, latent_size=latent_size, device=device).eval()
# elif args.model_type == 'cvae':
# model = load_net(model_loc).to(device).eval()
# gen_model = GenerativeCVAE(model=model, labels=labels, latent_size=latent_size, device=device).eval()
# Train for each of the labels, here the model_loc is not an actual loc, just the base
if args.model_type == 'vae' or args.model_type == 'feat_vae':
model_dict = {}
model_file = Path(model_loc).name
model_name = model_file.split('-')[0]
for label in labels:
model_file_cr = model_file + '_label_'+str(label)+'_model_best.pth.tar'
model_loc_cr = str(model_loc.parent / model_file_cr)
model_dict[label] = load_net(model_loc_cr, args).to(device).eval()
print('args.model_type', args.model_type)
if args.model_type == 'vae':
print(f'Loading VAE')
gen_model = GenerativeVAE(model_dict=model_dict, labels=labels, latent_size=latent_size, device=device).to(device).eval()
elif args.model_type == 'feat_vae':
print(f'Loading FEATURE VAE')
gen_model = GenerativeFeatVAE(model_dict=model_dict, labels=labels, latent_size=latent_size, device=device).to(device).eval()
else:
print("Invalid model_type")
# If CVAE, load model and predict normally:
elif args.model_type == 'cvae':
print(f'Loading CVAE')
model = load_net(model_loc, args).to(device).eval()
gen_model = GenerativeCVAE(model=model, labels=labels, latent_size=latent_size, device=device).to(device).eval()
else:
print("Incorrect Model Type")
# ATTACK:
fmodel = foolbox.models.PyTorchModel(gen_model, bounds=(0, 1), preprocessing=(mean, std),
num_classes=num_classes, device=device)
# attack = get_attack(args.attack_type, fmodel, distance=distance)
num_failed = 0
results = pd.DataFrame()
for i, batch in enumerate(tqdm(dataloaders['val'])):
# Foolbox always wants numpy arrays, and we are using single batch, so this batch dim is removed.
img, label, file_loc = batch[0].to(device), batch[1].to(device), batch[2][0]
# image, label, file_loc = batch[0].numpy(), int(batch[1].numpy()), batch[2][0]
# image = load_image(file_loc) See if should use this with the foolbox preprocessign instead
# np_img = np.expand_dims(np.squeeze(img.numpy()), 0)
# np_img = np.squeeze(img.cpu().numpy())
# trans_img = transform(img.squeeze().unsqueeze(0).cpu())
# trans_img = trans_img.unsqueeze(0).to(device)
# trans_orig_pred = np.argmax(gen_model(trans_img).cpu().numpy())
np_img = np.expand_dims(np.squeeze(img.cpu().numpy()), 0)
np_label = int(label.cpu().numpy()[0])
# print('original prediction foolbox: ', np.argmax(fmodel.predictions(np_img)))
# if args.attack_type == 'boundary':
# # adv_object = attack(np_img, np_label, unpack=False, tune_batch_size=False, verbose=True, log_every_n_steps=1)
# else:
# adv_object = attack(np_img, np_label, unpack=False)
# $$$$$$$$$$$$$$$$$$
att = fa.PointwiseAttack(fmodel)
metric = foolbox.distances.L0
criterion = foolbox.criteria.Misclassification()
# Estimate gradients from scores
if not gen_model.has_grad:
GE = foolbox.gradient_estimators.CoordinateWiseGradientEstimator(0.1)
fmodel = foolbox.models.ModelWithEstimatedGradients(fmodel, GE)
# gernate Adversarial
a = foolbox.adversarial.Adversarial(fmodel, criterion, np_img, np_label, distance=metric)
att(a)
print('pred', np.argmax(fmodel.predictions(a.image)))
print('distance', a.normalized_distance)
# $$$$$$$$$$$$$$$$$$
adv_distance = adv_object.distance
np_adv_img = adv_object.image
# adv_img = torch.from_numpy(np_adv_img).float().unsqueeze(0).to(device)
# orig_pred = np.argmax(gen_model(img).cpu().numpy())
# adv_pred = np.argmax(gen_model(adv_img).cpu().numpy())
# np_trans_img = np.expand_dims(np.squeeze(img.cpu().numpy()), 0)
# trans_adv_img = attack(trans_img, np_label)
# trans_adv_pred = np.argmax(gen_model(trans_adv_img).cpu().numpy())
f_orig_pred = np.argmax(fmodel.predictions(np_img))
f_adv_pred = np.argmax(fmodel.predictions(np_adv_img))
# print(f'FOOLBOX: orig_pred: {f_orig_pred}, adv_pred: {f_adv_pred}')
# print(f'Original Label {np_label}')
if np_adv_img is None:
print('-----np_adv_img is None, Attack Failed')
num_failed +=1
results = results.append({'path': file_loc, 'true_label': np.squeeze(label.item()), 'adv_distance': adv_distance.value,
'f_orig_pred': f_orig_pred, 'f_adv_pred': f_adv_pred,
'attack_type': attack_type, 'distance': distance}, ignore_index=True)
save_loc = str(RESULT_PATH)+'/results_'+str(Path(model_loc).name)+'_ns'+str(sample_num)+'_'+str(attack_type)+'_'+str(distance)+'_adv.pkl'
print(f'saving at: {save_loc}')
with open(save_loc, 'wb') as f:
pickle.dump(results, f, pickle.HIGHEST_PROTOCOL)
print(f'Finished testing {sample_num} images. Number of failures: {num_failed}')
bounds=(0., 1.),
num_classes=10,
device=u.dev())
else:
print('not implemented')
# test model
b, l = u.get_batch(bs=1000) # returns random batch as np.array
pred_label = np.argmax(fmodel.batch_predictions(b), axis=1)
print('score', float(np.sum(pred_label == l)) / b.shape[0])
# # Decision based attacks
# Note that this is only demo code. All experiments were optimized to our compute architecture.
# +
att = fa.PointwiseAttack(fmodel)
metric = foolbox.distances.L0
criterion = foolbox.criteria.Misclassification()
b, l = u.get_batch(bs=1) # returns random batch
plt.imshow(b[0, 0], cmap='gray')
plt.title('orig')
plt.axis('off')
plt.show()
# Estimate gradients from scores
if not model.has_grad:
GE = foolbox.gradient_estimators.CoordinateWiseGradientEstimator(0.1)
fmodel = foolbox.models.ModelWithEstimatedGradients(fmodel, GE)
# gernate Adversarial
import eagerpy as ep
import foolbox as fbn
import foolbox.attacks as fa
from foolbox.devutils import flatten
import pytest
from conftest import ModeAndDataAndDescription
def get_attack_id(x: fa.Attack) -> str:
return repr(x)
attacks: List[fa.Attack] = [
fa.PointwiseAttack(),
fa.PointwiseAttack(l2_binary_search=False),
]
@pytest.mark.parametrize("attack", attacks, ids=get_attack_id)
def test_pointwise_untargeted_attack(
request: Any,
fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription,
attack: fa.PointwiseAttack,
) -> None:
(fmodel, x,
y), real, low_dimensional_input = fmodel_and_data_ext_for_attacks
if not low_dimensional_input or not real:
pytest.skip()