def test_genattack_numpy(request: Any) -> None:
class Model:
def __call__(self, inputs: Any) -> Any:
return inputs.mean(axis=(2, 3))
model = Model()
with pytest.raises(ValueError):
fbn.NumPyModel(model, bounds=(0, 1), data_format="foo")
fmodel = fbn.NumPyModel(model, bounds=(0, 1))
x, y = ep.astensors(
*fbn.samples(
fmodel, dataset="imagenet", batchsize=16, data_format="channels_first"
)
)
with pytest.raises(ValueError, match="data_format"):
fbn.attacks.GenAttack(reduced_dims=(2, 2)).run(
fmodel, x, fbn.TargetedMisclassification(y), epsilon=0.3
)
with pytest.raises(ValueError, match="channel_axis"):
fbn.attacks.GenAttack(channel_axis=2, reduced_dims=(2, 2)).run(
fmodel, x, fbn.TargetedMisclassification(y), epsilon=0.3
)
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
model = models.resnet18(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
clean_acc = accuracy(fmodel, images, labels) * 100
print(f"clean accuracy: {clean_acc:.1f} %")
# the attack trys a combination of specified rotations and translations to an image
# stops early if adversarial shifts and translations for all images are found
attack = fa.SpatialAttack(
max_translation=6, # 6px so x in [x-6, x+6] and y in [y-6, y+6]
num_translations=6, # number of translations in x, y.
max_rotation=20, # +- rotation in degrees
num_rotations=5, # number of rotations
# max total iterations = num_rotations * num_translations**2
)
# report the success rate of the attack (percentage of samples that could
# be adversarially perturbed) and the robust accuracy (the remaining accuracy
# of the model when it is attacked)
xp_, _, success = attack(fmodel, images, labels)
suc = success.float32().mean().item() * 100
print(f"attack success: {suc:.1f} %"
" (for the specified rotation and translation bounds)")
print(f"robust accuracy: {100 - suc:.1f} %"
" (for the specified rotation and translation bounds)")
def test_astensors_tensor(t: Tensor) -> None:
ts = (t, t + 1, t + 2)
ys = ep.astensors(*ts)
assert isinstance(ys, tuple)
assert len(ts) == len(ys)
for ti, yi in zip(ts, ys):
assert (ti == yi).all()
def accuracy(fmodel: Model, inputs: Any, labels: Any) -> float:
inputs_, labels_ = ep.astensors(inputs, labels)
del inputs, labels
predictions = fmodel(inputs_).argmax(axis=-1)
accuracy = (predictions == labels_).float32().mean()
return accuracy.item()
def accuracy(self, inputs, labels):
inputs_, labels_ = ep.astensors(inputs, labels)
del inputs, labels
self.model.eval() # make sure in evaluating mode ...
predictions = self.fmodel(inputs_).argmax(axis=-1)
accuracy = (predictions == labels_)
return accuracy.sum().item()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--steps',
type=int,
default=20000,
help='Iteration of BA')
parser.add_argument('--targeted',
action='store',
default=False,
help='For targeted attack')
args = parser.parse_args()
model = Net()
model.load_state_dict(torch.load('mnist_cnn.pt'))
model.eval()
preprocessing = dict(mean=0.1307, std=0.3081)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
fmodel = fmodel.transform_bounds((0, 1))
assert fmodel.bounds == (0, 1)
images, labels = ep.astensors(
*samples(fmodel, dataset="mnist", batchsize=10))
print('Model accuracy on clean examples: {}'.format(
accuracy(fmodel, images, labels)))
if args.targeted:
target_class = (labels + 7) % 10
criterion = fb.criteria.TargetedMisclassification(target_class)
else:
criterion = fb.criteria.Misclassification(labels)
attack = fa.BoundaryAttack(steps=args.steps, tensorboard=None)
epsilons = np.linspace(0.01, 10, 20)
raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons)
robust_accuracy = 1 - success.float32().mean(axis=-1)
plt.plot(epsilons, robust_accuracy.numpy())
plt.xlabel("Epsilons")
plt.ylabel("Robust Accuracy")
plt.savefig('mnist_BA_robust_acc.jpg')
plt.show()
mean_distance = []
for i in range(len(clipped)):
dist = np.mean(fb.distances.l2(clipped[i], images).numpy())
mean_distance.append(dist)
plt.plot(epsilons, mean_distance)
plt.xlabel('Epsilons')
plt.ylabel('Mean L2 distance')
plt.savefig("mnist_BA_mean_L2distance.jpg")
plt.show()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--steps',
type=int,
default=10000,
help='Iteration of BA')
parser.add_argument('--targeted',
action='store',
default=False,
help='For targeted attack')
args = parser.parse_args()
model = Net()
model.load_state_dict(torch.load('mnist_cnn.pt'))
model.eval()
preprocessing = dict(mean=0.1307, std=0.3081)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
fmodel = fmodel.transform_bounds((0, 1))
assert fmodel.bounds == (0, 1)
images, labels = ep.astensors(
*samples(fmodel, dataset="mnist", batchsize=10))
print('Model accuracy on clean examples: {}'.format(
accuracy(fmodel, images, labels)))
epsilons = np.linspace(0.01, 10, 20)
boundary_attack = fa.BoundaryAttack(steps=args.steps, tensorboard=None)
_, _, ba_success = boundary_attack(fmodel,
images,
labels,
epsilons=epsilons)
ba_robust_accuracy = 1 - ba_success.float32().mean(axis=-1)
random_attack = fa.L2RepeatedAdditiveGaussianNoiseAttack(
repeats=args.steps)
_, _, ra_success = random_attack(fmodel, images, labels, epsilons=epsilons)
ra_robust_accuracy = 1 - ra_success.float32().mean(axis=-1)
legends = ["Boundary Attack", "Random Attack"]
plt.plot(epsilons, ba_robust_accuracy.numpy())
plt.plot(epsilons, ra_robust_accuracy.numpy())
plt.legend(legends, loc='upper right')
plt.xlabel("Perturbation Norm (L2)")
plt.ylabel("Robust Accuracy")
plt.title("{} Queries".format(args.steps))
plt.savefig('mnist_robust_acc.jpg')
plt.show()
def run_attacks(MODEL_DIR, res_path):
rel_dirs = [x for x in os.listdir(MODEL_DIR) if '2020' in x]
alpha = [re.findall('a=([0-9, \.]*)_', d)[0] for d in rel_dirs]
res = dict.fromkeys(alpha)
learner = prep_learner()
for model_path, curr_alpha in tqdm(zip(rel_dirs, alpha), total=len(alpha)):
conf.save_path = Path(path.join(MODEL_DIR, model_path))
fix_str = [
x for x in os.listdir(path.join(MODEL_DIR, model_path))
if 'model' in x
][0][8:]
learner.load_state(conf,
fix_str,
model_only=True,
from_save_folder=True)
# probs
set_probes(learner)
for model in learner.models:
model = torch.nn.DataParallel(model.cuda(),
device_ids=list(range(4)))
model.eval()
res[curr_alpha] = dict()
for (attack,
eps), attack_name in tqdm(zip(attack_list, attack_list_names),
desc='attaking ' + str(curr_alpha),
total=len(attack_list)):
fmodel = JointModelEP(
[PyTorchModel(m, bounds=(0, 1)) for m in learner.models],
'cuda')
attack = attack()
success_tot = []
for images, labels in tqdm(learner.eval_loader,
total=len(learner.eval_loader),
desc=attack_name):
images, labels = ep.astensors(images.to('cuda'),
labels.to('cuda'))
_, _, success = attack(fmodel, images, labels, epsilons=eps)
success_tot.append(success)
success_tot = ep.concatenate(success_tot, -1)
# calculate and report the robust accuracy
robust_accuracy = 1 - success_tot.float32().mean(axis=-1)
for epsilon, acc in zip(eps, robust_accuracy):
res[curr_alpha][attack_name + '_' + str(epsilon)] = acc.item()
pickle.dump(res, open(res_path, 'wb'))
pickle.dump(res, open(res_path, 'wb'))
def test_blur_numpy(request: Any) -> None:
class Model:
def __call__(self, inputs: Any) -> Any:
return inputs.mean(axis=(2, 3))
model = Model()
with pytest.raises(ValueError):
fbn.NumPyModel(model, bounds=(0, 1), data_format="foo")
fmodel = fbn.NumPyModel(model, bounds=(0, 1))
x, y = ep.astensors(*fbn.samples(
fmodel, dataset="imagenet", batchsize=16,
data_format="channels_first"))
with pytest.raises(ValueError, match="data_format"):
fbn.attacks.GaussianBlurAttack()(fmodel, x, y, epsilons=None)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--steps',
type=int,
default=1000,
help='Maximum number of steps to perform')
parser.add_argument('--targeted',
action='store',
default=False,
help='For targeted attack')
args = parser.parse_args()
model = Net()
model.load_state_dict(torch.load('mnist_cnn.pt'))
model.eval()
preprocessing = dict(mean=0.1307, std=0.3081)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
fmodel = fmodel.transform_bounds((0, 1))
assert fmodel.bounds == (0, 1)
images, labels = ep.astensors(
*samples(fmodel, dataset="mnist", batchsize=10))
print('Model accuracy on clean examples: {}'.format(
accuracy(fmodel, images, labels)))
if args.targeted:
target_class = (labels + 7) % 10
criterion = fb.criteria.TargetedMisclassification(target_class)
else:
criterion = fb.criteria.Misclassification(labels)
attack = fa.L2DeepFoolAttack(steps=args.steps)
epsilons = None
raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons)
robust_accuracy = 1 - success.float32().mean()
print("Robust Accuracy", robust_accuracy.item())
dist = np.mean(fb.distances.l2(clipped, images).numpy())
print("Average perturbation norm", dist)
def evaluate_l2(
fmodel: Model,
inputs: Any,
labels: Any,
*,
attacks: List[Attack],
epsilons: List[L2],
) -> Tuple[Any, Any]:
x, y = ep.astensors(inputs, labels)
del inputs, labels
attack_success = np.zeros((len(attacks), len(epsilons), len(x)), dtype=np.float32)
for i, attack in enumerate(attacks):
sig = signature(type(attack).__init__)
minimizing = "epsilon" not in sig.parameters
if minimizing:
# TODO: support hyperparameters
xp = attack(fmodel, x, y)
predictions = fmodel(xp).argmax(axis=-1)
correct = (predictions == y).float32().numpy().astype(np.bool)
perturbations = xp - x
norms = flatten(perturbations).square().sum(axis=-1).sqrt().numpy()
for j, epsilon in enumerate(epsilons):
attack_success[i, j] = np.logical_and(
np.logical_not(correct), norms <= epsilon
)
else:
for j, epsilon in enumerate(epsilons):
attack.epsilon = epsilon # type: ignore
xp = attack(fmodel, x, y)
predictions = fmodel(xp).argmax(axis=-1)
correct = (predictions == y).float32().numpy().astype(np.bool)
perturbations = xp - x
norms = flatten(perturbations).square().sum(axis=-1).sqrt().numpy()
# TODO: relax this norm check or pass a slightly stricter norm to the attack
attack_success[i, j] = np.logical_and(
np.logical_not(correct), norms <= epsilon
).astype(np.float32)
robust_accuracy = 1.0 - attack_success.max(axis=0).mean(axis=-1)
return attack_success, robust_accuracy
def attack_one_batch(fmodel, images, labels, iter=0, verbose=True):
images, labels = ep.astensors(images, labels)
raw_advs, clipped_advs, success = attack(fmodel,
images,
labels,
epsilons=epsilons)
if verbose: print("===" * 8, iter, "===" * 8)
if verbose:
robust_accuracy = 1 - success.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
if verbose:
fig = plt.gcf()
os.makedirs("./image/", exist_ok=True)
for i in range(len(raw_advs)):
img_v = raw_advs[i].raw
torchvision.utils.save_image(
img_v,
f'./image/{str(iter).zfill(4)}_{str(i).zfill(3)}_.png')
return [x.raw for x in raw_advs] #
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
model = models.resnet18(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
clean_acc = accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
# replace the gradient with the gradient from another model
model2 = fmodel # demo, we just use the same model
# TODO: this is still a bit annoying because we need
# to overwrite run to get the labels
class Attack(LinfPGD):
def value_and_grad(self, loss_fn, x):
val1 = loss_fn(x)
loss_fn2 = self.get_loss_fn(model2, self.labels)
_, grad2 = ep.value_and_grad(loss_fn2, x)
return val1, grad2
def run(self, model, inputs, criterion, *, epsilon, **kwargs):
criterion_ = get_criterion(criterion)
self.labels = criterion_.labels
return super().run(model,
inputs,
criterion_,
epsilon=epsilon,
**kwargs)
# apply the attack
attack = Attack()
epsilons = [
0.0,
0.0002,
0.0005,
0.0008,
0.001,
0.0015,
0.002,
0.003,
0.01,
0.1,
0.3,
0.5,
1.0,
]
raw_advs, clipped_advs, success = attack(fmodel,
images,
labels,
epsilons=epsilons)
# calculate and report the robust accuracy (the accuracy of the model when
# it is attacked)
robust_accuracy = 1 - success.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
# we can also manually check this
# we will use the clipped advs instead of the raw advs, otherwise
# we would need to check if the perturbation sizes are actually
# within the specified epsilon bound
print()
print("we can also manually check this:")
print()
print("robust accuracy for perturbations with")
for eps, advs_ in zip(epsilons, clipped_advs):
acc2 = accuracy(fmodel, advs_, labels)
print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
print(" perturbation sizes:")
perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2,
3)).numpy()
print(" ", str(perturbation_sizes).replace("\n", "\n" + " "))
if acc2 == 0:
break
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
#model = models.resnet18(pretrained=True).eval()
#model=torch.load('/data1/zyh/copycat/Framework/cifar_model.pth')
model = AlexNet()
path = "./cifar_net.pth"
#path = '/data1/zyh/copycat/Framework/cifar_model.pth'
#model.load_state_dict(torch.load('/data1/zyh/copycat/Framework/cifar_model.pth'))
#pretrained_dict = {k: v for k, v in model_pretrained.items() if k in model_dict}
#model_dict.update(pretrained_dict)
#model.load_state_dict(state_dict)
model.load_state_dict(torch.load(path), strict=True)
model = model.to(device)
model.eval()
print(type(model))
#preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
preprocessing = dict(mean=[0.5] * 3, std=[0.5] * 3, axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
test_dataset = torchvision.datasets.CIFAR10(
root='~/.torch/',
train=False,
#transform = transforms.Compose([transforms.Resize((256,256)),transforms.ToTensor()]),
transform=transforms.Compose([transforms.ToTensor()]),
download=True)
# test_dataset .data = test_dataset.data[:128*5]
test_loader = torch.utils.data.DataLoader(
dataset=test_dataset,
batch_size=128, #该参数表示每次读取的批样本个数
shuffle=False) #该参数表示读取时是否打乱样本顺序
# 创建迭代器
#data_iter = iter(test_loader)
#images, labels = next(data_iter)
# 当迭代开始时, 队列和线程开始读取数据
#images, labels = data_iter.next()
#im=images
#images=im.resize(100,3,128,128)
with torch.no_grad():
all_clean_acc_foolbox = []
## native predict
predict_func(test_loader, model)
for ii, (imgs, lbls) in tqdm.tqdm(enumerate(test_loader),
total=len(test_loader)):
imgs = imgs.to(device)
lbls = lbls.to(device)
images, labels = ep.astensors(imgs, lbls)
## calc with foolbox
pred_lbl_foolbox = fmodel(images)
clean_acc_one = accuracy(fmodel, imgs, lbls)
all_clean_acc_foolbox.append(clean_acc_one)
clean_acc = sum(all_clean_acc_foolbox) / len(all_clean_acc_foolbox)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
# apply the attack
attack = LinfPGD()
'''epsilons = [
0.0,
0.0002,
0.0005,
0.0008,
0.001,
0.0015,
0.002,
0.003,
0.01,
0.1,
0.3,
0.5,
1.0,
]'''
epsilons = [
0.0005,
0.001,
0.002,
0.01,
0.1,
]
def attack_one_batch(fmodel, images, labels, iter=0, verbose=True):
images, labels = ep.astensors(images, labels)
raw_advs, clipped_advs, success = attack(fmodel,
images,
labels,
epsilons=epsilons)
if verbose: print("===" * 8, iter, "===" * 8)
if verbose:
robust_accuracy = 1 - success.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
if verbose:
fig = plt.gcf()
os.makedirs("./image/", exist_ok=True)
for i in range(len(raw_advs)):
img_v = raw_advs[i].raw
torchvision.utils.save_image(
img_v,
f'./image/{str(iter).zfill(4)}_{str(i).zfill(3)}_.png')
return [x.raw for x in raw_advs] #
print("====" * 8, "start attack", "====" * 8)
collection_adv = []
collection_gt = []
for ii, (imgs, lbls) in tqdm.tqdm(enumerate(test_loader),
total=len(test_loader)):
imgs = imgs.to(device)
lbls = lbls.to(device)
# images, labels = ep.astensors(images,labels)
adv_ret = attack_one_batch(fmodel=fmodel,
images=imgs,
labels=lbls,
iter=ii,
verbose=True)
collection_adv.append(torch.stack(adv_ret))
collection_gt.append(lbls.cpu())
print("====" * 8, "start evaluation", "====" * 8)
with torch.no_grad():
adv_total_dataset = torch.cat(collection_adv, dim=1)
lbl_total_dataset = torch.cat(collection_gt).to(device)
# print (adv_total_dataset.mean(dim=(1,2,3,4)),"the mean if each eps")
for (eps, ep_adv_dataset) in zip(epsilons, adv_total_dataset):
# print ("eps:",eps,"===>"*8)
# print (ep_adv_dataset.mean(),"each...")
advs_ = ep_adv_dataset.to(device)
acc2 = accuracy(fmodel, advs_, lbl_total_dataset)
print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
dataset = torch.utils.data.TensorDataset(ep_adv_dataset,
lbl_total_dataset)
dl = torch.utils.data.DataLoader(dataset, batch_size=128)
predict_func(dl, model)
from foolbox import PyTorchModel, accuracy, samples
import foolbox.attacks as fa
import numpy as np
if __name__ == "__main__":
# instantiate a model (could also be a TensorFlow or JAX model)
model = models.resnet18(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
clean_acc = accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
print("")
attacks = [
fa.FGSM(),
fa.LinfPGD(),
fa.LinfBasicIterativeAttack(),
fa.LinfAdditiveUniformNoiseAttack(),
fa.LinfDeepFoolAttack(),
]
epsilons = [
0.0,
0.0005,
advacc = 0.0
if __name__ == "__main__":
total = 0
attack_success = 0
start = time.time()
print("==> Testing %s.." %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(start))))
# 4 make adversarial sample
for batch_idx, (images, labels) in enumerate(rawloader):
total += labels.size(0)
nepochs = nepochs + 1
if batch_idx > 10000:
break
images, labels = ep.astensors(images.cuda(), labels.cuda())
# TODO 注：转换类型，否则损失函数报错
labels = labels.astype(dtype=torch.long)
# TODO 计算对抗样本
advs, _, success = attack(fmodel, images, labels, epsilons=epsilons)
# calculate and report the robust accuracy
robust_accuracy = 1 - success.float32().mean(axis=-1)
rawacc += accuracy(fmodel, images, labels)
for adv, label, acc in zip(advs, labels, robust_accuracy):
if not os.path.exists(os.path.join(opt.outf, str(label.item()))):
os.mkdir(os.path.join(opt.outf, str(label.item())))
advacc = advacc + acc.item() * 100.0
model.fit(x_train,y_train, epochs=5) model.evaluate(x_test,y_test,verbose=2) #instantiate the model fmodel=TensorFlowModel(model, bounds=(0,1)) #get data and test the model #wrapping the tensors with ep.astensors is optional, but it allows #us to work with EagerPy tensors in the following ########################################################## images, labels = samples(fmodel, dataset="mnist", batchsize=100) images1, labels1=ep.astensors(*samples(fmodel, dataset="mnist", batchsize=100)) print(accuracy(fmodel, images1, labels1)) predict=fmodel(images).numpy() tf.nn.softmax(predict).numpy() correct_pred=tf.math.argmax(predict,1) print(correct_pred) #print(images) images_arr=np.array(images) #print(images_arr) #print(images_arr.shape) #16,28,28,1
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
model = models.resnet18(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
clean_acc = accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
# apply the attack
attack = LinfPGD()
epsilons = [
0.0,
0.0002,
0.0005,
0.0008,
0.001,
0.0015,
0.002,
0.003,
0.01,
0.1,
0.3,
0.5,
1.0,
]
raw_advs, clipped_advs, success = attack(fmodel,
images,
labels,
epsilons=epsilons)
# calculate and report the robust accuracy (the accuracy of the model when
# it is attacked)
robust_accuracy = 1 - success.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
# we can also manually check this
# we will use the clipped advs instead of the raw advs, otherwise
# we would need to check if the perturbation sizes are actually
# within the specified epsilon bound
print()
print("we can also manually check this:")
print()
print("robust accuracy for perturbations with")
for eps, advs_ in zip(epsilons, clipped_advs):
acc2 = accuracy(fmodel, advs_, labels)
print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
print(" perturbation sizes:")
perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2,
3)).numpy()
print(" ", str(perturbation_sizes).replace("\n", "\n" + " "))
if acc2 == 0:
break
def foolbox_attack(filter=None,
filter_preserve='low',
free_parm='eps',
plot_num=None):
# get model.
model = get_model()
model = nn.DataParallel(model).to(device)
model = model.eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
if plot_num:
free_parm = ''
val_loader = get_val_loader(plot_num)
else:
# Load images.
val_loader = get_val_loader(args.attack_batch_size)
if 'eps' in free_parm:
epsilons = [0.001, 0.003, 0.005, 0.008, 0.01, 0.1]
else:
epsilons = [0.01]
if 'step' in free_parm:
steps = [1, 5, 10, 30, 40, 50]
else:
steps = [args.iteration]
for step in steps:
# Adversarial attack.
if args.attack_type == 'LinfPGD':
attack = LinfPGD(steps=step)
elif args.attack_type == 'FGSM':
attack = FGSM()
clean_acc = 0.0
for i, data in enumerate(val_loader, 0):
# Samples (attack_batch_size * attack_epochs) images for adversarial attack.
if i >= args.attack_epochs:
break
images, labels = data[0].to(device), data[1].to(device)
if step == steps[0]:
clean_acc += (get_acc(
fmodel, images, labels
)) / args.attack_epochs # accumulate for attack epochs.
_images, _labels = ep.astensors(images, labels)
raw_advs, clipped_advs, success = attack(fmodel,
_images,
_labels,
epsilons=epsilons)
if plot_num:
grad = torch.from_numpy(
raw_advs[0].numpy()).to(device) - images
grad = grad.clone().detach_()
return grad
if filter:
robust_accuracy = torch.empty(len(epsilons))
for eps_id in range(len(epsilons)):
grad = torch.from_numpy(
raw_advs[eps_id].numpy()).to(device) - images
grad = grad.clone().detach_()
freq = dct.dct_2d(grad)
if filter_preserve == 'low':
mask = torch.zeros(freq.size()).to(device)
mask[:, :, :filter, :filter] = 1
elif filter_preserve == 'high':
mask = torch.zeros(freq.size()).to(device)
mask[:, :, filter:, filter:] = 1
masked_freq = torch.mul(freq, mask)
new_grad = dct.idct_2d(masked_freq)
x_adv = torch.clamp(images + new_grad, 0, 1).detach_()
robust_accuracy[eps_id] = (get_acc(fmodel, x_adv, labels))
else:
robust_accuracy = 1 - success.float32().mean(axis=-1)
if i == 0:
robust_acc = robust_accuracy / args.attack_epochs
else:
robust_acc += robust_accuracy / args.attack_epochs
if step == steps[0]:
print("sample size is : ",
args.attack_batch_size * args.attack_epochs)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
print(
f"Model {args.model} robust accuracy for {args.attack_type} perturbations with"
)
for eps, acc in zip(epsilons, robust_acc):
print(
f" Step {step}, Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %"
)
print(' -------------------')
baseline_model.compile(
loss='sparse_categorical_crossentropy',
optimizer=tf.keras.optimizers.Adam(0.001),
metrics=['accuracy'],
)
baseline_model.fit(ds_train,
epochs=1,
validation_data=ds_test,
steps_per_epoch=7500 // batch_size,
validation_steps=2500 // batch_size,
callbacks=[tensorboard_callback])
for images, labels in ds_train.take(1): # only take first element of dataset
images_ex = ep.astensors(images)
labels_ex = ep.astensors(labels)
fmodel = fb.TensorFlowModel(baseline_model, bounds=(0, 1))
attacks = [
fa.FGSM(),
fa.LinfPGD(),
fa.LinfBasicIterativeAttack(),
fa.LinfAdditiveUniformNoiseAttack(),
fa.LinfDeepFoolAttack(),
]
attacks_names = [
"FGSM", "LinfPGD", "LinfBasicIterativeAttack",
"LinfAdditiveUniformNoiseAttack", "LinfDeepFoolAttack"
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
#model = models.resnet18(pretrained=True).eval()
#model=torch.load('/data1/zyh/copycat/Framework/cifar_model.pth')
model =AlexNet()
path = "./cifar_net.pth"
#path = '/data1/zyh/copycat/Framework/cifar_model.pth'
#model.load_state_dict(torch.load('/data1/zyh/copycat/Framework/cifar_model.pth'))
#pretrained_dict = {k: v for k, v in model_pretrained.items() if k in model_dict}
#model_dict.update(pretrained_dict)
#model.load_state_dict(state_dict)
model.load_state_dict(torch.load(path),strict=True)
model.eval()
print(type(model))
#preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
preprocessing = dict(mean=[0.5]*3, std=[0.5]*3, axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
#test_dataset = torchvision.datasets.CIFAR10(root='~/.torch/',
# train=True,
# #transform = transforms.Compose([transforms.Resize((256,256)),transforms.ToTensor()]),
# transform = transforms.Compose([transforms.ToTensor()]),
# download=True)
#test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
# batch_size=128, #该参数表示每次读取的批样本个数
# shuffle=False) #该参数表示读取时是否打乱样本顺序
# # 创建迭代器
#data_iter = iter(test_loader)
#images, labels = next(data_iter)
# 当迭代开始时, 队列和线程开始读取数据
#images, labels = data_iter.next()
#images=images.to(device)
#labels=labels.to(device)
#im=images
#images=im.resize(100,3,128,128)
images, labels = ep.astensors(*samples(fmodel, dataset="cifar10", batchsize=16))
#images, labels = ep.astensors(*samples(fmodel, dataset="imagenet", batchsize=16))
#print(images.shape)
clean_acc = accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
# apply the attack
attack = LinfPGD()
'''epsilons = [
0.0,
0.0002,
0.0005,
0.0008,
0.001,
0.0015,
0.002,
0.003,
0.01,
0.1,
0.3,
0.5,
1.0,
]'''
epsilons = [
0.0005,
0.001,
0.002,
0.01,
0.1,
]
raw_advs, clipped_advs, success = attack(fmodel, images, labels, epsilons=epsilons)
print(type(raw_advs))
print("atest")
# calculate and report the robust accuracy (the accuracy of the model when
# it is attacked)
robust_accuracy = 1 - success.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
# we can also manually check this
# we will use the clipped advs instead of the raw advs, otherwise
# we would need to check if the perturbation sizes are actually
# within the specified epsilon bound
print()
print("we can also manually check this:")
print()
print("robust accuracy for perturbations with")
for eps, advs_ in zip(epsilons, clipped_advs):
acc2 = accuracy(fmodel, advs_, labels)
print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
print(" perturbation sizes:")
perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2, 3)).numpy()
print(" ", str(perturbation_sizes).replace("\n", "\n" + " "))
if acc2 == 0:
break
fig = plt.gcf()
os.makedirs("./image/",exist_ok=True)
for i in range(len(raw_advs)):
img_v = raw_advs[i].raw
torchvision.utils.save_image(img_v, './image/'+str(i) +'.png')
model.params.analysis_save_dir = os.path.join(
model.params.analysis_out_dir, 'savefiles')
if not os.path.exists(model.params.analysis_save_dir):
os.makedirs(model.params.analysis_save_dir)
model.to(params.device)
model.load_checkpoint()
fmodel = PyTorchModel(model.eval(), bounds=(0, 1))
print('\n', '~' * 79)
num_batches = len(test_loader.dataset) // model.params.batch_size
attack_success = np.zeros(
(len(attacks), len(epsilons), num_batches, model.params.batch_size),
dtype=np.bool)
for batch_index, (data, target) in enumerate(test_loader):
data = model.preprocess_data(data.to(model.params.device))
target = target.to(model.params.device)
images, labels = ep.astensors(*(data, target))
del data
del target
print(
f'Model type: {model.params.model_type} [{model_index+1} out of {len(log_files)}]'
)
print(f'Batch {batch_index+1} out of {num_batches}')
print(f'accuracy {accuracy(fmodel, images, labels)}')
for attack_index, attack in enumerate(attacks):
advs, inputs, success = attack(fmodel,
images,
labels,
epsilons=epsilons)
assert success.shape == (len(epsilons), len(images))
success_ = success.numpy()
assert success_.dtype == np.bool
994: 'stinkhorn, carrion fungus',
995: 'earthstar',
996: 'hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa',
997: 'bolete',
998: 'ear, spike, capitulum',
999: 'toilet tissue, toilet paper, bathroom tissue'}
if __name__ == '__main__':
advs_save_path = 'adv_data/'
r_save_path = 'r_data/'
model = models.resnet152(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
# preprocessing = dict(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
images, labels, file_names = ep.astensors(*samples(fmodel, dataset="imagenet", batchsize=1, vis=True))
file_names = file_names.raw
prediction = accuracy(fmodel, images, labels, True).raw.cpu().detach().numpy()
attack = FGSM()
# attack = LinfPGD()
# attack = L2DeepFoolAttack()
epsilons = [0.0, 0.001, 0.01, 0.03, 0.1, 0.3, 0.5, 1.0]
advs, _, _ = attack(fmodel, images, labels, epsilons=epsilons)
for i in range(len(epsilons)):
r = advs[0] - advs[i]
attack_labels = accuracy(fmodel, advs[i], labels, True).raw.cpu().detach().numpy()
imgs_path = advs_save_path + str(epsilons[i]) + '/'
# 0.03,
# 0.1,
# 0.3,
# 0.5,
# 1.0,
# ]
#print(x_test[0],y_test[0])
foolbox_model = foolbox.models.TensorFlowModel(
model=model,
bounds=(0, 1),
)
foolbox_model = foolbox_model.transform_bounds((0, 1))
assert foolbox_model.bounds == (0, 1)
images = ep.astensors(x_test[0:16])
labels = ep.astensors(y_test[0:16])
#doesn't work right
attack = foolbox.attacks.LinfPGD()
_, advs, success = attack(foolbox_model, images, labels, epsilons=0.03)
print(success)
# The end accuracy was 0.9809 compared to 0.9496 when not using KFold.
#
# The predictions for the first 5:
# array([[5.43997092e-23, 4.90765706e-14, 2.98600290e-15, 6.97289174e-03,
# 4.53084188e-32, 9.93027031e-01, 6.42024900e-24, 4.53798596e-17,
# 2.48018923e-16, 5.83970931e-16],
# [9.99999881e-01, 1.24962863e-16, 1.35353190e-07, 9.39377114e-16,
