def test_untargeted_attacks(
fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription,
attack_test_target: AttackTestTarget,
) -> None:
(fmodel, x, y), real, low_dimensional_input = fmodel_and_data_ext_for_attacks
if attack_test_target.requires_real_model and not real:
pytest.skip()
if attack_test_target.requires_low_dimensional_input and not low_dimensional_input:
pytest.skip()
if isinstance(x, ep.NumPyTensor) and attack_test_target.uses_grad:
pytest.skip()
x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
fmodel = fmodel.transform_bounds((0, 1))
acc = fbn.accuracy(fmodel, x, y)
assert acc > 0
# repeat stochastic attacks three times before we mark them as failed
attack_repetitions = 3 if attack_test_target.stochastic_attack else 1
for _ in range(attack_repetitions):
advs, _, _ = attack_test_target.attack(
fmodel, x, y, epsilons=attack_test_target.epsilon
)
adv_acc = fbn.accuracy(fmodel, advs, y)
if adv_acc < acc:
break
assert adv_acc < acc
def test_fast_minimum_norm_untargeted_attack(
fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription,
attack_and_p: Tuple[FMNAttackLp, Union[int, float]],
) -> None:
(fmodel, x,
y), real, low_dimensional_input = fmodel_and_data_ext_for_attacks
if isinstance(x, ep.NumPyTensor):
pytest.skip()
x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
fmodel = fmodel.transform_bounds((0, 1))
init_attack = fa.DatasetAttack()
init_attack.feed(fmodel, x)
init_advs = init_attack.run(fmodel, x, y)
attack, p = attack_and_p
advs = attack.run(fmodel, x, y, starting_points=init_advs)
init_norms = ep.norms.lp(flatten(init_advs - x), p=p, axis=-1)
norms = ep.norms.lp(flatten(advs - x), p=p, axis=-1)
is_smaller = norms < init_norms
assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y)
assert fbn.accuracy(fmodel, advs, y) <= fbn.accuracy(fmodel, init_advs, y)
assert is_smaller.any()
def test_pointwise_untargeted_attack(
request: Any,
fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription,
attack: fa.PointwiseAttack,
) -> None:
(fmodel, x,
y), real, low_dimensional_input = fmodel_and_data_ext_for_attacks
if not low_dimensional_input or not real:
pytest.skip()
x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
fmodel = fmodel.transform_bounds((0, 1))
init_attack = fa.SaltAndPepperNoiseAttack(steps=50)
init_advs = init_attack.run(fmodel, x, y)
advs = attack.run(fmodel, x, y, starting_points=init_advs)
init_norms_l0 = ep.norms.lp(flatten(init_advs - x), p=0, axis=-1)
norms_l0 = ep.norms.lp(flatten(advs - x), p=0, axis=-1)
init_norms_l2 = ep.norms.lp(flatten(init_advs - x), p=2, axis=-1)
norms_l2 = ep.norms.lp(flatten(advs - x), p=2, axis=-1)
is_smaller_l0 = norms_l0 < init_norms_l0
is_smaller_l2 = norms_l2 < init_norms_l2
assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y)
assert fbn.accuracy(fmodel, advs, y) <= fbn.accuracy(fmodel, init_advs, y)
assert is_smaller_l2.any()
assert is_smaller_l0.any()
def test_brendel_bethge_untargeted_attack(
request: Any,
fmodel_and_data_ext_for_attacks: ModelDescriptionAndData,
attack_and_p: Tuple[BrendelBethgeAttack, Union[int, float]],
) -> None:
if request.config.option.skipslow:
pytest.skip()
(fmodel, x, y), real = fmodel_and_data_ext_for_attacks
if isinstance(x, ep.NumPyTensor):
pytest.skip()
x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
fmodel = fmodel.transform_bounds((0, 1))
init_attack = fa.DatasetAttack()
init_attack.feed(fmodel, x)
init_advs = init_attack.run(fmodel, x, y)
attack, p = attack_and_p
advs = attack.run(fmodel, x, y, starting_points=init_advs)
init_norms = ep.norms.lp(flatten(init_advs - x), p=p, axis=-1)
norms = ep.norms.lp(flatten(advs - x), p=p, axis=-1)
is_smaller = norms < init_norms
assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y)
assert fbn.accuracy(fmodel, advs, y) <= fbn.accuracy(fmodel, init_advs, y)
assert is_smaller.any()
def test_dataset_attack(
fmodel_and_data_ext_for_attacks: ModelDescriptionAndData, ) -> None:
(fmodel, x, y), _ = fmodel_and_data_ext_for_attacks
x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
fmodel = fmodel.transform_bounds((0, 1))
attack = fbn.attacks.DatasetAttack()
attack.feed(fmodel, x)
assert fbn.accuracy(fmodel, x, y) > 0
advs, _, success = attack(fmodel, x, y, epsilons=None)
assert success.shape == (len(x), )
assert success.all()
assert fbn.accuracy(fmodel, advs, y) == 0
with pytest.raises(ValueError, match="unknown distance"):
attack(fmodel, x, y, epsilons=[500.0, 1000.0])
attack = fbn.attacks.DatasetAttack(distance=fbn.distances.l2)
attack.feed(fmodel, x)
advss, _, success = attack(fmodel, x, y, epsilons=[500.0, 1000.0])
assert success.shape == (2, len(x))
assert success.all()
assert fbn.accuracy(fmodel, advss[0], y) == 0
assert fbn.accuracy(fmodel, advss[1], y) == 0
with pytest.raises(TypeError, match="unexpected keyword argument"):
attack(fmodel, x, y, epsilons=None, invalid=True)
def test_accuracy(fmodel_and_data: ModelAndData) -> None:
fmodel, x, y = fmodel_and_data
accuracy = fbn.accuracy(fmodel, x, y)
assert 0 <= accuracy <= 1
assert accuracy > 0.5
y = fmodel(x).argmax(axis=-1)
accuracy = fbn.accuracy(fmodel, x, y)
assert accuracy == 1
def test_binarization_attack(
fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription,
) -> None:
# get a model with thresholding
(fmodel, x, y), _, low_dimensional_input = fmodel_and_data_ext_for_attacks
# binarization doesn't work well for imagenet models
if not low_dimensional_input:
pytest.skip()
x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
fmodel = fmodel.transform_bounds((0, 1))
fmodel = ThresholdingWrapper(fmodel, threshold=0.5)
acc = accuracy(fmodel, x, y)
assert acc > 0
# find some adversarials and check that they are non-trivial
attack = BinarySearchContrastReductionAttack(target=0)
advs, _, _ = attack(fmodel, x, y, epsilons=None)
assert accuracy(fmodel, advs, y) < acc
# run the refinement attack
attack2 = BinarizationRefinementAttack(threshold=0.5, included_in="upper")
advs2, _, _ = attack2(fmodel, x, y, starting_points=advs, epsilons=None)
# make sure the predicted classes didn't change
assert (fmodel(advs).argmax(axis=-1) == fmodel(advs2).argmax(axis=-1)).all()
# make sure the perturbations didn't get larger and some got smaller
norms1 = flatten(advs - x).norms.l2(axis=-1)
norms2 = flatten(advs2 - x).norms.l2(axis=-1)
assert (norms2 <= norms1).all()
assert (norms2 < norms1).any()
# run the refinement attack
attack2 = BinarizationRefinementAttack(included_in="upper")
advs2, _, _ = attack2(fmodel, x, y, starting_points=advs, epsilons=None)
# make sure the predicted classes didn't change
assert (fmodel(advs).argmax(axis=-1) == fmodel(advs2).argmax(axis=-1)).all()
# make sure the perturbations didn't get larger and some got smaller
norms1 = flatten(advs - x).norms.l2(axis=-1)
norms2 = flatten(advs2 - x).norms.l2(axis=-1)
assert (norms2 <= norms1).all()
assert (norms2 < norms1).any()
with pytest.raises(ValueError, match="starting_points"):
attack2(fmodel, x, y, epsilons=None)
attack2 = BinarizationRefinementAttack(included_in="lower")
with pytest.raises(ValueError, match="does not match"):
attack2(fmodel, x, y, starting_points=advs, epsilons=None)
attack2 = BinarizationRefinementAttack(included_in="invalid") # type: ignore
with pytest.raises(ValueError, match="expected included_in"):
attack2(fmodel, x, y, starting_points=advs, epsilons=None)
def test_evaluate(
fmodel_and_data: Tuple[fbn.Model, ep.Tensor, ep.Tensor]) -> None:
pytest.skip()
assert False
fmodel, x, y = fmodel_and_data # type: ignore
attacks = [
# L2BasicIterativeAttack,
# L2CarliniWagnerAttack,
# L2ContrastReductionAttack,
# BinarySearchContrastReductionAttack,
# LinearSearchContrastReductionAttack,
]
epsilons = [0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0]
acc = fbn.accuracy(fmodel, x, y)
assert acc > 0
_, robust_accuracy = fbn.evaluate_l2(fmodel,
x,
y,
attacks=attacks,
epsilons=epsilons)
assert robust_accuracy[0] == acc
assert robust_accuracy[-1] == 0.0
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
model = models.resnet18(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
clean_acc = accuracy(fmodel, images, labels) * 100
print(f"clean accuracy: {clean_acc:.1f} %")
# the attack trys a combination of specified rotations and translations to an image
# stops early if adversarial shifts and translations for all images are found
attack = fa.SpatialAttack(
max_translation=6, # 6px so x in [x-6, x+6] and y in [y-6, y+6]
num_translations=6, # number of translations in x, y.
max_rotation=20, # +- rotation in degrees
num_rotations=5, # number of rotations
# max total iterations = num_rotations * num_translations**2
)
# report the success rate of the attack (percentage of samples that could
# be adversarially perturbed) and the robust accuracy (the remaining accuracy
# of the model when it is attacked)
xp_, _, success = attack(fmodel, images, labels)
suc = success.float32().mean().item() * 100
print(f"attack success: {suc:.1f} %"
" (for the specified rotation and translation bounds)")
print(f"robust accuracy: {100 - suc:.1f} %"
" (for the specified rotation and translation bounds)")
def validate(fmodel, val_loader, attack, epsilons):
clean_acc_list = list()
robust_acc_list = list()
for i, (images, labels) in enumerate(tqdm(val_loader)):
images = images.cuda()
images = ep.PyTorchTensor(images)
labels = labels.cuda()
labels = ep.PyTorchTensor(labels)
# accuracy(fmodel, images, labels)
clean_acc = accuracy(fmodel, images, labels)
clean_acc_list.append(clean_acc * 100)
print(clean_acc * 100)
raw_advs, clipped_advs, success = attack(fmodel,
images,
labels,
epsilons=epsilons)
robust_accuracy = 1 - success.float32().mean(axis=-1)
robust_acc_list.append(robust_accuracy.raw)
return np.mean(clean_acc_list), torch.stack(robust_acc_list).mean(
dim=0).tolist()
def exp_balckbox_attack():
model = model_name(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
total_success = 0
widgets = [
'train :',
Percentage(), ' ',
Bar('#'), ' ',
Timer(), ' ',
ETA(), ' ',
FileTransferSpeed()
]
pbar = ProgressBar(widgets=widgets)
length = len(train_loader)
print("length train loader: ", length)
total_imgs = 0
for batch_data in pbar(train_loader):
images, labels = batch_data['image'].to(
DEVICE), batch_data['label_idx'].to(DEVICE)
acc = accuracy(fmodel, images, labels)
print(images.shape[0])
total_imgs += images.shape[0]
total_success += int(acc * images.shape[0])
print("success attack is ", str(total_imgs - total_success),
"/" + str(total_imgs))
print("watermark attack rate is: ", str(1 - total_success / total_imgs))
def test_loading_model(request: Any, url: str) -> None:
backend = request.config.option.backend
if backend != "tensorflow":
pytest.skip()
# download model
try:
fmodel = fbn.zoo.get_model(url, name="MobileNetV2", overwrite=True)
except fbn.zoo.GitCloneError:
pytest.skip()
# download again (test overwriting)
try:
fmodel = fbn.zoo.get_model(url, name="MobileNetV2", overwrite=True)
except fbn.zoo.GitCloneError:
pytest.skip()
# create a dummy image
# x = np.zeros(dim, dtype=np.float32)
# x[:] = np.random.randn(*x.shape)
x, y = fbn.samples(fmodel, dataset="imagenet", batchsize=16)
# run the model
# logits = model(x)
# probabilities = ep.softmax(logits)
# predicted_class = np.argmax(logits)
assert fbn.accuracy(fmodel, x, y) > 0.9
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--steps',
type=int,
default=20000,
help='Iteration of BA')
parser.add_argument('--targeted',
action='store',
default=False,
help='For targeted attack')
args = parser.parse_args()
model = Net()
model.load_state_dict(torch.load('mnist_cnn.pt'))
model.eval()
preprocessing = dict(mean=0.1307, std=0.3081)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
fmodel = fmodel.transform_bounds((0, 1))
assert fmodel.bounds == (0, 1)
images, labels = ep.astensors(
*samples(fmodel, dataset="mnist", batchsize=10))
print('Model accuracy on clean examples: {}'.format(
accuracy(fmodel, images, labels)))
if args.targeted:
target_class = (labels + 7) % 10
criterion = fb.criteria.TargetedMisclassification(target_class)
else:
criterion = fb.criteria.Misclassification(labels)
attack = fa.BoundaryAttack(steps=args.steps, tensorboard=None)
epsilons = np.linspace(0.01, 10, 20)
raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons)
robust_accuracy = 1 - success.float32().mean(axis=-1)
plt.plot(epsilons, robust_accuracy.numpy())
plt.xlabel("Epsilons")
plt.ylabel("Robust Accuracy")
plt.savefig('mnist_BA_robust_acc.jpg')
plt.show()
mean_distance = []
for i in range(len(clipped)):
dist = np.mean(fb.distances.l2(clipped[i], images).numpy())
mean_distance.append(dist)
plt.plot(epsilons, mean_distance)
plt.xlabel('Epsilons')
plt.ylabel('Mean L2 distance')
plt.savefig("mnist_BA_mean_L2distance.jpg")
plt.show()
def test_spatial_attacks(
fmodel_and_data_ext_for_attacks: ModelDescriptionAndData,
attack_grad_real: Tuple[fbn.Attack, bool],
) -> None:
attack, repeated = attack_grad_real
if repeated:
attack = attack.repeat(2)
(fmodel, x, y), real = fmodel_and_data_ext_for_attacks
if not real:
pytest.skip()
x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
fmodel = fmodel.transform_bounds((0, 1))
acc = fbn.accuracy(fmodel, x, y)
assert acc > 0
advs, _, _ = attack(fmodel, x, y) # type: ignore
assert fbn.accuracy(fmodel, advs, y) < acc
def test_untargeted_attacks(
fmodel_and_data_ext_for_attacks: Tuple[Tuple[fbn.Model, ep.Tensor,
ep.Tensor], bool],
attack_eps_grad_real: Tuple[fbn.Attack, Optional[float], bool, bool],
) -> None:
attack, eps, attack_uses_grad, requires_real_model = attack_eps_grad_real
(fmodel, x, y), real = fmodel_and_data_ext_for_attacks
if requires_real_model and not real:
pytest.skip()
if isinstance(x, ep.NumPyTensor) and attack_uses_grad:
pytest.skip()
x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
fmodel = fmodel.transform_bounds((0, 1))
acc = fbn.accuracy(fmodel, x, y)
assert acc > 0
advs, _, _ = attack(fmodel, x, y, epsilons=eps)
assert fbn.accuracy(fmodel, advs, y) < acc
def test_untargeted_attacks(
fmodel_and_data_ext_for_attacks: ModelDescriptionAndData,
attack_test_target: AttackTestTarget,
) -> None:
(fmodel, x, y), real = fmodel_and_data_ext_for_attacks
if attack_test_target.requires_real_model and not real:
pytest.skip()
if isinstance(x, ep.NumPyTensor) and attack_test_target.uses_grad:
pytest.skip()
x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
fmodel = fmodel.transform_bounds((0, 1))
acc = fbn.accuracy(fmodel, x, y)
assert acc > 0
advs, _, _ = attack_test_target.attack(fmodel,
x,
y,
epsilons=attack_test_target.epsilon)
assert fbn.accuracy(fmodel, advs, y) < acc
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--steps',
type=int,
default=10000,
help='Iteration of BA')
parser.add_argument('--targeted',
action='store',
default=False,
help='For targeted attack')
args = parser.parse_args()
model = Net()
model.load_state_dict(torch.load('mnist_cnn.pt'))
model.eval()
preprocessing = dict(mean=0.1307, std=0.3081)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
fmodel = fmodel.transform_bounds((0, 1))
assert fmodel.bounds == (0, 1)
images, labels = ep.astensors(
*samples(fmodel, dataset="mnist", batchsize=10))
print('Model accuracy on clean examples: {}'.format(
accuracy(fmodel, images, labels)))
epsilons = np.linspace(0.01, 10, 20)
boundary_attack = fa.BoundaryAttack(steps=args.steps, tensorboard=None)
_, _, ba_success = boundary_attack(fmodel,
images,
labels,
epsilons=epsilons)
ba_robust_accuracy = 1 - ba_success.float32().mean(axis=-1)
random_attack = fa.L2RepeatedAdditiveGaussianNoiseAttack(
repeats=args.steps)
_, _, ra_success = random_attack(fmodel, images, labels, epsilons=epsilons)
ra_robust_accuracy = 1 - ra_success.float32().mean(axis=-1)
legends = ["Boundary Attack", "Random Attack"]
plt.plot(epsilons, ba_robust_accuracy.numpy())
plt.plot(epsilons, ra_robust_accuracy.numpy())
plt.legend(legends, loc='upper right')
plt.xlabel("Perturbation Norm (L2)")
plt.ylabel("Robust Accuracy")
plt.title("{} Queries".format(args.steps))
plt.savefig('mnist_robust_acc.jpg')
plt.show()
def test_pointwise_targeted_attack(
request: Any,
fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription,
attack: fa.PointwiseAttack,
) -> None:
(fmodel, x,
y), real, low_dimensional_input = fmodel_and_data_ext_for_attacks
if not low_dimensional_input or not real:
pytest.skip()
x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
fmodel = fmodel.transform_bounds((0, 1))
init_attack = fa.SaltAndPepperNoiseAttack(steps=50)
init_advs = init_attack.run(fmodel, x, y)
logits = fmodel(init_advs)
num_classes = logits.shape[-1]
target_classes = logits.argmax(-1)
target_classes = ep.where(target_classes == y,
(target_classes + 1) % num_classes,
target_classes)
criterion = fbn.TargetedMisclassification(target_classes)
advs = attack.run(fmodel, x, criterion, starting_points=init_advs)
init_norms_l0 = ep.norms.lp(flatten(init_advs - x), p=0, axis=-1)
norms_l0 = ep.norms.lp(flatten(advs - x), p=0, axis=-1)
init_norms_l2 = ep.norms.lp(flatten(init_advs - x), p=2, axis=-1)
norms_l2 = ep.norms.lp(flatten(advs - x), p=2, axis=-1)
is_smaller_l0 = norms_l0 < init_norms_l0
is_smaller_l2 = norms_l2 < init_norms_l2
assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y)
assert fbn.accuracy(fmodel, advs, y) <= fbn.accuracy(fmodel, init_advs, y)
assert fbn.accuracy(fmodel, advs, target_classes) > fbn.accuracy(
fmodel, x, target_classes)
assert fbn.accuracy(fmodel, advs, target_classes) >= fbn.accuracy(
fmodel, init_advs, target_classes)
assert is_smaller_l2.any()
assert is_smaller_l0.any()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--steps',
type=int,
default=1000,
help='Maximum number of steps to perform')
parser.add_argument('--targeted',
action='store',
default=False,
help='For targeted attack')
args = parser.parse_args()
model = Net()
model.load_state_dict(torch.load('mnist_cnn.pt'))
model.eval()
preprocessing = dict(mean=0.1307, std=0.3081)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
fmodel = fmodel.transform_bounds((0, 1))
assert fmodel.bounds == (0, 1)
images, labels = ep.astensors(
*samples(fmodel, dataset="mnist", batchsize=10))
print('Model accuracy on clean examples: {}'.format(
accuracy(fmodel, images, labels)))
if args.targeted:
target_class = (labels + 7) % 10
criterion = fb.criteria.TargetedMisclassification(target_class)
else:
criterion = fb.criteria.Misclassification(labels)
attack = fa.L2DeepFoolAttack(steps=args.steps)
epsilons = None
raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons)
robust_accuracy = 1 - success.float32().mean()
print("Robust Accuracy", robust_accuracy.item())
dist = np.mean(fb.distances.l2(clipped, images).numpy())
print("Average perturbation norm", dist)
def main() -> None:
print(DEVICE)
# instantiate a model (could also be a TensorFlow or JAX model)
model = model_name(pretrained=True).eval()
# preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
# image_mean = torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1)
# image_std = torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1)
preprocessing = dict(mean=[0.491, 0.482, 0.447],
std=[0.247, 0.243, 0.262],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
total_success = 0
# headers = ['img', 'advIndex', 'alpha ', 'angle', 'class', 'l2']
# with open('result_inception_v3_gen4_30.csv', 'w')as f:
# f_csv = csv.writer(f)
# f_csv.writerow(headers)
attack = advDigitalMark()
# SpatialAttack()
epoch = 0
length = len(train_loader)
print("length train loader: ", length)
total_msg = ''
for batch_data in train_loader:
# if epoch == 1:
# break
images, labels, filenames = batch_data['image'].to(
DEVICE), batch_data['label_idx'].to(DEVICE), batch_data['filename']
clean_acc = accuracy(fmodel, images, labels)
logging.info("\nepoch " + str(epoch) + ":\nclean accuracy: " +
str(clean_acc))
total_msg += "\nepoch " + str(epoch) + ":\nclean accuracy: " + str(
clean_acc)
count, msg = attack.run(fmodel, images, labels, filenames=filenames)
total_success += count
total_msg += msg
logging.info("current success attack is " + str(total_success))
send_email(total_msg, title='fsy天池比赛V1模型3000-5000结果报告')
epoch += 1
total_msg += "\n total attack success img is " + str(total_success)
# logging.info("total succes is "+str(total_success)+"/100")
send_email(total_msg, title='fsy天池比赛V1模型3000-5000结果报告')
def test_hsj_targeted_attack(
request: Any,
fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription,
attack_and_p: Tuple[fa.HopSkipJumpAttack, Union[int, float]],
) -> None:
if request.config.option.skipslow:
pytest.skip()
(fmodel, x, y), real, _ = fmodel_and_data_ext_for_attacks
if isinstance(x, ep.NumPyTensor):
pytest.skip()
if not real:
pytest.skip()
x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
fmodel = fmodel.transform_bounds((0, 1))
logits_np = fmodel(x).numpy()
num_classes = logits_np.shape[-1]
y_np = logits_np.argmax(-1)
target_classes_np = (y_np + 1) % num_classes
for i in range(len(target_classes_np)):
while target_classes_np[i] not in y_np:
target_classes_np[i] = (target_classes_np[i] + 1) % num_classes
target_classes = ep.from_numpy(y, target_classes_np)
criterion = fbn.TargetedMisclassification(target_classes)
init_attack = fa.DatasetAttack()
init_attack.feed(fmodel, x)
init_advs = init_attack.run(fmodel, x, criterion)
attack, p = attack_and_p
advs = attack.run(fmodel, x, criterion, starting_points=init_advs)
init_norms = ep.norms.lp(flatten(init_advs - x), p=p, axis=-1)
norms = ep.norms.lp(flatten(advs - x), p=p, axis=-1)
is_smaller = norms < init_norms
assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y)
assert fbn.accuracy(fmodel, advs, target_classes) > fbn.accuracy(
fmodel, x, target_classes)
assert fbn.accuracy(fmodel, advs, target_classes) >= fbn.accuracy(
fmodel, init_advs, target_classes)
assert is_smaller.any()
def exp_otherAttack():
model = model_name(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
total_success = 0
attack = SpatialAttack()
epoch = 0
widgets = [
'train :',
Percentage(), ' ',
Bar('#'), ' ',
Timer(), ' ',
ETA(), ' ',
FileTransferSpeed()
]
pbar = ProgressBar(widgets=widgets)
length = len(train_loader)
print("length train loader: ", length)
for batch_data in pbar(train_loader):
if epoch == 25:
break
images, labels = batch_data['image'].to(
DEVICE), batch_data['label_idx'].to(DEVICE)
clean_acc = accuracy(fmodel, images, labels)
print(f"\nepoch " + str(epoch) + ":\nclean accuracy: ", clean_acc)
epsilons = [1]
_, _, isSuceess = attack(fmodel, images, labels, epsilons=epsilons)
isSuceess = isSuceess.cpu().numpy()
total_success += np.sum(isSuceess == True)
print("current success : ", total_success)
# total_success += int(adv_acc*10)
epoch += 1
print("success attack is ", str(total_success))
def exp1():
print(DEVICE)
parser = argparse.ArgumentParser(
description='CIFAR10 Training against DDN Attack')
parser.add_argument('--data',
default='../defenses/data/cifar10',
help='path to dataset')
parser.add_argument('--workers',
default=2,
type=int,
help='number of data loading workers')
parser.add_argument('--cpu',
action='store_true',
help='force training on cpu')
parser.add_argument('--save-folder',
'--sf',
default='weights/cifar10_inception_v3/',
help='folder to save state dicts')
parser.add_argument('--save-freq',
'--sfr',
default=10,
type=int,
help='save frequency')
parser.add_argument('--save-name',
'--sn',
default='cifar10',
help='name for saving the final state dict')
parser.add_argument('--batch-size',
'-b',
default=20,
type=int,
help='mini-batch size')
parser.add_argument('--epochs',
'-e',
default=100,
type=int,
help='number of total epochs to run')
parser.add_argument('--lr',
'--learning-rate',
default=0.001,
type=float,
help='initial learning rate')
parser.add_argument('--lr-decay',
'--lrd',
default=0.2,
type=float,
help='decay for learning rate')
parser.add_argument('--lr-step',
'--lrs',
default=10,
type=int,
help='step size for learning rate decay')
parser.add_argument('--momentum', default=0.9, type=float, help='momentum')
parser.add_argument('--weight-decay',
'--wd',
default=5e-4,
type=float,
help='weight decay')
parser.add_argument('--drop',
default=0.3,
type=float,
help='dropout rate of the classifier')
parser.add_argument('--adv',
type=int,
default=None,
help='epoch to start training with adversarial images')
parser.add_argument('--max-norm',
type=float,
default=1,
help='max norm for the adversarial perturbations')
parser.add_argument('--steps',
default=10,
type=int,
help='number of steps for the attack')
parser.add_argument(
'--visdom-port',
'--vp',
type=int,
default=8097,
help='For visualization, which port visdom is running.')
parser.add_argument('--print-freq',
'--pf',
default=10,
type=int,
help='print frequency')
args = parser.parse_args()
print(args)
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
test_transform = transforms.Compose([
transforms.ToTensor(),
])
train_set = data.Subset(
CIFAR10(args.data,
train=True,
transform=train_transform,
download=True), list(range(30000, 50000)))
val_set = data.Subset(
CIFAR10(args.data, train=True, transform=test_transform,
download=True), list(range(48000, 50000)))
test_set = CIFAR10(args.data,
train=False,
transform=test_transform,
download=True)
train_loader = data.DataLoader(train_set,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
drop_last=True,
pin_memory=True)
val_loader = data.DataLoader(val_set,
batch_size=100,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
test_loader = data.DataLoader(test_set,
batch_size=100,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
image_mean = torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1)
image_std = torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1)
m = AlexNet().eval()
model = NormalizedModel(model=m, mean=image_mean, std=image_std).to(
DEVICE) # keep images in the [0, 1] range
model_file = '/home/frankfeng/researchData/code/adversarial_training_code/PLP/fast_adv/defenses/weights/cifar10_AlexNet/cifar10_valacc0.8019999772310257.pth'
model_dict = torch.load(model_file)
model.load_state_dict(model_dict)
# preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
# image_mean = torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1)
# image_std = torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1)
preprocessing = dict(mean=[0.491, 0.482, 0.447],
std=[0.247, 0.243, 0.262],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1))
total_success = 0
# headers = ['img', 'advIndex', 'alpha ', 'angle', 'class', 'l2']
# with open('result_inception_v3_gen4_30.csv', 'w')as f:
# f_csv = csv.writer(f)
# f_csv.writerow(headers)
attack = advDigitalMark()
# SpatialAttack()
epoch = 0
widgets = [
'train :',
Percentage(), ' ',
Bar('#'), ' ',
Timer(), ' ',
ETA(), ' ',
FileTransferSpeed()
]
pbar = ProgressBar(widgets=widgets)
length = len(train_loader)
print("length train loader: ", length)
for epoch in range(args.epochs):
if epoch == 25:
break
for i, (images, labels) in enumerate(tqdm.tqdm(train_loader,
ncols=80)):
images, labels = images.to(DEVICE), labels.to(DEVICE)
clean_acc = accuracy(fmodel, images, labels)
print(f"\nepoch " + str(epoch) + ":\nclean accuracy: ", clean_acc)
count = attack.run(fmodel, images, labels)
total_success += count
epoch += 1
print("current success attack is ", str(total_success))
print(
str(model_name) + "_" + str(max_gen) + "_" + str(max_gen) +
": total attack success img is ", str(total_success))
import numpy as np
if __name__ == "__main__":
# instantiate a model (could also be a TensorFlow or JAX model)
model = models.resnet18(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
clean_acc = accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
print("")
attacks = [
fa.FGSM(),
fa.LinfPGD(),
fa.LinfBasicIterativeAttack(),
fa.LinfAdditiveUniformNoiseAttack(),
fa.LinfDeepFoolAttack(),
]
epsilons = [
0.0,
0.0005,
0.001,
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
model = models.resnet18(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
clean_acc = accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
# replace the gradient with the gradient from another model
model2 = fmodel # demo, we just use the same model
# TODO: this is still a bit annoying because we need
# to overwrite run to get the labels
class Attack(LinfPGD):
def value_and_grad(self, loss_fn, x):
val1 = loss_fn(x)
loss_fn2 = self.get_loss_fn(model2, self.labels)
_, grad2 = ep.value_and_grad(loss_fn2, x)
return val1, grad2
def run(self, model, inputs, criterion, *, epsilon, **kwargs):
criterion_ = get_criterion(criterion)
self.labels = criterion_.labels
return super().run(model,
inputs,
criterion_,
epsilon=epsilon,
**kwargs)
# apply the attack
attack = Attack()
epsilons = [
0.0,
0.0002,
0.0005,
0.0008,
0.001,
0.0015,
0.002,
0.003,
0.01,
0.1,
0.3,
0.5,
1.0,
]
raw_advs, clipped_advs, success = attack(fmodel,
images,
labels,
epsilons=epsilons)
# calculate and report the robust accuracy (the accuracy of the model when
# it is attacked)
robust_accuracy = 1 - success.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
# we can also manually check this
# we will use the clipped advs instead of the raw advs, otherwise
# we would need to check if the perturbation sizes are actually
# within the specified epsilon bound
print()
print("we can also manually check this:")
print()
print("robust accuracy for perturbations with")
for eps, advs_ in zip(epsilons, clipped_advs):
acc2 = accuracy(fmodel, advs_, labels)
print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
print(" perturbation sizes:")
perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2,
3)).numpy()
print(" ", str(perturbation_sizes).replace("\n", "\n" + " "))
if acc2 == 0:
break
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
model = models.resnet18(pretrained=True).eval()
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
clean_acc = accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
# apply the attack
attack = LinfPGD()
epsilons = [
0.0,
0.0002,
0.0005,
0.0008,
0.001,
0.0015,
0.002,
0.003,
0.01,
0.1,
0.3,
0.5,
1.0,
]
raw_advs, clipped_advs, success = attack(fmodel,
images,
labels,
epsilons=epsilons)
# calculate and report the robust accuracy (the accuracy of the model when
# it is attacked)
robust_accuracy = 1 - success.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
# we can also manually check this
# we will use the clipped advs instead of the raw advs, otherwise
# we would need to check if the perturbation sizes are actually
# within the specified epsilon bound
print()
print("we can also manually check this:")
print()
print("robust accuracy for perturbations with")
for eps, advs_ in zip(epsilons, clipped_advs):
acc2 = accuracy(fmodel, advs_, labels)
print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
print(" perturbation sizes:")
perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2,
3)).numpy()
print(" ", str(perturbation_sizes).replace("\n", "\n" + " "))
if acc2 == 0:
break
def main() -> None:
# instantiate a model (could also be a TensorFlow or JAX model)
#model = models.resnet18(pretrained=True).eval()
#model=torch.load('/data1/zyh/copycat/Framework/cifar_model.pth')
model =AlexNet()
path = "./cifar_net.pth"
#path = '/data1/zyh/copycat/Framework/cifar_model.pth'
#model.load_state_dict(torch.load('/data1/zyh/copycat/Framework/cifar_model.pth'))
#pretrained_dict = {k: v for k, v in model_pretrained.items() if k in model_dict}
#model_dict.update(pretrained_dict)
#model.load_state_dict(state_dict)
model.load_state_dict(torch.load(path),strict=True)
model.eval()
print(type(model))
#preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3)
preprocessing = dict(mean=[0.5]*3, std=[0.5]*3, axis=-3)
fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
#test_dataset = torchvision.datasets.CIFAR10(root='~/.torch/',
# train=True,
# #transform = transforms.Compose([transforms.Resize((256,256)),transforms.ToTensor()]),
# transform = transforms.Compose([transforms.ToTensor()]),
# download=True)
#test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
# batch_size=128, #该参数表示每次读取的批样本个数
# shuffle=False) #该参数表示读取时是否打乱样本顺序
# # 创建迭代器
#data_iter = iter(test_loader)
#images, labels = next(data_iter)
# 当迭代开始时, 队列和线程开始读取数据
#images, labels = data_iter.next()
#images=images.to(device)
#labels=labels.to(device)
#im=images
#images=im.resize(100,3,128,128)
images, labels = ep.astensors(*samples(fmodel, dataset="cifar10", batchsize=16))
#images, labels = ep.astensors(*samples(fmodel, dataset="imagenet", batchsize=16))
#print(images.shape)
clean_acc = accuracy(fmodel, images, labels)
print(f"clean accuracy: {clean_acc * 100:.1f} %")
# apply the attack
attack = LinfPGD()
'''epsilons = [
0.0,
0.0002,
0.0005,
0.0008,
0.001,
0.0015,
0.002,
0.003,
0.01,
0.1,
0.3,
0.5,
1.0,
]'''
epsilons = [
0.0005,
0.001,
0.002,
0.01,
0.1,
]
raw_advs, clipped_advs, success = attack(fmodel, images, labels, epsilons=epsilons)
print(type(raw_advs))
print("atest")
# calculate and report the robust accuracy (the accuracy of the model when
# it is attacked)
robust_accuracy = 1 - success.float32().mean(axis=-1)
print("robust accuracy for perturbations with")
for eps, acc in zip(epsilons, robust_accuracy):
print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %")
# we can also manually check this
# we will use the clipped advs instead of the raw advs, otherwise
# we would need to check if the perturbation sizes are actually
# within the specified epsilon bound
print()
print("we can also manually check this:")
print()
print("robust accuracy for perturbations with")
for eps, advs_ in zip(epsilons, clipped_advs):
acc2 = accuracy(fmodel, advs_, labels)
print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %")
print(" perturbation sizes:")
perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2, 3)).numpy()
print(" ", str(perturbation_sizes).replace("\n", "\n" + " "))
if acc2 == 0:
break
fig = plt.gcf()
os.makedirs("./image/",exist_ok=True)
for i in range(len(raw_advs)):
img_v = raw_advs[i].raw
torchvision.utils.save_image(img_v, './image/'+str(i) +'.png')
import tensorflow as tf
import eagerpy as ep
from foolbox import TensorFlowModel, accuracy, samples
from foolbox.attacks import LinfPGD
#pre and advs, _,
if __name__ == '__main__':
#instantiate the model
model = tf.keras.applications.MobileNet(weights="imagenet")
pre = dict(flip_axis=-1, mean=[104.0, 116.0, 123.0]) #RGB to BGR
fmodel = TensorFlowModel(model, bounds=(0, 255), preprocessing=pre)
#get data and test the model
#wrapping the tensors with ep.astensors is optional, but it allows
#us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
print(accuracy(fmodel, images, labels))
#apply the attack
attack = LinfPGD()
epsilons = [0.0, 0.001, 0.01, 0.03, 0.1, 0.3, 0.5, 1.0]
advs, _, success = attack(fmodel, images, labels, epsilons=epsilons)
#print("This is the success", success)
#calculate and report the robust accuracy
robust_accuracy = 1 - success.float32().mean(axis=-1)
for eps, acc in zip(epsilons, robust_accuracy):
print(eps, acc.item())
model.fit(x_train,y_train, epochs=5) model.evaluate(x_test,y_test,verbose=2) #instantiate the model fmodel=TensorFlowModel(model, bounds=(0,1)) #get data and test the model #wrapping the tensors with ep.astensors is optional, but it allows #us to work with EagerPy tensors in the following ########################################################## images, labels = samples(fmodel, dataset="mnist", batchsize=100) images1, labels1=ep.astensors(*samples(fmodel, dataset="mnist", batchsize=100)) print(accuracy(fmodel, images1, labels1)) predict=fmodel(images).numpy() tf.nn.softmax(predict).numpy() correct_pred=tf.math.argmax(predict,1) print(correct_pred) #print(images) images_arr=np.array(images) #print(images_arr) #print(images_arr.shape) #16,28,28,1 #plot images before attack
import eagerpy as ep
from foolbox import TensorFlowModel, accuracy, samples
from foolbox.attacks import LinfPGD
if __name__ == "__main__":
# instantiate a model
model = tf.keras.applications.ResNet50(weights="imagenet")
pre = dict(flip_axis=-1, mean=[104.0, 116.0, 123.0]) # RGB to BGR
fmodel = TensorFlowModel(model, bounds=(0, 255), preprocessing=pre)
# get data and test the model
# wrapping the tensors with ep.astensors is optional, but it allows
# us to work with EagerPy tensors in the following
images, labels = ep.astensors(
*samples(fmodel, dataset="imagenet", batchsize=16))
print(accuracy(fmodel, images, labels))
# apply the attack
attack = LinfPGD()
epsilons = [0.0, 0.001, 0.01, 0.03, 0.1, 0.3, 0.5, 1.0]
advs, _, success = attack(fmodel, images, labels, epsilons=epsilons)
# calculate and report the robust accuracy
robust_accuracy = 1 - success.float32().mean(axis=-1)
for eps, acc in zip(epsilons, robust_accuracy):
print(eps, acc.item())
# we can also manually check this
for eps, advs_ in zip(epsilons, advs):
print(eps, accuracy(fmodel, advs_, labels))
# but then we also need to look at the perturbation sizes
