def test_untargeted_attacks( fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription, attack_test_target: AttackTestTarget, ) -> None: (fmodel, x, y), real, low_dimensional_input = fmodel_and_data_ext_for_attacks if attack_test_target.requires_real_model and not real: pytest.skip() if attack_test_target.requires_low_dimensional_input and not low_dimensional_input: pytest.skip() if isinstance(x, ep.NumPyTensor) and attack_test_target.uses_grad: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) acc = fbn.accuracy(fmodel, x, y) assert acc > 0 # repeat stochastic attacks three times before we mark them as failed attack_repetitions = 3 if attack_test_target.stochastic_attack else 1 for _ in range(attack_repetitions): advs, _, _ = attack_test_target.attack( fmodel, x, y, epsilons=attack_test_target.epsilon ) adv_acc = fbn.accuracy(fmodel, advs, y) if adv_acc < acc: break assert adv_acc < acc
def test_fast_minimum_norm_untargeted_attack( fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription, attack_and_p: Tuple[FMNAttackLp, Union[int, float]], ) -> None: (fmodel, x, y), real, low_dimensional_input = fmodel_and_data_ext_for_attacks if isinstance(x, ep.NumPyTensor): pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) init_attack = fa.DatasetAttack() init_attack.feed(fmodel, x) init_advs = init_attack.run(fmodel, x, y) attack, p = attack_and_p advs = attack.run(fmodel, x, y, starting_points=init_advs) init_norms = ep.norms.lp(flatten(init_advs - x), p=p, axis=-1) norms = ep.norms.lp(flatten(advs - x), p=p, axis=-1) is_smaller = norms < init_norms assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y) assert fbn.accuracy(fmodel, advs, y) <= fbn.accuracy(fmodel, init_advs, y) assert is_smaller.any()
def test_pointwise_untargeted_attack( request: Any, fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription, attack: fa.PointwiseAttack, ) -> None: (fmodel, x, y), real, low_dimensional_input = fmodel_and_data_ext_for_attacks if not low_dimensional_input or not real: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) init_attack = fa.SaltAndPepperNoiseAttack(steps=50) init_advs = init_attack.run(fmodel, x, y) advs = attack.run(fmodel, x, y, starting_points=init_advs) init_norms_l0 = ep.norms.lp(flatten(init_advs - x), p=0, axis=-1) norms_l0 = ep.norms.lp(flatten(advs - x), p=0, axis=-1) init_norms_l2 = ep.norms.lp(flatten(init_advs - x), p=2, axis=-1) norms_l2 = ep.norms.lp(flatten(advs - x), p=2, axis=-1) is_smaller_l0 = norms_l0 < init_norms_l0 is_smaller_l2 = norms_l2 < init_norms_l2 assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y) assert fbn.accuracy(fmodel, advs, y) <= fbn.accuracy(fmodel, init_advs, y) assert is_smaller_l2.any() assert is_smaller_l0.any()
def test_brendel_bethge_untargeted_attack( request: Any, fmodel_and_data_ext_for_attacks: ModelDescriptionAndData, attack_and_p: Tuple[BrendelBethgeAttack, Union[int, float]], ) -> None: if request.config.option.skipslow: pytest.skip() (fmodel, x, y), real = fmodel_and_data_ext_for_attacks if isinstance(x, ep.NumPyTensor): pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) init_attack = fa.DatasetAttack() init_attack.feed(fmodel, x) init_advs = init_attack.run(fmodel, x, y) attack, p = attack_and_p advs = attack.run(fmodel, x, y, starting_points=init_advs) init_norms = ep.norms.lp(flatten(init_advs - x), p=p, axis=-1) norms = ep.norms.lp(flatten(advs - x), p=p, axis=-1) is_smaller = norms < init_norms assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y) assert fbn.accuracy(fmodel, advs, y) <= fbn.accuracy(fmodel, init_advs, y) assert is_smaller.any()
def test_dataset_attack( fmodel_and_data_ext_for_attacks: ModelDescriptionAndData, ) -> None: (fmodel, x, y), _ = fmodel_and_data_ext_for_attacks x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) attack = fbn.attacks.DatasetAttack() attack.feed(fmodel, x) assert fbn.accuracy(fmodel, x, y) > 0 advs, _, success = attack(fmodel, x, y, epsilons=None) assert success.shape == (len(x), ) assert success.all() assert fbn.accuracy(fmodel, advs, y) == 0 with pytest.raises(ValueError, match="unknown distance"): attack(fmodel, x, y, epsilons=[500.0, 1000.0]) attack = fbn.attacks.DatasetAttack(distance=fbn.distances.l2) attack.feed(fmodel, x) advss, _, success = attack(fmodel, x, y, epsilons=[500.0, 1000.0]) assert success.shape == (2, len(x)) assert success.all() assert fbn.accuracy(fmodel, advss[0], y) == 0 assert fbn.accuracy(fmodel, advss[1], y) == 0 with pytest.raises(TypeError, match="unexpected keyword argument"): attack(fmodel, x, y, epsilons=None, invalid=True)
def test_accuracy(fmodel_and_data: ModelAndData) -> None: fmodel, x, y = fmodel_and_data accuracy = fbn.accuracy(fmodel, x, y) assert 0 <= accuracy <= 1 assert accuracy > 0.5 y = fmodel(x).argmax(axis=-1) accuracy = fbn.accuracy(fmodel, x, y) assert accuracy == 1
def test_binarization_attack( fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription, ) -> None: # get a model with thresholding (fmodel, x, y), _, low_dimensional_input = fmodel_and_data_ext_for_attacks # binarization doesn't work well for imagenet models if not low_dimensional_input: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) fmodel = ThresholdingWrapper(fmodel, threshold=0.5) acc = accuracy(fmodel, x, y) assert acc > 0 # find some adversarials and check that they are non-trivial attack = BinarySearchContrastReductionAttack(target=0) advs, _, _ = attack(fmodel, x, y, epsilons=None) assert accuracy(fmodel, advs, y) < acc # run the refinement attack attack2 = BinarizationRefinementAttack(threshold=0.5, included_in="upper") advs2, _, _ = attack2(fmodel, x, y, starting_points=advs, epsilons=None) # make sure the predicted classes didn't change assert (fmodel(advs).argmax(axis=-1) == fmodel(advs2).argmax(axis=-1)).all() # make sure the perturbations didn't get larger and some got smaller norms1 = flatten(advs - x).norms.l2(axis=-1) norms2 = flatten(advs2 - x).norms.l2(axis=-1) assert (norms2 <= norms1).all() assert (norms2 < norms1).any() # run the refinement attack attack2 = BinarizationRefinementAttack(included_in="upper") advs2, _, _ = attack2(fmodel, x, y, starting_points=advs, epsilons=None) # make sure the predicted classes didn't change assert (fmodel(advs).argmax(axis=-1) == fmodel(advs2).argmax(axis=-1)).all() # make sure the perturbations didn't get larger and some got smaller norms1 = flatten(advs - x).norms.l2(axis=-1) norms2 = flatten(advs2 - x).norms.l2(axis=-1) assert (norms2 <= norms1).all() assert (norms2 < norms1).any() with pytest.raises(ValueError, match="starting_points"): attack2(fmodel, x, y, epsilons=None) attack2 = BinarizationRefinementAttack(included_in="lower") with pytest.raises(ValueError, match="does not match"): attack2(fmodel, x, y, starting_points=advs, epsilons=None) attack2 = BinarizationRefinementAttack(included_in="invalid") # type: ignore with pytest.raises(ValueError, match="expected included_in"): attack2(fmodel, x, y, starting_points=advs, epsilons=None)
def test_evaluate( fmodel_and_data: Tuple[fbn.Model, ep.Tensor, ep.Tensor]) -> None: pytest.skip() assert False fmodel, x, y = fmodel_and_data # type: ignore attacks = [ # L2BasicIterativeAttack, # L2CarliniWagnerAttack, # L2ContrastReductionAttack, # BinarySearchContrastReductionAttack, # LinearSearchContrastReductionAttack, ] epsilons = [0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0] acc = fbn.accuracy(fmodel, x, y) assert acc > 0 _, robust_accuracy = fbn.evaluate_l2(fmodel, x, y, attacks=attacks, epsilons=epsilons) assert robust_accuracy[0] == acc assert robust_accuracy[-1] == 0.0
def main() -> None: # instantiate a model (could also be a TensorFlow or JAX model) model = models.resnet18(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) # get data and test the model # wrapping the tensors with ep.astensors is optional, but it allows # us to work with EagerPy tensors in the following images, labels = ep.astensors( *samples(fmodel, dataset="imagenet", batchsize=16)) clean_acc = accuracy(fmodel, images, labels) * 100 print(f"clean accuracy: {clean_acc:.1f} %") # the attack trys a combination of specified rotations and translations to an image # stops early if adversarial shifts and translations for all images are found attack = fa.SpatialAttack( max_translation=6, # 6px so x in [x-6, x+6] and y in [y-6, y+6] num_translations=6, # number of translations in x, y. max_rotation=20, # +- rotation in degrees num_rotations=5, # number of rotations # max total iterations = num_rotations * num_translations**2 ) # report the success rate of the attack (percentage of samples that could # be adversarially perturbed) and the robust accuracy (the remaining accuracy # of the model when it is attacked) xp_, _, success = attack(fmodel, images, labels) suc = success.float32().mean().item() * 100 print(f"attack success: {suc:.1f} %" " (for the specified rotation and translation bounds)") print(f"robust accuracy: {100 - suc:.1f} %" " (for the specified rotation and translation bounds)")
def validate(fmodel, val_loader, attack, epsilons): clean_acc_list = list() robust_acc_list = list() for i, (images, labels) in enumerate(tqdm(val_loader)): images = images.cuda() images = ep.PyTorchTensor(images) labels = labels.cuda() labels = ep.PyTorchTensor(labels) # accuracy(fmodel, images, labels) clean_acc = accuracy(fmodel, images, labels) clean_acc_list.append(clean_acc * 100) print(clean_acc * 100) raw_advs, clipped_advs, success = attack(fmodel, images, labels, epsilons=epsilons) robust_accuracy = 1 - success.float32().mean(axis=-1) robust_acc_list.append(robust_accuracy.raw) return np.mean(clean_acc_list), torch.stack(robust_acc_list).mean( dim=0).tolist()
def exp_balckbox_attack(): model = model_name(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) total_success = 0 widgets = [ 'train :', Percentage(), ' ', Bar('#'), ' ', Timer(), ' ', ETA(), ' ', FileTransferSpeed() ] pbar = ProgressBar(widgets=widgets) length = len(train_loader) print("length train loader: ", length) total_imgs = 0 for batch_data in pbar(train_loader): images, labels = batch_data['image'].to( DEVICE), batch_data['label_idx'].to(DEVICE) acc = accuracy(fmodel, images, labels) print(images.shape[0]) total_imgs += images.shape[0] total_success += int(acc * images.shape[0]) print("success attack is ", str(total_imgs - total_success), "/" + str(total_imgs)) print("watermark attack rate is: ", str(1 - total_success / total_imgs))
def test_loading_model(request: Any, url: str) -> None: backend = request.config.option.backend if backend != "tensorflow": pytest.skip() # download model try: fmodel = fbn.zoo.get_model(url, name="MobileNetV2", overwrite=True) except fbn.zoo.GitCloneError: pytest.skip() # download again (test overwriting) try: fmodel = fbn.zoo.get_model(url, name="MobileNetV2", overwrite=True) except fbn.zoo.GitCloneError: pytest.skip() # create a dummy image # x = np.zeros(dim, dtype=np.float32) # x[:] = np.random.randn(*x.shape) x, y = fbn.samples(fmodel, dataset="imagenet", batchsize=16) # run the model # logits = model(x) # probabilities = ep.softmax(logits) # predicted_class = np.argmax(logits) assert fbn.accuracy(fmodel, x, y) > 0.9
def main(): parser = argparse.ArgumentParser() parser.add_argument('--steps', type=int, default=20000, help='Iteration of BA') parser.add_argument('--targeted', action='store', default=False, help='For targeted attack') args = parser.parse_args() model = Net() model.load_state_dict(torch.load('mnist_cnn.pt')) model.eval() preprocessing = dict(mean=0.1307, std=0.3081) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) fmodel = fmodel.transform_bounds((0, 1)) assert fmodel.bounds == (0, 1) images, labels = ep.astensors( *samples(fmodel, dataset="mnist", batchsize=10)) print('Model accuracy on clean examples: {}'.format( accuracy(fmodel, images, labels))) if args.targeted: target_class = (labels + 7) % 10 criterion = fb.criteria.TargetedMisclassification(target_class) else: criterion = fb.criteria.Misclassification(labels) attack = fa.BoundaryAttack(steps=args.steps, tensorboard=None) epsilons = np.linspace(0.01, 10, 20) raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons) robust_accuracy = 1 - success.float32().mean(axis=-1) plt.plot(epsilons, robust_accuracy.numpy()) plt.xlabel("Epsilons") plt.ylabel("Robust Accuracy") plt.savefig('mnist_BA_robust_acc.jpg') plt.show() mean_distance = [] for i in range(len(clipped)): dist = np.mean(fb.distances.l2(clipped[i], images).numpy()) mean_distance.append(dist) plt.plot(epsilons, mean_distance) plt.xlabel('Epsilons') plt.ylabel('Mean L2 distance') plt.savefig("mnist_BA_mean_L2distance.jpg") plt.show()
def test_spatial_attacks( fmodel_and_data_ext_for_attacks: ModelDescriptionAndData, attack_grad_real: Tuple[fbn.Attack, bool], ) -> None: attack, repeated = attack_grad_real if repeated: attack = attack.repeat(2) (fmodel, x, y), real = fmodel_and_data_ext_for_attacks if not real: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) acc = fbn.accuracy(fmodel, x, y) assert acc > 0 advs, _, _ = attack(fmodel, x, y) # type: ignore assert fbn.accuracy(fmodel, advs, y) < acc
def test_untargeted_attacks( fmodel_and_data_ext_for_attacks: Tuple[Tuple[fbn.Model, ep.Tensor, ep.Tensor], bool], attack_eps_grad_real: Tuple[fbn.Attack, Optional[float], bool, bool], ) -> None: attack, eps, attack_uses_grad, requires_real_model = attack_eps_grad_real (fmodel, x, y), real = fmodel_and_data_ext_for_attacks if requires_real_model and not real: pytest.skip() if isinstance(x, ep.NumPyTensor) and attack_uses_grad: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) acc = fbn.accuracy(fmodel, x, y) assert acc > 0 advs, _, _ = attack(fmodel, x, y, epsilons=eps) assert fbn.accuracy(fmodel, advs, y) < acc
def test_untargeted_attacks( fmodel_and_data_ext_for_attacks: ModelDescriptionAndData, attack_test_target: AttackTestTarget, ) -> None: (fmodel, x, y), real = fmodel_and_data_ext_for_attacks if attack_test_target.requires_real_model and not real: pytest.skip() if isinstance(x, ep.NumPyTensor) and attack_test_target.uses_grad: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) acc = fbn.accuracy(fmodel, x, y) assert acc > 0 advs, _, _ = attack_test_target.attack(fmodel, x, y, epsilons=attack_test_target.epsilon) assert fbn.accuracy(fmodel, advs, y) < acc
def main(): parser = argparse.ArgumentParser() parser.add_argument('--steps', type=int, default=10000, help='Iteration of BA') parser.add_argument('--targeted', action='store', default=False, help='For targeted attack') args = parser.parse_args() model = Net() model.load_state_dict(torch.load('mnist_cnn.pt')) model.eval() preprocessing = dict(mean=0.1307, std=0.3081) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) fmodel = fmodel.transform_bounds((0, 1)) assert fmodel.bounds == (0, 1) images, labels = ep.astensors( *samples(fmodel, dataset="mnist", batchsize=10)) print('Model accuracy on clean examples: {}'.format( accuracy(fmodel, images, labels))) epsilons = np.linspace(0.01, 10, 20) boundary_attack = fa.BoundaryAttack(steps=args.steps, tensorboard=None) _, _, ba_success = boundary_attack(fmodel, images, labels, epsilons=epsilons) ba_robust_accuracy = 1 - ba_success.float32().mean(axis=-1) random_attack = fa.L2RepeatedAdditiveGaussianNoiseAttack( repeats=args.steps) _, _, ra_success = random_attack(fmodel, images, labels, epsilons=epsilons) ra_robust_accuracy = 1 - ra_success.float32().mean(axis=-1) legends = ["Boundary Attack", "Random Attack"] plt.plot(epsilons, ba_robust_accuracy.numpy()) plt.plot(epsilons, ra_robust_accuracy.numpy()) plt.legend(legends, loc='upper right') plt.xlabel("Perturbation Norm (L2)") plt.ylabel("Robust Accuracy") plt.title("{} Queries".format(args.steps)) plt.savefig('mnist_robust_acc.jpg') plt.show()
def test_pointwise_targeted_attack( request: Any, fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription, attack: fa.PointwiseAttack, ) -> None: (fmodel, x, y), real, low_dimensional_input = fmodel_and_data_ext_for_attacks if not low_dimensional_input or not real: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) init_attack = fa.SaltAndPepperNoiseAttack(steps=50) init_advs = init_attack.run(fmodel, x, y) logits = fmodel(init_advs) num_classes = logits.shape[-1] target_classes = logits.argmax(-1) target_classes = ep.where(target_classes == y, (target_classes + 1) % num_classes, target_classes) criterion = fbn.TargetedMisclassification(target_classes) advs = attack.run(fmodel, x, criterion, starting_points=init_advs) init_norms_l0 = ep.norms.lp(flatten(init_advs - x), p=0, axis=-1) norms_l0 = ep.norms.lp(flatten(advs - x), p=0, axis=-1) init_norms_l2 = ep.norms.lp(flatten(init_advs - x), p=2, axis=-1) norms_l2 = ep.norms.lp(flatten(advs - x), p=2, axis=-1) is_smaller_l0 = norms_l0 < init_norms_l0 is_smaller_l2 = norms_l2 < init_norms_l2 assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y) assert fbn.accuracy(fmodel, advs, y) <= fbn.accuracy(fmodel, init_advs, y) assert fbn.accuracy(fmodel, advs, target_classes) > fbn.accuracy( fmodel, x, target_classes) assert fbn.accuracy(fmodel, advs, target_classes) >= fbn.accuracy( fmodel, init_advs, target_classes) assert is_smaller_l2.any() assert is_smaller_l0.any()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--steps', type=int, default=1000, help='Maximum number of steps to perform') parser.add_argument('--targeted', action='store', default=False, help='For targeted attack') args = parser.parse_args() model = Net() model.load_state_dict(torch.load('mnist_cnn.pt')) model.eval() preprocessing = dict(mean=0.1307, std=0.3081) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) fmodel = fmodel.transform_bounds((0, 1)) assert fmodel.bounds == (0, 1) images, labels = ep.astensors( *samples(fmodel, dataset="mnist", batchsize=10)) print('Model accuracy on clean examples: {}'.format( accuracy(fmodel, images, labels))) if args.targeted: target_class = (labels + 7) % 10 criterion = fb.criteria.TargetedMisclassification(target_class) else: criterion = fb.criteria.Misclassification(labels) attack = fa.L2DeepFoolAttack(steps=args.steps) epsilons = None raw, clipped, success = attack(fmodel, images, labels, epsilons=epsilons) robust_accuracy = 1 - success.float32().mean() print("Robust Accuracy", robust_accuracy.item()) dist = np.mean(fb.distances.l2(clipped, images).numpy()) print("Average perturbation norm", dist)
def main() -> None: print(DEVICE) # instantiate a model (could also be a TensorFlow or JAX model) model = model_name(pretrained=True).eval() # preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) # image_mean = torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1) # image_std = torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1) preprocessing = dict(mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) total_success = 0 # headers = ['img', 'advIndex', 'alpha ', 'angle', 'class', 'l2'] # with open('result_inception_v3_gen4_30.csv', 'w')as f: # f_csv = csv.writer(f) # f_csv.writerow(headers) attack = advDigitalMark() # SpatialAttack() epoch = 0 length = len(train_loader) print("length train loader: ", length) total_msg = '' for batch_data in train_loader: # if epoch == 1: # break images, labels, filenames = batch_data['image'].to( DEVICE), batch_data['label_idx'].to(DEVICE), batch_data['filename'] clean_acc = accuracy(fmodel, images, labels) logging.info("\nepoch " + str(epoch) + ":\nclean accuracy: " + str(clean_acc)) total_msg += "\nepoch " + str(epoch) + ":\nclean accuracy: " + str( clean_acc) count, msg = attack.run(fmodel, images, labels, filenames=filenames) total_success += count total_msg += msg logging.info("current success attack is " + str(total_success)) send_email(total_msg, title='fsy天池比赛V1模型3000-5000结果报告') epoch += 1 total_msg += "\n total attack success img is " + str(total_success) # logging.info("total succes is "+str(total_success)+"/100") send_email(total_msg, title='fsy天池比赛V1模型3000-5000结果报告')
def test_hsj_targeted_attack( request: Any, fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription, attack_and_p: Tuple[fa.HopSkipJumpAttack, Union[int, float]], ) -> None: if request.config.option.skipslow: pytest.skip() (fmodel, x, y), real, _ = fmodel_and_data_ext_for_attacks if isinstance(x, ep.NumPyTensor): pytest.skip() if not real: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) logits_np = fmodel(x).numpy() num_classes = logits_np.shape[-1] y_np = logits_np.argmax(-1) target_classes_np = (y_np + 1) % num_classes for i in range(len(target_classes_np)): while target_classes_np[i] not in y_np: target_classes_np[i] = (target_classes_np[i] + 1) % num_classes target_classes = ep.from_numpy(y, target_classes_np) criterion = fbn.TargetedMisclassification(target_classes) init_attack = fa.DatasetAttack() init_attack.feed(fmodel, x) init_advs = init_attack.run(fmodel, x, criterion) attack, p = attack_and_p advs = attack.run(fmodel, x, criterion, starting_points=init_advs) init_norms = ep.norms.lp(flatten(init_advs - x), p=p, axis=-1) norms = ep.norms.lp(flatten(advs - x), p=p, axis=-1) is_smaller = norms < init_norms assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y) assert fbn.accuracy(fmodel, advs, target_classes) > fbn.accuracy( fmodel, x, target_classes) assert fbn.accuracy(fmodel, advs, target_classes) >= fbn.accuracy( fmodel, init_advs, target_classes) assert is_smaller.any()
def exp_otherAttack(): model = model_name(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) total_success = 0 attack = SpatialAttack() epoch = 0 widgets = [ 'train :', Percentage(), ' ', Bar('#'), ' ', Timer(), ' ', ETA(), ' ', FileTransferSpeed() ] pbar = ProgressBar(widgets=widgets) length = len(train_loader) print("length train loader: ", length) for batch_data in pbar(train_loader): if epoch == 25: break images, labels = batch_data['image'].to( DEVICE), batch_data['label_idx'].to(DEVICE) clean_acc = accuracy(fmodel, images, labels) print(f"\nepoch " + str(epoch) + ":\nclean accuracy: ", clean_acc) epsilons = [1] _, _, isSuceess = attack(fmodel, images, labels, epsilons=epsilons) isSuceess = isSuceess.cpu().numpy() total_success += np.sum(isSuceess == True) print("current success : ", total_success) # total_success += int(adv_acc*10) epoch += 1 print("success attack is ", str(total_success))
def exp1(): print(DEVICE) parser = argparse.ArgumentParser( description='CIFAR10 Training against DDN Attack') parser.add_argument('--data', default='../defenses/data/cifar10', help='path to dataset') parser.add_argument('--workers', default=2, type=int, help='number of data loading workers') parser.add_argument('--cpu', action='store_true', help='force training on cpu') parser.add_argument('--save-folder', '--sf', default='weights/cifar10_inception_v3/', help='folder to save state dicts') parser.add_argument('--save-freq', '--sfr', default=10, type=int, help='save frequency') parser.add_argument('--save-name', '--sn', default='cifar10', help='name for saving the final state dict') parser.add_argument('--batch-size', '-b', default=20, type=int, help='mini-batch size') parser.add_argument('--epochs', '-e', default=100, type=int, help='number of total epochs to run') parser.add_argument('--lr', '--learning-rate', default=0.001, type=float, help='initial learning rate') parser.add_argument('--lr-decay', '--lrd', default=0.2, type=float, help='decay for learning rate') parser.add_argument('--lr-step', '--lrs', default=10, type=int, help='step size for learning rate decay') parser.add_argument('--momentum', default=0.9, type=float, help='momentum') parser.add_argument('--weight-decay', '--wd', default=5e-4, type=float, help='weight decay') parser.add_argument('--drop', default=0.3, type=float, help='dropout rate of the classifier') parser.add_argument('--adv', type=int, default=None, help='epoch to start training with adversarial images') parser.add_argument('--max-norm', type=float, default=1, help='max norm for the adversarial perturbations') parser.add_argument('--steps', default=10, type=int, help='number of steps for the attack') parser.add_argument( '--visdom-port', '--vp', type=int, default=8097, help='For visualization, which port visdom is running.') parser.add_argument('--print-freq', '--pf', default=10, type=int, help='print frequency') args = parser.parse_args() print(args) train_transform = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), ]) test_transform = transforms.Compose([ transforms.ToTensor(), ]) train_set = data.Subset( CIFAR10(args.data, train=True, transform=train_transform, download=True), list(range(30000, 50000))) val_set = data.Subset( CIFAR10(args.data, train=True, transform=test_transform, download=True), list(range(48000, 50000))) test_set = CIFAR10(args.data, train=False, transform=test_transform, download=True) train_loader = data.DataLoader(train_set, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, drop_last=True, pin_memory=True) val_loader = data.DataLoader(val_set, batch_size=100, shuffle=False, num_workers=args.workers, pin_memory=True) test_loader = data.DataLoader(test_set, batch_size=100, shuffle=False, num_workers=args.workers, pin_memory=True) image_mean = torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1) image_std = torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1) m = AlexNet().eval() model = NormalizedModel(model=m, mean=image_mean, std=image_std).to( DEVICE) # keep images in the [0, 1] range model_file = '/home/frankfeng/researchData/code/adversarial_training_code/PLP/fast_adv/defenses/weights/cifar10_AlexNet/cifar10_valacc0.8019999772310257.pth' model_dict = torch.load(model_file) model.load_state_dict(model_dict) # preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) # image_mean = torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1) # image_std = torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1) preprocessing = dict(mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1)) total_success = 0 # headers = ['img', 'advIndex', 'alpha ', 'angle', 'class', 'l2'] # with open('result_inception_v3_gen4_30.csv', 'w')as f: # f_csv = csv.writer(f) # f_csv.writerow(headers) attack = advDigitalMark() # SpatialAttack() epoch = 0 widgets = [ 'train :', Percentage(), ' ', Bar('#'), ' ', Timer(), ' ', ETA(), ' ', FileTransferSpeed() ] pbar = ProgressBar(widgets=widgets) length = len(train_loader) print("length train loader: ", length) for epoch in range(args.epochs): if epoch == 25: break for i, (images, labels) in enumerate(tqdm.tqdm(train_loader, ncols=80)): images, labels = images.to(DEVICE), labels.to(DEVICE) clean_acc = accuracy(fmodel, images, labels) print(f"\nepoch " + str(epoch) + ":\nclean accuracy: ", clean_acc) count = attack.run(fmodel, images, labels) total_success += count epoch += 1 print("current success attack is ", str(total_success)) print( str(model_name) + "_" + str(max_gen) + "_" + str(max_gen) + ": total attack success img is ", str(total_success))
import numpy as np if __name__ == "__main__": # instantiate a model (could also be a TensorFlow or JAX model) model = models.resnet18(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) # get data and test the model # wrapping the tensors with ep.astensors is optional, but it allows # us to work with EagerPy tensors in the following images, labels = ep.astensors( *samples(fmodel, dataset="imagenet", batchsize=16)) clean_acc = accuracy(fmodel, images, labels) print(f"clean accuracy: {clean_acc * 100:.1f} %") print("") attacks = [ fa.FGSM(), fa.LinfPGD(), fa.LinfBasicIterativeAttack(), fa.LinfAdditiveUniformNoiseAttack(), fa.LinfDeepFoolAttack(), ] epsilons = [ 0.0, 0.0005, 0.001,
def main() -> None: # instantiate a model (could also be a TensorFlow or JAX model) model = models.resnet18(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) # get data and test the model # wrapping the tensors with ep.astensors is optional, but it allows # us to work with EagerPy tensors in the following images, labels = ep.astensors( *samples(fmodel, dataset="imagenet", batchsize=16)) clean_acc = accuracy(fmodel, images, labels) print(f"clean accuracy: {clean_acc * 100:.1f} %") # replace the gradient with the gradient from another model model2 = fmodel # demo, we just use the same model # TODO: this is still a bit annoying because we need # to overwrite run to get the labels class Attack(LinfPGD): def value_and_grad(self, loss_fn, x): val1 = loss_fn(x) loss_fn2 = self.get_loss_fn(model2, self.labels) _, grad2 = ep.value_and_grad(loss_fn2, x) return val1, grad2 def run(self, model, inputs, criterion, *, epsilon, **kwargs): criterion_ = get_criterion(criterion) self.labels = criterion_.labels return super().run(model, inputs, criterion_, epsilon=epsilon, **kwargs) # apply the attack attack = Attack() epsilons = [ 0.0, 0.0002, 0.0005, 0.0008, 0.001, 0.0015, 0.002, 0.003, 0.01, 0.1, 0.3, 0.5, 1.0, ] raw_advs, clipped_advs, success = attack(fmodel, images, labels, epsilons=epsilons) # calculate and report the robust accuracy (the accuracy of the model when # it is attacked) robust_accuracy = 1 - success.float32().mean(axis=-1) print("robust accuracy for perturbations with") for eps, acc in zip(epsilons, robust_accuracy): print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %") # we can also manually check this # we will use the clipped advs instead of the raw advs, otherwise # we would need to check if the perturbation sizes are actually # within the specified epsilon bound print() print("we can also manually check this:") print() print("robust accuracy for perturbations with") for eps, advs_ in zip(epsilons, clipped_advs): acc2 = accuracy(fmodel, advs_, labels) print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %") print(" perturbation sizes:") perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2, 3)).numpy() print(" ", str(perturbation_sizes).replace("\n", "\n" + " ")) if acc2 == 0: break
def main() -> None: # instantiate a model (could also be a TensorFlow or JAX model) model = models.resnet18(pretrained=True).eval() preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) # get data and test the model # wrapping the tensors with ep.astensors is optional, but it allows # us to work with EagerPy tensors in the following images, labels = ep.astensors( *samples(fmodel, dataset="imagenet", batchsize=16)) clean_acc = accuracy(fmodel, images, labels) print(f"clean accuracy: {clean_acc * 100:.1f} %") # apply the attack attack = LinfPGD() epsilons = [ 0.0, 0.0002, 0.0005, 0.0008, 0.001, 0.0015, 0.002, 0.003, 0.01, 0.1, 0.3, 0.5, 1.0, ] raw_advs, clipped_advs, success = attack(fmodel, images, labels, epsilons=epsilons) # calculate and report the robust accuracy (the accuracy of the model when # it is attacked) robust_accuracy = 1 - success.float32().mean(axis=-1) print("robust accuracy for perturbations with") for eps, acc in zip(epsilons, robust_accuracy): print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %") # we can also manually check this # we will use the clipped advs instead of the raw advs, otherwise # we would need to check if the perturbation sizes are actually # within the specified epsilon bound print() print("we can also manually check this:") print() print("robust accuracy for perturbations with") for eps, advs_ in zip(epsilons, clipped_advs): acc2 = accuracy(fmodel, advs_, labels) print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %") print(" perturbation sizes:") perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2, 3)).numpy() print(" ", str(perturbation_sizes).replace("\n", "\n" + " ")) if acc2 == 0: break
def main() -> None: # instantiate a model (could also be a TensorFlow or JAX model) #model = models.resnet18(pretrained=True).eval() #model=torch.load('/data1/zyh/copycat/Framework/cifar_model.pth') model =AlexNet() path = "./cifar_net.pth" #path = '/data1/zyh/copycat/Framework/cifar_model.pth' #model.load_state_dict(torch.load('/data1/zyh/copycat/Framework/cifar_model.pth')) #pretrained_dict = {k: v for k, v in model_pretrained.items() if k in model_dict} #model_dict.update(pretrained_dict) #model.load_state_dict(state_dict) model.load_state_dict(torch.load(path),strict=True) model.eval() print(type(model)) #preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) preprocessing = dict(mean=[0.5]*3, std=[0.5]*3, axis=-3) fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) # get data and test the model # wrapping the tensors with ep.astensors is optional, but it allows # us to work with EagerPy tensors in the following #test_dataset = torchvision.datasets.CIFAR10(root='~/.torch/', # train=True, # #transform = transforms.Compose([transforms.Resize((256,256)),transforms.ToTensor()]), # transform = transforms.Compose([transforms.ToTensor()]), # download=True) #test_loader = torch.utils.data.DataLoader(dataset=test_dataset, # batch_size=128, #该参数表示每次读取的批样本个数 # shuffle=False) #该参数表示读取时是否打乱样本顺序 # # 创建迭代器 #data_iter = iter(test_loader) #images, labels = next(data_iter) # 当迭代开始时, 队列和线程开始读取数据 #images, labels = data_iter.next() #images=images.to(device) #labels=labels.to(device) #im=images #images=im.resize(100,3,128,128) images, labels = ep.astensors(*samples(fmodel, dataset="cifar10", batchsize=16)) #images, labels = ep.astensors(*samples(fmodel, dataset="imagenet", batchsize=16)) #print(images.shape) clean_acc = accuracy(fmodel, images, labels) print(f"clean accuracy: {clean_acc * 100:.1f} %") # apply the attack attack = LinfPGD() '''epsilons = [ 0.0, 0.0002, 0.0005, 0.0008, 0.001, 0.0015, 0.002, 0.003, 0.01, 0.1, 0.3, 0.5, 1.0, ]''' epsilons = [ 0.0005, 0.001, 0.002, 0.01, 0.1, ] raw_advs, clipped_advs, success = attack(fmodel, images, labels, epsilons=epsilons) print(type(raw_advs)) print("atest") # calculate and report the robust accuracy (the accuracy of the model when # it is attacked) robust_accuracy = 1 - success.float32().mean(axis=-1) print("robust accuracy for perturbations with") for eps, acc in zip(epsilons, robust_accuracy): print(f" Linf norm ≤ {eps:<6}: {acc.item() * 100:4.1f} %") # we can also manually check this # we will use the clipped advs instead of the raw advs, otherwise # we would need to check if the perturbation sizes are actually # within the specified epsilon bound print() print("we can also manually check this:") print() print("robust accuracy for perturbations with") for eps, advs_ in zip(epsilons, clipped_advs): acc2 = accuracy(fmodel, advs_, labels) print(f" Linf norm ≤ {eps:<6}: {acc2 * 100:4.1f} %") print(" perturbation sizes:") perturbation_sizes = (advs_ - images).norms.linf(axis=(1, 2, 3)).numpy() print(" ", str(perturbation_sizes).replace("\n", "\n" + " ")) if acc2 == 0: break fig = plt.gcf() os.makedirs("./image/",exist_ok=True) for i in range(len(raw_advs)): img_v = raw_advs[i].raw torchvision.utils.save_image(img_v, './image/'+str(i) +'.png')
import tensorflow as tf import eagerpy as ep from foolbox import TensorFlowModel, accuracy, samples from foolbox.attacks import LinfPGD #pre and advs, _, if __name__ == '__main__': #instantiate the model model = tf.keras.applications.MobileNet(weights="imagenet") pre = dict(flip_axis=-1, mean=[104.0, 116.0, 123.0]) #RGB to BGR fmodel = TensorFlowModel(model, bounds=(0, 255), preprocessing=pre) #get data and test the model #wrapping the tensors with ep.astensors is optional, but it allows #us to work with EagerPy tensors in the following images, labels = ep.astensors( *samples(fmodel, dataset="imagenet", batchsize=16)) print(accuracy(fmodel, images, labels)) #apply the attack attack = LinfPGD() epsilons = [0.0, 0.001, 0.01, 0.03, 0.1, 0.3, 0.5, 1.0] advs, _, success = attack(fmodel, images, labels, epsilons=epsilons) #print("This is the success", success) #calculate and report the robust accuracy robust_accuracy = 1 - success.float32().mean(axis=-1) for eps, acc in zip(epsilons, robust_accuracy): print(eps, acc.item())
model.fit(x_train,y_train, epochs=5) model.evaluate(x_test,y_test,verbose=2) #instantiate the model fmodel=TensorFlowModel(model, bounds=(0,1)) #get data and test the model #wrapping the tensors with ep.astensors is optional, but it allows #us to work with EagerPy tensors in the following ########################################################## images, labels = samples(fmodel, dataset="mnist", batchsize=100) images1, labels1=ep.astensors(*samples(fmodel, dataset="mnist", batchsize=100)) print(accuracy(fmodel, images1, labels1)) predict=fmodel(images).numpy() tf.nn.softmax(predict).numpy() correct_pred=tf.math.argmax(predict,1) print(correct_pred) #print(images) images_arr=np.array(images) #print(images_arr) #print(images_arr.shape) #16,28,28,1 #plot images before attack
import eagerpy as ep from foolbox import TensorFlowModel, accuracy, samples from foolbox.attacks import LinfPGD if __name__ == "__main__": # instantiate a model model = tf.keras.applications.ResNet50(weights="imagenet") pre = dict(flip_axis=-1, mean=[104.0, 116.0, 123.0]) # RGB to BGR fmodel = TensorFlowModel(model, bounds=(0, 255), preprocessing=pre) # get data and test the model # wrapping the tensors with ep.astensors is optional, but it allows # us to work with EagerPy tensors in the following images, labels = ep.astensors( *samples(fmodel, dataset="imagenet", batchsize=16)) print(accuracy(fmodel, images, labels)) # apply the attack attack = LinfPGD() epsilons = [0.0, 0.001, 0.01, 0.03, 0.1, 0.3, 0.5, 1.0] advs, _, success = attack(fmodel, images, labels, epsilons=epsilons) # calculate and report the robust accuracy robust_accuracy = 1 - success.float32().mean(axis=-1) for eps, acc in zip(epsilons, robust_accuracy): print(eps, acc.item()) # we can also manually check this for eps, advs_ in zip(epsilons, advs): print(eps, accuracy(fmodel, advs_, labels)) # but then we also need to look at the perturbation sizes