def test_genattack_numpy(request: Any) -> None: class Model: def __call__(self, inputs: Any) -> Any: return inputs.mean(axis=(2, 3)) model = Model() with pytest.raises(ValueError): fbn.NumPyModel(model, bounds=(0, 1), data_format="foo") fmodel = fbn.NumPyModel(model, bounds=(0, 1)) x, y = ep.astensors( *fbn.samples( fmodel, dataset="imagenet", batchsize=16, data_format="channels_first" ) ) with pytest.raises(ValueError, match="data_format"): fbn.attacks.GenAttack(reduced_dims=(2, 2)).run( fmodel, x, fbn.TargetedMisclassification(y), epsilon=0.3 ) with pytest.raises(ValueError, match="channel_axis"): fbn.attacks.GenAttack(channel_axis=2, reduced_dims=(2, 2)).run( fmodel, x, fbn.TargetedMisclassification(y), epsilon=0.3 )
def test_wrong_unperturbed( fmodel_and_data: Tuple[fbn.Model, ep.Tensor, ep.Tensor]) -> None: fmodel, inputs, _ = fmodel_and_data perturbed = inputs logits = fmodel(perturbed) _, num_classes = logits.shape labels = logits.argmax(axis=-1) labels = (labels + 1) % num_classes is_adv = fbn.Misclassification(labels)(perturbed, logits) assert is_adv.all() target_classes = (labels + 1) % num_classes is_adv = fbn.TargetedMisclassification(target_classes)(perturbed, logits) if num_classes > 2: assert not is_adv.any() else: assert is_adv.all() is_adv = (fbn.Misclassification(labels) & fbn.Misclassification(labels))( perturbed, logits) assert is_adv.all() combined = fbn.TargetedMisclassification( labels) & fbn.TargetedMisclassification(target_classes) is_adv = combined(perturbed, logits) assert not is_adv.any()
def test_targeted_attacks( fmodel_and_data_ext_for_attacks: ModelDescriptionAndData, attack_test_target: AttackTestTarget, ) -> None: (fmodel, x, y), real = fmodel_and_data_ext_for_attacks if attack_test_target.requires_real_model and not real: pytest.skip() if isinstance(x, ep.NumPyTensor) and attack_test_target.uses_grad: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) num_classes = fmodel(x).shape[-1] target_classes = (y + 1) % num_classes criterion = fbn.TargetedMisclassification(target_classes) adv_before_attack = criterion(x, fmodel(x)) assert not adv_before_attack.all() advs, _, _ = attack_test_target.attack(fmodel, x, criterion, epsilons=attack_test_target.epsilon) adv_after_attack = criterion(advs, fmodel(advs)) assert adv_after_attack.sum().item() > adv_before_attack.sum().item()
def test_targeted_attacks( fmodel_and_data_ext_for_attacks: Tuple[Tuple[fbn.Model, ep.Tensor, ep.Tensor], bool], attack_eps_grad_real: Tuple[fbn.Attack, Optional[float], bool, bool], ) -> None: attack, eps, attack_uses_grad, requires_real_model = attack_eps_grad_real (fmodel, x, y), real = fmodel_and_data_ext_for_attacks if requires_real_model and not real: pytest.skip() if isinstance(x, ep.NumPyTensor) and attack_uses_grad: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) num_classes = fmodel(x).shape[-1] target_classes = (y + 1) % num_classes criterion = fbn.TargetedMisclassification(target_classes) adv_before_attack = criterion(x, fmodel(x)) assert not adv_before_attack.all() advs, _, _ = attack(fmodel, x, criterion, epsilons=eps) adv_after_attack = criterion(advs, fmodel(advs)) assert adv_after_attack.sum().item() > adv_before_attack.sum().item()
def test_fgsm_run_raises( fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription, ) -> None: (fmodel, x, y), _, _ = fmodel_and_data_ext_for_attacks if isinstance(x, ep.NumPyTensor): pytest.skip() with pytest.raises(ValueError, match="unsupported criterion"): attack = fbn.attacks.FGSM() attack.run(fmodel, x, fbn.TargetedMisclassification(y), epsilon=1000)
def test_newtonfool_run_raises( fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription, ) -> None: (fmodel, x, y), _, _ = fmodel_and_data_ext_for_attacks if isinstance(x, ep.NumPyTensor): pytest.skip() with pytest.raises(ValueError, match="unsupported criterion"): attack = fbn.attacks.NewtonFoolAttack() attack.run(fmodel, x, fbn.TargetedMisclassification(y)) with pytest.raises(ValueError, match="expected labels to have shape"): attack = fbn.attacks.NewtonFoolAttack(steps=10) attack.run(fmodel, x, ep.concatenate((y, y), 0))
def test_vat_run_raises( fmodel_and_data_ext_for_attacks: ModelDescriptionAndData, ) -> None: (fmodel, x, y), _ = fmodel_and_data_ext_for_attacks if isinstance(x, ep.NumPyTensor): pytest.skip() with pytest.raises(ValueError, match="unsupported criterion"): attack = fbn.attacks.VirtualAdversarialAttack(steps=10) attack.run(fmodel, x, fbn.TargetedMisclassification(y), epsilon=1.0) with pytest.raises(ValueError, match="expected labels to have shape"): attack = fbn.attacks.VirtualAdversarialAttack(steps=10) attack.run(fmodel, x, ep.concatenate((y, y), 0), epsilon=1.0)
def test_hsj_targeted_attack( request: Any, fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription, attack_and_p: Tuple[fa.HopSkipJumpAttack, Union[int, float]], ) -> None: if request.config.option.skipslow: pytest.skip() (fmodel, x, y), real, _ = fmodel_and_data_ext_for_attacks if isinstance(x, ep.NumPyTensor): pytest.skip() if not real: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) logits_np = fmodel(x).numpy() num_classes = logits_np.shape[-1] y_np = logits_np.argmax(-1) target_classes_np = (y_np + 1) % num_classes for i in range(len(target_classes_np)): while target_classes_np[i] not in y_np: target_classes_np[i] = (target_classes_np[i] + 1) % num_classes target_classes = ep.from_numpy(y, target_classes_np) criterion = fbn.TargetedMisclassification(target_classes) init_attack = fa.DatasetAttack() init_attack.feed(fmodel, x) init_advs = init_attack.run(fmodel, x, criterion) attack, p = attack_and_p advs = attack.run(fmodel, x, criterion, starting_points=init_advs) init_norms = ep.norms.lp(flatten(init_advs - x), p=p, axis=-1) norms = ep.norms.lp(flatten(advs - x), p=p, axis=-1) is_smaller = norms < init_norms assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y) assert fbn.accuracy(fmodel, advs, target_classes) > fbn.accuracy( fmodel, x, target_classes) assert fbn.accuracy(fmodel, advs, target_classes) >= fbn.accuracy( fmodel, init_advs, target_classes) assert is_smaller.any()
def test_pointwise_targeted_attack( request: Any, fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription, attack: fa.PointwiseAttack, ) -> None: (fmodel, x, y), real, low_dimensional_input = fmodel_and_data_ext_for_attacks if not low_dimensional_input or not real: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) init_attack = fa.SaltAndPepperNoiseAttack(steps=50) init_advs = init_attack.run(fmodel, x, y) logits = fmodel(init_advs) num_classes = logits.shape[-1] target_classes = logits.argmax(-1) target_classes = ep.where(target_classes == y, (target_classes + 1) % num_classes, target_classes) criterion = fbn.TargetedMisclassification(target_classes) advs = attack.run(fmodel, x, criterion, starting_points=init_advs) init_norms_l0 = ep.norms.lp(flatten(init_advs - x), p=0, axis=-1) norms_l0 = ep.norms.lp(flatten(advs - x), p=0, axis=-1) init_norms_l2 = ep.norms.lp(flatten(init_advs - x), p=2, axis=-1) norms_l2 = ep.norms.lp(flatten(advs - x), p=2, axis=-1) is_smaller_l0 = norms_l0 < init_norms_l0 is_smaller_l2 = norms_l2 < init_norms_l2 assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y) assert fbn.accuracy(fmodel, advs, y) <= fbn.accuracy(fmodel, init_advs, y) assert fbn.accuracy(fmodel, advs, target_classes) > fbn.accuracy( fmodel, x, target_classes) assert fbn.accuracy(fmodel, advs, target_classes) >= fbn.accuracy( fmodel, init_advs, target_classes) assert is_smaller_l2.any() assert is_smaller_l0.any()
def test_targeted_attacks_call_raises_exception( fmodel_and_data_ext_for_attacks: Tuple[Tuple[fbn.Model, ep.Tensor, ep.Tensor], bool], attack_exception_text_and_grad: Tuple[fbn.Attack, bool], ) -> None: attack, attack_uses_grad = attack_exception_text_and_grad (fmodel, x, y), _ = fmodel_and_data_ext_for_attacks if isinstance(x, ep.NumPyTensor) and attack_uses_grad: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) num_classes = fmodel(x).shape[-1] target_classes = (y + 1) % num_classes invalid_target_classes = ep.concatenate((target_classes, target_classes), 0) invalid_targeted_criterion = fbn.TargetedMisclassification( invalid_target_classes) class DummyCriterion(fbn.Criterion): """Criterion without any functionality which is just meant to be rejected by the attacks """ def __repr__(self) -> str: return "" def __call__(self, perturbed: fbn.criteria.T, outputs: fbn.criteria.T) -> fbn.criteria.T: return perturbed invalid_criterion = DummyCriterion() # check if targeted attack criterion with invalid number of classes is rejected with pytest.raises(ValueError): attack(fmodel, x, invalid_targeted_criterion, epsilons=1000.0) # check if only the two valid criteria are accepted with pytest.raises(ValueError): attack(fmodel, x, invalid_criterion, epsilons=1000.0)
def test_targeted_attacks( fmodel_and_data_ext_for_attacks: ModeAndDataAndDescription, attack_test_target: AttackTestTarget, ) -> None: (fmodel, x, y), real, low_dimensional_input = fmodel_and_data_ext_for_attacks if attack_test_target.requires_real_model and not real: pytest.skip() if attack_test_target.requires_low_dimensional_input and not low_dimensional_input: pytest.skip() if isinstance(x, ep.NumPyTensor) and attack_test_target.uses_grad: pytest.skip() x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower) fmodel = fmodel.transform_bounds((0, 1)) num_classes = fmodel(x).shape[-1] target_classes = (y + 1) % num_classes criterion = fbn.TargetedMisclassification(target_classes) adv_before_attack = criterion(x, fmodel(x)) assert not adv_before_attack.all() asr = adv_before_attack.sum().item() # repeat stochastic attacks three times before we mark them as failed attack_repetitions = 3 if attack_test_target.stochastic_attack else 1 for _ in range(attack_repetitions): advs, _, _ = attack_test_target.attack( fmodel, x, criterion, epsilons=attack_test_target.epsilon ) adv_after_attack = criterion(advs, fmodel(advs)) adv_asr = adv_after_attack.sum().item() if adv_asr > asr: break assert adv_asr > asr
def test_repr_targeted_misclassification(dummy: ep.Tensor) -> None: target_classes = ep.arange(dummy, 10) assert not repr( fbn.TargetedMisclassification(target_classes)).startswith("<")