def test_accuracy(fmodel_and_data: ModelAndData) -> None:
fmodel, x, y = fmodel_and_data
accuracy = fbn.accuracy(fmodel, x, y)
assert 0 <= accuracy <= 1
assert accuracy > 0.5
y = fmodel(x).argmax(axis=-1)
accuracy = fbn.accuracy(fmodel, x, y)
assert accuracy == 1
def test_untargeted_attacks(
fmodel_and_data: Tuple[fbn.Model, ep.Tensor, ep.Tensor],
attack_and_grad: Tuple[fbn.Attack, bool],
) -> None:
attack, attack_uses_grad = attack_and_grad
fmodel, x, y = fmodel_and_data
if isinstance(x, ep.NumPyTensor) and attack_uses_grad:
pytest.skip()
x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
fmodel = fmodel.transform_bounds((0, 1))
advs = attack(fmodel, x, y)
assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y)
def test_evaluate(fmodel_and_data: Tuple[fbn.Model, ep.Tensor, ep.Tensor]) -> None:
pytest.skip()
assert False
fmodel, x, y = fmodel_and_data # type: ignore
attacks = [
# L2BasicIterativeAttack,
# L2CarliniWagnerAttack,
# L2ContrastReductionAttack,
# BinarySearchContrastReductionAttack,
# LinearSearchContrastReductionAttack,
]
epsilons = [0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0]
acc = fbn.accuracy(fmodel, x, y)
assert acc > 0
_, robust_accuracy = fbn.evaluate_l2(
fmodel, x, y, attacks=attacks, epsilons=epsilons
)
assert robust_accuracy[0] == acc
assert robust_accuracy[-1] == 0.0
def test_inversion_attack(fmodel_and_data):
fmodel, x, y = fmodel_and_data
attack = fbn.attacks.InversionAttack(fmodel)
advs = attack(x, y)
assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y)