def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
attack_params = {
"max_iter": 1,
"attacker": "ead",
"attacker_params": {
"max_iter": 5,
"targeted": False
}
}
attack = UniversalPerturbation(classifier)
attack.set_params(**attack_params)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info(
'Accuracy on Iris with universal adversarial examples: %.2f%%',
(acc * 100))
def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
# Test untargeted attack
attack = BasicIterativeMethod(classifier, eps=1, eps_step=0.1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with BIM adversarial examples: %.2f%%',
(acc * 100))
# Test targeted attack
targets = random_targets(y_test, nb_classes=3)
attack = BasicIterativeMethod(classifier,
targeted=True,
eps=1,
eps_step=0.1,
batch_size=128)
x_test_adv = attack.generate(x_test, **{'y': targets})
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any())
acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / y_test.shape[0]
logger.info('Success rate of targeted BIM on Iris: %.2f%%',
(acc * 100))
def test_iris_pt(self):
classifier = get_iris_classifier_pt()
# Test untargeted attack
attack = FastGradientMethod(classifier, eps=.1)
x_test_adv = attack.generate(self.x_test)
self.assertFalse((self.x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test,
axis=1) == predictions_adv).all())
accuracy = np.sum(predictions_adv == np.argmax(
self.y_test, axis=1)) / self.y_test.shape[0]
logger.info('Accuracy on Iris with FGM adversarial examples: %.2f%%',
(accuracy * 100))
# Test targeted attack
targets = random_targets(self.y_test, nb_classes=3)
attack = FastGradientMethod(classifier,
targeted=True,
eps=.1,
batch_size=128)
x_test_adv = attack.generate(self.x_test, **{'y': targets})
self.assertFalse((self.x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == predictions_adv).any())
accuracy = np.sum(predictions_adv == np.argmax(
targets, axis=1)) / self.y_test.shape[0]
logger.info('Success rate of targeted FGM on Iris: %.2f%%',
(accuracy * 100))
def test_failure_feature_vectors(self):
attack_params = {"rotation_max": 22.5, "scale_min": 0.1, "scale_max": 1.0,
"learning_rate": 5.0, "number_of_steps": 5, "patch_shape": (1, 28, 28), "batch_size": 10}
classifier = get_iris_classifier_pt()
data = np.random.rand(10, 4)
# Assert that value error is raised for feature vectors
with self.assertRaises(ValueError) as context:
attack = ZooAttack(classifier=classifier)
attack.set_params(**attack_params)
attack.generate(data)
self.assertIn('Feature vectors detected.', str(context.exception))
def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
attack = CarliniLInfMethod(classifier, targeted=False, max_iter=10, eps=0.5)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with C&W adversarial examples: %.2f%%', (acc * 100))
def test_iris_pt(self):
classifier = get_iris_classifier_pt()
attack = DeepFool(classifier, max_iter=5, batch_size=128)
x_test_adv = attack.generate(self.x_test)
self.assertFalse((self.x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test, axis=1) == predictions_adv).all())
accuracy = np.sum(predictions_adv == np.argmax(self.y_test, axis=1)) / self.y_test.shape[0]
logger.info('Accuracy on Iris with DeepFool adversarial examples: %.2f%%', (accuracy * 100))
def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
attack = VirtualAdversarialMethod(classifier, eps=.1)
with self.assertRaises(TypeError) as context:
x_test_adv = attack.generate(x_test.astype(np.float32))
self.assertIn(
'This attack requires a classifier predicting probabilities in the range [0, 1] as output.'
'Values smaller than 0.0 or larger than 1.0 have been detected.',
str(context.exception))
def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
attack = ElasticNet(classifier, targeted=False, max_iter=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = 1. - np.sum(
preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('EAD success rate on Iris: %.2f%%', (acc * 100))
def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
attack = SaliencyMapMethod(classifier, theta=1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with JSMA adversarial examples: %.2f%%', (acc * 100))
def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
attack = NewtonFool(classifier, max_iter=5, batch_size=128)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with NewtonFool adversarial examples: %.2f%%', (acc * 100))
def test_iris_pt(self):
classifier = get_iris_classifier_pt()
attack = BoundaryAttack(classifier, targeted=False, max_iter=10)
x_test_adv = attack.generate(self.x_test.astype(np.float32))
self.assertFalse((self.x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test, axis=1) == preds_adv).all())
accuracy = np.sum(
preds_adv == np.argmax(self.y_test, axis=1)) / self.y_test.shape[0]
logger.info(
'Accuracy on Iris with boundary adversarial examples: %.2f%%',
(accuracy * 100))
def test_iris_pt(self):
classifier = get_iris_classifier_pt()
attack = CarliniLInfMethod(classifier,
targeted=False,
max_iter=10,
eps=0.5)
x_test_adv = attack.generate(self.x_test.astype(np.float32))
self.assertFalse((self.x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test,
axis=1) == predictions_adv).all())
accuracy = np.sum(predictions_adv == np.argmax(
self.y_test, axis=1)) / self.y_test.shape[0]
logger.info('Accuracy on Iris with C&W adversarial examples: %.2f%%',
(accuracy * 100))
def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
x_test = x_test.astype(np.float32)
# Norm=2
attack = HopSkipJump(classifier,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info(
'Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%',
(acc * 100))
# Norm=np.inf
attack = HopSkipJump(classifier,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info(
'Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%',
(acc * 100))
