def test_iris_k_unbounded(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model,
use_logits=False,
channel_index=1)
attack_params = {
"max_iter": 1,
"attacker": "newtonfool",
"attacker_params": {
"max_iter": 5
}
}
attack = UniversalPerturbation(classifier)
attack.set_params(**attack_params)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info(
'Accuracy on Iris with universal adversarial examples: %.2f%%',
(acc * 100))
def test_iris_k_clipped(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
# Test untargeted attack
attack = BasicIterativeMethod(classifier, eps=1, eps_step=0.1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with BIM adversarial examples: %.2f%%',
(acc * 100))
# Test targeted attack
targets = random_targets(y_test, nb_classes=3)
attack = BasicIterativeMethod(classifier,
targeted=True,
eps=1,
eps_step=0.1)
x_test_adv = attack.generate(x_test, **{'y': targets})
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any())
acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / y_test.shape[0]
logger.info('Success rate of targeted BIM on Iris: %.2f%%',
(acc * 100))
def test_iris_k_clipped(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
# Test untargeted attack
attack_params = {
"max_iter": 1,
"attacker": "newtonfool",
"attacker_params": {
"max_iter": 5
}
}
attack = UniversalPerturbation(classifier)
attack.set_params(**attack_params)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info(
'Accuracy on Iris with universal adversarial examples: %.2f%%',
(acc * 100))
def test_iris_k_clipped(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
attack = ElasticNet(classifier, targeted=False, max_iter=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = 1. - np.sum(
preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('EAD success rate on Iris: %.2f%%', (acc * 100))
def test_iris_k_clipped(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
attack = CarliniL2Method(classifier, targeted=False, max_iter=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with C&W adversarial examples: %.2f%%',
(acc * 100))
def test_iris_k_unbounded(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model, use_logits=False, channel_index=1)
attack = SaliencyMapMethod(classifier, theta=1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with JSMA adversarial examples: %.2f%%', (acc * 100))
def test_iris_k_clipped(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
# Test untargeted attack
attack = VirtualAdversarialMethod(classifier, eps=.1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with VAT adversarial examples: %.2f%%',
(acc * 100))
def test_iris_k_unbounded(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model,
use_logits=False,
channel_index=1)
attack = ElasticNet(classifier, targeted=False, max_iter=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = 1. - np.sum(
preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('EAD success rate on Iris: %.2f%%', (acc * 100))
def test_iris_k_unbounded(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model,
use_logits=False,
channel_index=1)
attack = ProjectedGradientDescent(classifier, eps=1, eps_step=0.2)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv > 1).any())
self.assertTrue((x_test_adv < 0).any())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with PGD adversarial examples: %.2f%%',
(acc * 100))
def test_iris_clipped(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
classifier = QueryEfficientBBGradientEstimation(classifier,
20,
1 / 64.,
round_samples=1 / 255.)
# Test untargeted attack
attack = FastGradientMethod(classifier, eps=.1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with limited query info: %.2f%%',
(acc * 100))
def test_iris_unbounded(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model,
use_logits=False,
channel_index=1)
classifier = QueryEfficientBBGradientEstimation(classifier,
20,
1 / 64.,
round_samples=1 / 255.)
attack = FastGradientMethod(classifier, eps=1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv > 1).any())
self.assertTrue((x_test_adv < 0).any())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with limited query info: %.2f%%',
(acc * 100))
def test_iris_clipped(self):
(_, _), (x_test, y_test) = self.iris
def t(x):
return x
def transformation():
while True:
yield t
classifier, _ = get_iris_classifier_kr()
classifier = ExpectationOverTransformations(classifier, sample_size=1, transformation=transformation)
# Test untargeted attack
attack = FastGradientMethod(classifier, eps=.1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with limited query info: %.2f%%', (acc * 100))
def test_iris_unbounded(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
def t(x):
return x
def transformation():
while True:
yield t
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model, use_logits=False, channel_index=1)
classifier = ExpectationOverTransformations(classifier, sample_size=1, transformation=transformation)
attack = FastGradientMethod(classifier, eps=1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv > 1).any())
self.assertTrue((x_test_adv < 0).any())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with limited query info: %.2f%%', (acc * 100))
