def test_keras_iris_clipped(self):
classifier = get_tabular_classifier_kr()
# Norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Clean-up session
k.clear_session()
def test_pytorch_iris(self):
classifier = get_tabular_classifier_pt()
x_test = self.x_test_iris.astype(np.float32)
# Norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
def test_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier
from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier, ExtraTreesClassifier
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from art.classifiers.scikitlearn import ScikitlearnDecisionTreeClassifier, ScikitlearnExtraTreeClassifier
from art.classifiers.scikitlearn import ScikitlearnAdaBoostClassifier, ScikitlearnBaggingClassifier
from art.classifiers.scikitlearn import ScikitlearnExtraTreesClassifier, ScikitlearnGradientBoostingClassifier
from art.classifiers.scikitlearn import ScikitlearnRandomForestClassifier, ScikitlearnLogisticRegression
from art.classifiers.scikitlearn import ScikitlearnSVC
scikitlearn_test_cases = {DecisionTreeClassifier: ScikitlearnDecisionTreeClassifier,
ExtraTreeClassifier: ScikitlearnExtraTreeClassifier,
AdaBoostClassifier: ScikitlearnAdaBoostClassifier,
BaggingClassifier: ScikitlearnBaggingClassifier,
ExtraTreesClassifier: ScikitlearnExtraTreesClassifier,
GradientBoostingClassifier: ScikitlearnGradientBoostingClassifier,
RandomForestClassifier: ScikitlearnRandomForestClassifier,
LogisticRegression: ScikitlearnLogisticRegression,
SVC: ScikitlearnSVC,
LinearSVC: ScikitlearnSVC}
(_, _), (x_test, y_test) = self.iris
x_test_original = x_test.copy()
for (model_class, classifier_class) in scikitlearn_test_cases.items():
model = model_class()
classifier = classifier_class(model=model, clip_values=(0, 1))
classifier.fit(x=x_test, y=y_test)
# Norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy of ' + classifier.__class__.__name__ + ' on Iris with HopSkipJump adversarial '
'examples: %.2f%%', (acc * 100))
# Norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy of ' + classifier.__class__.__name__ + ' on Iris with HopSkipJump adversarial '
'examples: %.2f%%', (acc * 100))
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_keras_iris_unbounded(self):
classifier = get_tabular_classifier_kr()
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model, use_logits=False, channel_index=1)
# Norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Clean-up session
k.clear_session()
def test_tensorflow_iris(self):
classifier, sess = get_tabular_classifier_tf()
# Test untargeted attack and norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Test untargeted attack and norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Test targeted attack and norm=2
targets = random_targets(self.y_test_iris, nb_classes=3)
attack = HopSkipJump(classifier, targeted=True, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(self.x_test_iris, **{"y": targets})
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any())
acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / self.y_test_iris.shape[0]
logger.info("Success rate of targeted HopSkipJump on Iris: %.2f%%", (acc * 100))
# Test targeted attack and norm=np.inf
targets = random_targets(self.y_test_iris, nb_classes=3)
attack = HopSkipJump(classifier, targeted=True, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(self.x_test_iris, **{"y": targets})
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any())
acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / self.y_test_iris.shape[0]
logger.info("Success rate of targeted HopSkipJump on Iris: %.2f%%", (acc * 100))
# Clean-up session
if sess is not None:
sess.close()
def test_tensorflow_iris(self):
(_, _), (x_test, y_test) = self.iris
classifier, sess = get_iris_classifier_tf()
# Test untargeted attack and norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%', (acc * 100))
# Test untargeted attack and norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%', (acc * 100))
# Test targeted attack and norm=2
targets = random_targets(y_test, nb_classes=3)
attack = HopSkipJump(classifier, targeted=True, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(x_test, **{'y': targets})
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any())
acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / y_test.shape[0]
logger.info('Success rate of targeted HopSkipJump on Iris: %.2f%%', (acc * 100))
# Test targeted attack and norm=np.inf
targets = random_targets(y_test, nb_classes=3)
attack = HopSkipJump(classifier, targeted=True, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(x_test, **{'y': targets})
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any())
acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / y_test.shape[0]
logger.info('Success rate of targeted HopSkipJump on Iris: %.2f%%', (acc * 100))
# Clean-up session
sess.close()
def test_pytorch_resume(self):
x_test = np.reshape(self.x_test_mnist, (self.x_test_mnist.shape[0], 1, 28, 28)).astype(np.float32)
# Build PyTorchClassifier
ptc = get_image_classifier_pt()
# HSJ attack
hsj = HopSkipJump(classifier=ptc, targeted=True, max_iter=10, max_eval=100, init_eval=10)
params = {"y": self.y_test_mnist[2:3], "x_adv_init": x_test[2:3]}
x_test_adv1 = hsj.generate(x_test[0:1], **params)
diff1 = np.linalg.norm(x_test_adv1 - x_test)
params.update(resume=True, x_adv_init=x_test_adv1)
x_test_adv2 = hsj.generate(x_test[0:1], **params)
params.update(x_adv_init=x_test_adv2)
x_test_adv2 = hsj.generate(x_test[0:1], **params)
diff2 = np.linalg.norm(x_test_adv2 - x_test)
self.assertGreater(diff1, diff2)
def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
# Norm=2
attack = HopSkipJump(classifier,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info(
'Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%',
(acc * 100))
# Norm=np.inf
attack = HopSkipJump(classifier,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info(
'Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%',
(acc * 100))
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
x_test = np.swapaxes(x_test, 1, 3).astype(np.float32)
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=ptc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10)
params = {'y': random_targets(y_test, ptc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=ptc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
params = {'y': random_targets(y_test, ptc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=ptc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(ptc.predict(x_test), axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=ptc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(ptc.predict(x_test), axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc = get_classifier_kr()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=krc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10)
params = {'y': random_targets(y_test, krc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=krc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
params = {'y': random_targets(y_test, krc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=krc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(krc.predict(x_test), axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=krc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(krc.predict(x_test), axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Clean-up session
k.clear_session()
tf.reset_default_graph()
def test_tensorflow_mnist(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
x_test_original = self.x_test_mnist.copy()
# Build TensorFlowClassifier
tfc, sess = get_image_classifier_tf()
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=tfc, targeted=True, max_iter=2, max_eval=100, init_eval=10)
params = {"y": random_targets(self.y_test_mnist, tfc.nb_classes())}
x_test_adv = hsj.generate(self.x_test_mnist, **params)
self.assertFalse((self.x_test_mnist == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params["y"], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=tfc, targeted=True, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
params = {"y": random_targets(self.y_test_mnist, tfc.nb_classes())}
x_test_adv = hsj.generate(self.x_test_mnist, **params)
self.assertFalse((self.x_test_mnist == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params["y"], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=tfc, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = hsj.generate(self.x_test_mnist)
self.assertFalse((self.x_test_mnist == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(self.x_test_mnist), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=tfc, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = hsj.generate(self.x_test_mnist)
self.assertFalse((self.x_test_mnist == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(self.x_test_mnist), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test_mnist))), 0.0, delta=0.00001)
# Clean-up session
if sess is not None:
sess.close()
def test_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier
from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier, ExtraTreesClassifier
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from art.classifiers.scikitlearn import SklearnClassifier
scikitlearn_test_cases = [
DecisionTreeClassifier(),
ExtraTreeClassifier(),
AdaBoostClassifier(),
BaggingClassifier(),
ExtraTreesClassifier(n_estimators=10),
GradientBoostingClassifier(n_estimators=10),
RandomForestClassifier(n_estimators=10),
LogisticRegression(solver="lbfgs", multi_class="auto"),
SVC(gamma="auto"),
LinearSVC(),
]
x_test_original = self.x_test_iris.copy()
for model in scikitlearn_test_cases:
classifier = SklearnClassifier(model=model, clip_values=(0, 1))
classifier.fit(x=self.x_test_iris, y=self.y_test_iris)
# Norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ + " on Iris with HopSkipJump adversarial "
"examples: %.2f%%",
(acc * 100),
)
# Norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ + " on Iris with HopSkipJump adversarial "
"examples: %.2f%%",
(acc * 100),
)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test_iris))), 0.0, delta=0.00001)
def test_pytorch_classifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
x_test = np.swapaxes(self.x_test_mnist, 1, 3).astype(np.float32)
x_test_original = x_test.copy()
# Build PyTorchClassifier
ptc = get_image_classifier_pt()
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=ptc, targeted=True, max_iter=2, max_eval=100, init_eval=10)
params = {"y": random_targets(self.y_test_mnist, ptc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params["y"], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=ptc, targeted=True, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
params = {"y": random_targets(self.y_test_mnist, ptc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params["y"], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=ptc, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(ptc.predict(x_test), axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=ptc, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(ptc.predict(x_test), axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_keras_mnist(self):
"""
Second test with the KerasClassifier.
:return:
"""
(_, _), (x_test, y_test) = self.mnist
x_test_original = x_test.copy()
# Build KerasClassifier
krc = get_classifier_kr()
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=krc, targeted=True, max_iter=2, max_eval=100, init_eval=10)
params = {'y': random_targets(y_test, krc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=krc, targeted=True, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
params = {'y': random_targets(y_test, krc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=krc, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(krc.predict(x_test), axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=krc, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(krc.predict(x_test), axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
# Clean-up session
k.clear_session()
def test_tfclassifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=tfc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10)
params = {'y': random_targets(y_test, tfc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=tfc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
params = {'y': random_targets(y_test, tfc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=tfc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(x_test), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=tfc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(x_test), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Clean-up session
sess.close()
predictions = np.argmax(classifier.predict(test_data), axis=1)
attack = HopSkipJump(classifier=classifier, norm = np.inf)
n_selected = 100
corrected = []
c_labels = []
for i in range(len(test_label)):
if test_label[i] == predictions[i]:
corrected.append(test_data[i])
c_labels.append(test_label[i])
if len(corrected) >= n_selected:
break
corrected = np.array(corrected)
start = time.time()
test_adv = attack.generate(corrected)
end = time.time()
p_adv = np.argmax(classifier.predict(test_adv), axis=1)
avg_time = (end-start) / test_adv.shape[0]
dis = 0
for i in range(test_adv.shape[0]):
d = np.max(np.abs(corrected[i] - test_adv[i]))
print(str(d))
dis += d
avg_dis = dis / test_adv.shape[0]
os.chdir('{}/attack/HSJ'.format(home))
with open('{}_HSJ_{}_ave.txt'.format(d_name, m_name), 'w') as f:
f.write('average distance: ' + str(avg_dis))
# 1.5 this is the label we want to perturb to
plt.imshow(image_init)
plt.show()
print('Tesseract output is: ' + label_dict[np.argmax(classifier.predict(np.array([image_init])))])
"""
Tesseract output is: assent
"""
# 2. 使用HopSkipJump进行攻击
attack = HopSkipJump(classifier=classifier, targeted=True, norm=2, max_iter=0, max_eval=1000, init_eval=10)
iter_step = 10
x_adv = np.array([image_init])
for i in range(16):
x_adv = attack.generate(x=np.array([image_target]), x_adv_init=x_adv, y=to_categorical([1], 3))
#clear_output()
if i%3 == 0:
print("Adversarial image at step %d." % (i * iter_step), "L2 error",
np.linalg.norm(np.reshape(x_adv[0] - image_target, [-1])),
"and Tesseract output %s." % label_dict[np.argmax(classifier.predict(x_adv)[0])])
plt.imshow(x_adv[0])
plt.show(block=False)
attack.max_iter = iter_step
"""
Adversarial image at step 0. L2 error 7213.825 and Tesseract output assent.
Adversarial image at step 30. L2 error 4137.465 and Tesseract output assent.
Adversarial image at step 60. L2 error 2795.272 and Tesseract output assent.
Adversarial image at step 90. L2 error 2537.3455 and Tesseract output assent.