def test_5_pytorch_mnist(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
x_test = np.swapaxes(self.x_test_mnist, 1, 3).astype(np.float32)
x_test_original = x_test.copy()
# Build PyTorchClassifier
ptc = get_image_classifier_pt()
# Attack
nf = NewtonFool(ptc, max_iter=5, batch_size=100)
x_test_adv = nf.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
y_pred = ptc.predict(x_test)
y_pred_adv = ptc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= 0.9 * y_pred_adv_max).all())
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))),
0.0,
delta=0.00001)
def test_9_keras_mnist(self):
"""
Second test with the KerasClassifier.
:return:
"""
x_test_original = self.x_test_mnist.copy()
# Build KerasClassifier
krc = get_image_classifier_kr()
# Attack
nf = NewtonFool(krc, max_iter=5, batch_size=100)
x_test_adv = nf.generate(self.x_test_mnist)
self.assertFalse((self.x_test_mnist == x_test_adv).all())
y_pred = krc.predict(self.x_test_mnist)
y_pred_adv = krc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= 0.9 * y_pred_adv_max).all())
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(
np.max(np.abs(x_test_original - self.x_test_mnist))),
0.0,
delta=0.00001)
def test_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from art.estimators.classification.scikitlearn import SklearnClassifier
scikitlearn_test_cases = [
LogisticRegression(solver="lbfgs", multi_class="auto"),
SVC(gamma="auto"),
LinearSVC(),
]
x_test_original = self.x_test_iris.copy()
for model in scikitlearn_test_cases:
classifier = SklearnClassifier(model=model, clip_values=(0, 1))
classifier.fit(x=self.x_test_iris, y=self.y_test_iris)
attack = NewtonFool(classifier, max_iter=5, batch_size=128)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ + " on Iris with NewtonFool adversarial examples"
": %.2f%%",
(acc * 100),
)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test_iris))), 0.0, delta=0.00001)
def test_3_tensorflow_mnist(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
x_test_original = self.x_test_mnist.copy()
# Build TensorFlowClassifier
tfc, sess = get_image_classifier_tf()
# Attack
nf = NewtonFool(tfc, max_iter=5, batch_size=100, verbose=False)
x_test_adv = nf.generate(self.x_test_mnist)
self.assertFalse((self.x_test_mnist == x_test_adv).all())
y_pred = tfc.predict(self.x_test_mnist)
y_pred_adv = tfc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= 0.9 * y_pred_adv_max).all())
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(
np.max(np.abs(x_test_original - self.x_test_mnist))),
0.0,
delta=0.00001)
def test_pytorch_iris(self):
classifier = get_tabular_classifier_pt()
attack = NewtonFool(classifier, max_iter=5, batch_size=128)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with NewtonFool adversarial examples: %.2f%%", (acc * 100))
def test_keras_iris_unbounded(self):
classifier = get_tabular_classifier_kr()
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model, use_logits=False, channels_first=True)
attack = NewtonFool(classifier, max_iter=5, batch_size=128)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with NewtonFool adversarial examples: %.2f%%", (acc * 100))
def test_check_params(self):
ptc = get_image_classifier_pt(from_logits=True)
with self.assertRaises(ValueError):
_ = NewtonFool(ptc, max_iter=-1)
with self.assertRaises(ValueError):
_ = NewtonFool(ptc, eta=-1)
with self.assertRaises(ValueError):
_ = NewtonFool(ptc, batch_size=-1)
with self.assertRaises(ValueError):
_ = NewtonFool(ptc, verbose="False")