def _test_backend_mnist(self, classifier, x_test, y_test): x_test_original = x_test.copy() df = VirtualAdversarialMethod(classifier, batch_size=100) from art.classifiers import TensorFlowClassifier if isinstance(classifier, TensorFlowClassifier): with self.assertRaises(TypeError) as context: x_test_adv = df.generate(x_test) self.assertIn('This attack requires a classifier predicting probabilities in the range [0, 1] as output.' 'Values smaller than 0.0 or larger than 1.0 have been detected.', str(context.exception)) else: x_test_adv = df.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) y_pred = get_labels_np_array(classifier.predict(x_test_adv)) self.assertFalse((y_test == y_pred).all()) acc = np.sum(np.argmax(y_pred, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0] logger.info('Accuracy on adversarial examples: %.2f%%', (acc * 100)) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def _test_backend_mnist(self, classifier): # Get MNIST (_, _), (x_test, y_test) = self.mnist x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST] df = VirtualAdversarialMethod(classifier, batch_size=100) from art.classifiers import TensorFlowClassifier if isinstance(classifier, TensorFlowClassifier): with self.assertRaises(TypeError) as context: x_test_adv = df.generate(x_test) self.assertIn( 'This attack requires a classifier predicting probabilities in the range [0, 1] as output.' 'Values smaller than 0.0 or larger than 1.0 have been detected.', str(context.exception)) else: x_test_adv = df.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) y_pred = get_labels_np_array(classifier.predict(x_test_adv)) self.assertFalse((y_test == y_pred).all()) acc = np.sum( np.argmax(y_pred, axis=1) == np.argmax( y_test, axis=1)) / y_test.shape[0] logging.info('Accuracy on adversarial examples: %.2f%%', (acc * 100))
def test_pytorch_iris(self): (_, _), (x_test, y_test) = self.iris classifier = get_iris_classifier_pt() attack = VirtualAdversarialMethod(classifier, eps=.1) with self.assertRaises(TypeError) as context: x_test_adv = attack.generate(x_test.astype(np.float32)) self.assertIn('This attack requires a classifier predicting probabilities in the range [0, 1] as output.' 'Values smaller than 0.0 or larger than 1.0 have been detected.', str(context.exception))
def test_tensorflow_iris(self): classifier, _ = get_tabular_classifier_tf() attack = VirtualAdversarialMethod(classifier, eps=0.1) with self.assertRaises(TypeError) as context: x_test_iris_adv = attack.generate(self.x_test_iris) self.assertIn( "This attack requires a classifier predicting probabilities in the range [0, 1] as output." "Values smaller than 0.0 or larger than 1.0 have been detected.", str(context.exception), )
def test_keras_iris_clipped(self): classifier = get_tabular_classifier_kr() # Test untargeted attack attack = VirtualAdversarialMethod(classifier, eps=0.1) x_test_iris_adv = attack.generate(self.x_test_iris) self.assertFalse((self.x_test_iris == x_test_iris_adv).all()) self.assertTrue((x_test_iris_adv <= 1).all()) self.assertTrue((x_test_iris_adv >= 0).all()) preds_adv = np.argmax(classifier.predict(x_test_iris_adv), axis=1) self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all()) acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0] logger.info("Accuracy on Iris with VAT adversarial examples: %.2f%%", (acc * 100))
def test_keras_iris_unbounded(self): classifier = get_tabular_classifier_kr() # Recreate a classifier without clip values classifier = KerasClassifier(model=classifier._model, use_logits=False, channel_index=1) attack = VirtualAdversarialMethod(classifier, eps=1) x_test_iris_adv = attack.generate(self.x_test_iris) self.assertFalse((self.x_test_iris == x_test_iris_adv).all()) self.assertTrue((x_test_iris_adv > 1).any()) self.assertTrue((x_test_iris_adv < 0).any()) preds_adv = np.argmax(classifier.predict(x_test_iris_adv), axis=1) self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all()) acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0] logger.info("Accuracy on Iris with VAT adversarial examples: %.2f%%", (acc * 100))
def test_keras_iris_clipped(self): (_, _), (x_test, y_test) = self.iris classifier = get_iris_classifier_kr() # Test untargeted attack attack = VirtualAdversarialMethod(classifier, eps=.1) x_test_adv = attack.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1).all()) self.assertTrue((x_test_adv >= 0).all()) preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1) self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all()) acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0] logger.info('Accuracy on Iris with VAT adversarial examples: %.2f%%', (acc * 100))
def _test_backend_mnist(self, classifier, x_test, y_test): x_test_original = x_test.copy() df = VirtualAdversarialMethod(classifier, batch_size=100, max_iter=2) x_test_adv = df.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) y_pred = get_labels_np_array(classifier.predict(x_test_adv)) self.assertFalse((y_test == y_pred).all()) acc = np.sum(np.argmax(y_pred, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0] logger.info("Accuracy on adversarial examples: %.2f%%", (acc * 100)) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def GetAttackers(classifier, x_test, attacker_name): """ Function: Load classifier and generate adversarial samples """ t_start = time.time() if attacker_name == "FGSM": attacker = FastGradientMethod(classifier=classifier, eps=0.3) elif attacker_name == "Elastic": attacker = ElasticNet(classifier=classifier, confidence=0.5) elif attacker_name == "BasicIterativeMethod": attacker = BasicIterativeMethod(classifier=classifier, eps=0.3) elif attacker_name == "NewtonFool": attacker = NewtonFool(classifier=classifier, max_iter=20) elif attacker_name == "HopSkipJump": attacker = HopSkipJump(classifier=classifier, max_iter=20) elif attacker_name == "ZooAttack": attacker = ZooAttack(classifier=classifier, max_iter=20) elif attacker_name == "VirtualAdversarialMethod": attacker = VirtualAdversarialMethod(classifier=classifier, max_iter=20) elif attacker_name == "UniversalPerturbation": attacker = UniversalPerturbation(classifier=classifier, max_iter=20) elif attacker_name == "AdversarialPatch": attacker = AdversarialPatch(classifier=classifier, max_iter=20) elif attacker_name == "Attack": attacker = Attack(classifier=classifier) elif attacker_name == "BoundaryAttack": attacker = BoundaryAttack(classifier=classifier, targeted=False, epsilon=0.05, max_iter=20) #, max_iter=20 elif attacker_name == "CarliniL2": attacker = CarliniL2Method(classifier=classifier, confidence=0.5, learning_rate=0.001, max_iter=15) elif attacker_name == "CarliniLinf": attacker = CarliniLInfMethod(classifier=classifier, confidence=0.5, learning_rate=0.001, max_iter=15) elif attacker_name == "DeepFool": attacker = DeepFool(classifier) elif attacker_name == "SMM": attacker = SaliencyMapMethod(classifier=classifier, theta=2) elif attacker_name == "PGD": attacker = ProjectedGradientDescent(classifier=classifier, norm=2, eps=1, eps_step=0.5) else: raise ValueError("Please get the right attacker's name for the input.") test_adv = attacker.generate(x_test) dt = time.time() - t_start return test_adv, dt
def test_classifier_type_check_fail_classifier(self): # Use a useless test classifier to test basic classifier properties class ClassifierNoAPI: pass classifier = ClassifierNoAPI with self.assertRaises(TypeError) as context: _ = VirtualAdversarialMethod(classifier=classifier) self.assertIn('For `VirtualAdversarialMethod` classifier must be an instance of ' '`art.classifiers.classifier.Classifier`, the provided classifier is instance of ' '(<class \'object\'>,).', str(context.exception))
def test_classifier_type_check_fail_gradients(self): # Use a test classifier not providing gradients required by white-box attack from art.classifiers.scikitlearn import ScikitlearnDecisionTreeClassifier from sklearn.tree import DecisionTreeClassifier classifier = ScikitlearnDecisionTreeClassifier(model=DecisionTreeClassifier()) with self.assertRaises(TypeError) as context: _ = VirtualAdversarialMethod(classifier=classifier) self.assertIn('For `VirtualAdversarialMethod` classifier must be an instance of ' '`art.classifiers.classifier.ClassifierNeuralNetwork` and ' '`art.classifiers.classifier.ClassifierGradients`, the provided classifier is instance of ' '(<class \'art.classifiers.scikitlearn.ScikitlearnClassifier\'>,).', str(context.exception))