def test_scikitlearn(self): from sklearn.linear_model import LogisticRegression from sklearn.svm import SVC, LinearSVC from art.estimators.classification.scikitlearn import SklearnClassifier scikitlearn_test_cases = [ LogisticRegression(solver="lbfgs", multi_class="auto"), SVC(gamma="auto"), LinearSVC(), ] x_test_original = self.x_test_iris.copy() for model in scikitlearn_test_cases: classifier = SklearnClassifier(model=model, clip_values=(0, 1)) classifier.fit(x=self.x_test_iris, y=self.y_test_iris) # Test untargeted attack attack = ElasticNet(classifier, targeted=False, max_iter=2) x_test_adv = attack.generate(self.x_test_iris) self.assertFalse((self.x_test_iris == x_test_adv).all()) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1) self.assertFalse((np.argmax(self.y_test_iris, axis=1) == predictions_adv).all()) accuracy = 1.0 - np.sum(predictions_adv == np.argmax( self.y_test_iris, axis=1)) / self.y_test_iris.shape[0] logger.info( "EAD success rate of " + classifier.__class__.__name__ + " on Iris: %.2f%%", (accuracy * 100)) # Test targeted attack targets = random_targets(self.y_test_iris, nb_classes=3) attack = ElasticNet(classifier, targeted=True, max_iter=2) x_test_adv = attack.generate(self.x_test_iris, **{"y": targets}) self.assertFalse((self.x_test_iris == x_test_adv).all()) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1) self.assertTrue((np.argmax(targets, axis=1) == predictions_adv).any()) accuracy = np.sum(predictions_adv == np.argmax( targets, axis=1)) / self.y_test_iris.shape[0] logger.info( "Targeted EAD success rate of " + classifier.__class__.__name__ + " on Iris: %.2f%%", (accuracy * 100)) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float( np.max(np.abs(x_test_original - self.x_test_iris))), 0.0, delta=0.00001)
def test_7_scikitlearn(self): from sklearn.linear_model import LogisticRegression from sklearn.svm import SVC, LinearSVC from art.estimators.classification.scikitlearn import SklearnClassifier scikitlearn_test_cases = [ LogisticRegression(solver="lbfgs", multi_class="auto"), SVC(gamma="auto"), LinearSVC(), ] x_test_original = self.x_test_iris.copy() for model in scikitlearn_test_cases: classifier = SklearnClassifier(model=model, clip_values=(0, 1)) classifier.fit(x=self.x_test_iris, y=self.y_test_iris) # Test untargeted attack attack = ProjectedGradientDescent(classifier, eps=1.0, eps_step=0.1, max_iter=5, verbose=False) x_test_adv = attack.generate(self.x_test_iris) self.assertFalse((self.x_test_iris == x_test_adv).all()) self.assertTrue((x_test_adv <= 1).all()) self.assertTrue((x_test_adv >= 0).all()) preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1) self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all()) acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0] logger.info( "Accuracy of " + classifier.__class__.__name__ + " on Iris with PGD adversarial examples: " "%.2f%%", (acc * 100), ) # Test targeted attack targets = random_targets(self.y_test_iris, nb_classes=3) attack = ProjectedGradientDescent( classifier, targeted=True, eps=1.0, eps_step=0.1, max_iter=5, verbose=False ) x_test_adv = attack.generate(self.x_test_iris, **{"y": targets}) self.assertFalse((self.x_test_iris == x_test_adv).all()) self.assertTrue((x_test_adv <= 1).all()) self.assertTrue((x_test_adv >= 0).all()) preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1) self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any()) acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / self.y_test_iris.shape[0] logger.info( "Success rate of " + classifier.__class__.__name__ + " on targeted PGD on Iris: %.2f%%", (acc * 100) ) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test_iris))), 0.0, delta=0.00001)
def test_performance_diff(self): from art.estimators.classification.scikitlearn import SklearnClassifier from sklearn.svm import SVC (x_train, y_train), (x_test, y_test), min_, max_ = load_iris() full_model = SklearnClassifier(model=SVC(kernel="linear", gamma="auto"), clip_values=(min_, max_)) full_model.fit(x_train, y_train) limited_model = SklearnClassifier(model=SVC(kernel="linear", gamma="auto"), clip_values=(min_, max_)) limited_model.fit(x_train[:10], y_train[:10]) self.assertEqual( performance_diff(full_model, limited_model, x_test[:20], y_test[:20], perf_function="accuracy"), 0.35) self.assertEqual( performance_diff(full_model, limited_model, x_test[:20], y_test[:20]), 0.35) diff = performance_diff(full_model, limited_model, x_test[:20], y_test[:20], perf_function="f1", average="weighted") self.assertGreater(diff, 0.43) self.assertLess(diff, 0.44) def first_class(true_labels, model_labels, idx=0): return np.average(np.argmax(model_labels, axis=1) == idx) self.assertEqual( performance_diff(full_model, limited_model, x_test, y_test, perf_function=first_class), 1.0 / 3) self.assertEqual( performance_diff(full_model, limited_model, x_test, y_test, perf_function=first_class, idx=1), -1.0 / 3)
def test_6_scikitlearn(self): from sklearn.linear_model import LogisticRegression from sklearn.svm import SVC, LinearSVC from art.estimators.classification.scikitlearn import SklearnClassifier scikitlearn_test_cases = [ LogisticRegression(solver="lbfgs", multi_class="auto"), SVC(gamma="auto"), LinearSVC(), ] x_test_original = self.x_test_iris.copy() for model in scikitlearn_test_cases: classifier = SklearnClassifier(model=model, clip_values=(0, 1)) classifier.fit(x=self.x_test_iris, y=self.y_test_iris) attack = SaliencyMapMethod(classifier, theta=1, batch_size=128, verbose=False) x_test_iris_adv = attack.generate(self.x_test_iris) self.assertFalse((self.x_test_iris == x_test_iris_adv).all()) self.assertTrue((x_test_iris_adv <= 1).all()) self.assertTrue((x_test_iris_adv >= 0).all()) preds_adv = np.argmax(classifier.predict(x_test_iris_adv), axis=1) self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all()) accuracy = np.sum(preds_adv == np.argmax( self.y_test_iris, axis=1)) / self.y_test_iris.shape[0] logger.info( "Accuracy of " + classifier.__class__.__name__ + " on Iris with JSMA adversarial examples: " "%.2f%%", (accuracy * 100), ) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float( np.max(np.abs(x_test_original - self.x_test_iris))), 0.0, delta=0.00001)
def test_linearSVC(self): """ Test using a attack on LinearSVC """ (x_train, y_train), (x_test, y_test), min_, max_ = self.iris x_test_original = x_test.copy() # Build Scikitlearn Classifier clip_values = (min_, max_) clean = SklearnClassifier(model=LinearSVC(), clip_values=clip_values) clean.fit(x_train, y_train) poison = SklearnClassifier(model=LinearSVC(), clip_values=clip_values) poison.fit(x_train, y_train) attack = PoisoningAttackSVM(poison, 0.01, 1.0, x_train, y_train, x_test, y_test, 100) attack_y = np.array([1, 1]) - y_train[0] attack_point, _ = attack.poison(np.array([x_train[0]]), y=np.array([attack_y])) poison.fit(x=np.vstack([x_train, attack_point]), y=np.vstack([y_train, np.copy(y_train[0].reshape((1, 2)))])) acc = np.average(np.all(clean.predict(x_test) == y_test, axis=1)) * 100 poison_acc = np.average( np.all(poison.predict(x_test) == y_test, axis=1)) * 100 logger.info("Clean Accuracy {}%".format(acc)) logger.info("Poison Accuracy {}%".format(poison_acc)) self.assertGreaterEqual(acc, poison_acc) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_SVC_kernels(self): """ First test with the TensorFlowClassifier. :return: """ # Get MNIST (x_train, y_train), (x_test, y_test), min_, max_ = self.iris x_test_original = x_test.copy() # Build Scikitlearn Classifier clip_values = (min_, max_) for kernel in ["linear"]: # ["linear", "poly", "rbf"] clean = SklearnClassifier(model=SVC(kernel=kernel, gamma="auto"), clip_values=clip_values) clean.fit(x_train, y_train) poison = SklearnClassifier(model=SVC(kernel=kernel, gamma="auto"), clip_values=clip_values) poison.fit(x_train, y_train) attack = PoisoningAttackSVM(poison, 0.01, 1.0, x_train, y_train, x_test, y_test, 100) attack_y = np.array([1, 1]) - y_train[0] attack_point, _ = attack.poison(np.array([x_train[0]]), y=np.array([attack_y])) poison.fit( x=np.vstack([x_train, attack_point]), y=np.vstack([y_train, np.array([1, 1]) - np.copy(y_train[0].reshape((1, 2)))]), ) acc = np.average(np.all(clean.predict(x_test) == y_test, axis=1)) * 100 poison_acc = np.average(np.all(poison.predict(x_test) == y_test, axis=1)) * 100 logger.info("Clean Accuracy {}%".format(acc)) logger.info("Poison Accuracy {}%".format(poison_acc)) self.assertGreaterEqual(acc, poison_acc) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_6_scikitlearn(self): from sklearn.linear_model import LogisticRegression from sklearn.svm import SVC, LinearSVC from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier, ExtraTreesClassifier from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier from art.estimators.classification.scikitlearn import SklearnClassifier scikitlearn_test_cases = [ DecisionTreeClassifier(), ExtraTreeClassifier(), AdaBoostClassifier(), BaggingClassifier(), ExtraTreesClassifier(n_estimators=10), GradientBoostingClassifier(n_estimators=10), RandomForestClassifier(n_estimators=10), LogisticRegression(solver="lbfgs", multi_class="auto"), SVC(gamma="auto"), LinearSVC(), ] x_test_original = self.x_test_iris.copy() for model in scikitlearn_test_cases: classifier = SklearnClassifier(model=model, clip_values=(0, 1)) classifier.fit(x=self.x_test_iris, y=self.y_test_iris) # Norm=2 attack = HopSkipJump(classifier, targeted=False, max_iter=20, max_eval=100, init_eval=10) x_test_adv = attack.generate(self.x_test_iris) self.assertFalse((self.x_test_iris == x_test_adv).all()) self.assertTrue((x_test_adv <= 1).all()) self.assertTrue((x_test_adv >= 0).all()) preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1) self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all()) acc = np.sum(preds_adv == np.argmax( self.y_test_iris, axis=1)) / self.y_test_iris.shape[0] logger.info( "Accuracy of " + classifier.__class__.__name__ + " on Iris with HopSkipJump adversarial " "examples: %.2f%%", (acc * 100), ) # Norm=np.inf attack = HopSkipJump(classifier, targeted=False, max_iter=20, max_eval=100, init_eval=10, norm=np.Inf) x_test_adv = attack.generate(self.x_test_iris) self.assertFalse((self.x_test_iris == x_test_adv).all()) self.assertTrue((x_test_adv <= 1).all()) self.assertTrue((x_test_adv >= 0).all()) preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1) self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all()) acc = np.sum(preds_adv == np.argmax( self.y_test_iris, axis=1)) / self.y_test_iris.shape[0] logger.info( "Accuracy of " + classifier.__class__.__name__ + " on Iris with HopSkipJump adversarial " "examples: %.2f%%", (acc * 100), ) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float( np.max(np.abs(x_test_original - self.x_test_iris))), 0.0, delta=0.00001)