def test_ptclassifier(self): """ Third test with the PyTorchClassifier. :return: """ # Build PyTorchClassifier ptc = get_classifier_pt() # Get MNIST (_, _), (x_test, y_test) = self.mnist x_test = np.swapaxes(x_test, 1, 3) # First attack ead = ElasticNet(classifier=ptc, targeted=True, max_iter=2) params = {'y': random_targets(y_test, ptc.nb_classes)} x_test_adv = ead.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1) self.assertTrue((target == y_pred_adv).any()) # Second attack ead = ElasticNet(classifier=ptc, targeted=False, max_iter=2) params = {'y': random_targets(y_test, ptc.nb_classes)} x_test_adv = ead.generate(x_test, **params) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1) self.assertTrue((target != y_pred_adv).any())
def test_ptclassifier(self): """ Third test with the PyTorchClassifier. :return: """ # Build PyTorchClassifier ptc = get_classifier_pt() # Get MNIST (x_train, _), (x_test, _) = self.mnist x_train = np.swapaxes(x_train, 1, 3) x_test = np.swapaxes(x_test, 1, 3) # Attack attack_params = {"max_translation": 10.0, "num_translations": 3, "max_rotation": 30.0, "num_rotations": 3} attack_st = SpatialTransformation(ptc) x_train_adv = attack_st.generate(x_train, **attack_params) self.assertTrue(abs(x_train_adv[0, 0, 13, 5] - 0.374206543) <= 0.01) self.assertTrue(abs(attack_st.fooling_rate - 0.361) <= 0.01) self.assertTrue(attack_st.attack_trans_x == 0) self.assertTrue(attack_st.attack_trans_y == -3) self.assertTrue(attack_st.attack_rot == 30.0) x_test_adv = attack_st.generate(x_test) self.assertTrue(abs(x_test_adv[0, 0, 14, 14] - 0.008591662) <= 0.01)
def test_ptclassifier(self): """ Third test with the PyTorchClassifier. :return: """ # Build PyTorchClassifier ptc = get_classifier_pt() # Get MNIST (_, _), (x_test, y_test) = self.mnist x_test = np.swapaxes(x_test, 1, 3) # First attack cl2m = CarliniL2Method(classifier=ptc, targeted=True, max_iter=10) params = {'y': random_targets(y_test, ptc.nb_classes)} x_test_adv = cl2m.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1) self.assertTrue((target == y_pred_adv).any()) logger.info('CW2 Success Rate: %.2f', (sum(target == y_pred_adv) / float(len(target)))) # Second attack cl2m = CarliniL2Method(classifier=ptc, targeted=False, max_iter=10) x_test_adv = cl2m.generate(x_test) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1) self.assertTrue((target != y_pred_adv).any()) logger.info('CW2 Success Rate: %.2f', (sum(target != y_pred_adv) / float(len(target))))
def test_ptclassifier(self): """ Third test with the PyTorchClassifier. :return: """ # Build PyTorchClassifier ptc = get_classifier_pt() # Get MNIST (x_train, _), (_, _) = self.mnist x_train = np.swapaxes(x_train, 1, 3) # Attack attack_ap = AdversarialPatch(ptc, rotation_max=22.5, scale_min=0.1, scale_max=1.0, learning_rate=5.0, patch_shape=(1, 28, 28), batch_size=10) patch_adv, _ = attack_ap.generate(x_train) self.assertTrue(patch_adv[0, 8, 8] - (-3.1423605902784875) < 0.01) self.assertTrue(patch_adv[0, 14, 14] - 19.790434152473054 < 0.01) self.assertTrue(np.sum(patch_adv) - 383.5670772794207 < 0.01)
def test_ptclassifier(self): """ Third test with the PyTorchClassifier. :return: """ # Build PyTorchClassifier ptc = get_classifier_pt() # Get MNIST x_test, y_test = self.mnist x_test = np.swapaxes(x_test, 1, 3) # First attack zoo = ZooAttack(classifier=ptc, targeted=True, max_iter=10) params = {'y': random_targets(y_test, ptc.nb_classes)} x_test_adv = zoo.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1) logger.debug('ZOO target: %s', target) logger.debug('ZOO actual: %s', y_pred_adv) logger.info('ZOO success rate on MNIST: %.2f', (sum(target != y_pred_adv) / float(len(target)))) # Second attack zoo = ZooAttack(classifier=ptc, targeted=False, max_iter=10) x_test_adv = zoo.generate(x_test) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1) y_pred = np.argmax(ptc.predict(x_test), axis=1) logger.debug('ZOO actual: %s', y_pred_adv) logger.info('ZOO success rate on MNIST: %.2f', (sum(y_pred != y_pred_adv) / float(len(y_pred))))
def test_ptclassifier(self): """ Third test with the PyTorchClassifier. :return: """ # Build PyTorchClassifier ptc = get_classifier_pt() # Get MNIST (x_train, y_train), (x_test, y_test) = self.mnist x_train = np.swapaxes(x_train, 1, 3) x_test = np.swapaxes(x_test, 1, 3) # Attack attack_params = {"max_iter": 1, "attacker": "newtonfool", "attacker_params": {"max_iter": 5}} up = UniversalPerturbation(ptc) x_train_adv = up.generate(x_train, **attack_params) self.assertTrue((up.fooling_rate >= 0.2) or not up.converged) x_test_adv = x_test + up.v self.assertFalse((x_test == x_test_adv).all()) train_y_pred = np.argmax(ptc.predict(x_train_adv), axis=1) test_y_pred = np.argmax(ptc.predict(x_test_adv), axis=1) self.assertFalse((np.argmax(y_test, axis=1) == test_y_pred).all()) self.assertFalse((np.argmax(y_train, axis=1) == train_y_pred).all())
def setUpClass(cls): k.set_learning_phase(1) # Get MNIST (x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist') x_train, y_train, x_test, y_test = x_train[: NB_TRAIN], y_train[: NB_TRAIN], x_test[: NB_TEST], y_test[: NB_TEST] cls.mnist = (x_train, y_train), (x_test, y_test) # Keras classifier cls.classifier_k, sess = get_classifier_kr() scores = cls.classifier_k._model.evaluate(x_train, y_train) logger.info('[Keras, MNIST] Accuracy on training set: %.2f%%', (scores[1] * 100)) scores = cls.classifier_k._model.evaluate(x_test, y_test) logger.info('[Keras, MNIST] Accuracy on test set: %.2f%%', (scores[1] * 100)) # Create basic CNN on MNIST using TensorFlow cls.classifier_tf, sess = get_classifier_tf() scores = get_labels_np_array(cls.classifier_tf.predict(x_train)) acc = np.sum(np.argmax(scores, axis=1) == np.argmax( y_train, axis=1)) / y_train.shape[0] logger.info('[TF, MNIST] Accuracy on training set: %.2f%%', (acc * 100)) scores = get_labels_np_array(cls.classifier_tf.predict(x_test)) acc = np.sum(np.argmax(scores, axis=1) == np.argmax( y_test, axis=1)) / y_test.shape[0] logger.info('[TF, MNIST] Accuracy on test set: %.2f%%', (acc * 100)) # Create basic PyTorch model cls.classifier_py = get_classifier_pt() x_train, x_test = np.swapaxes(x_train, 1, 3), np.swapaxes(x_test, 1, 3) scores = get_labels_np_array(cls.classifier_py.predict(x_train)) acc = np.sum(np.argmax(scores, axis=1) == np.argmax( y_train, axis=1)) / y_train.shape[0] logger.info('[PyTorch, MNIST] Accuracy on training set: %.2f%%', (acc * 100)) scores = get_labels_np_array(cls.classifier_py.predict(x_test)) acc = np.sum(np.argmax(scores, axis=1) == np.argmax( y_test, axis=1)) / y_test.shape[0] logger.info('[PyTorch, MNIST] Accuracy on test set: %.2f%%', (acc * 100))
def test_ptclassifier(self): """ Third test with the PyTorchClassifier. :return: """ # Build PyTorchClassifier ptc = get_classifier_pt() # Get MNIST (_, _), (x_test, _) = self.mnist x_test = np.swapaxes(x_test, 1, 3) # Attack # import time nf = NewtonFool(ptc, max_iter=5) # print("Test Pytorch....") # starttime = time.clock() # x_test_adv = nf.generate(x_test, batch_size=1) # endtime = time.clock() # print(1, endtime - starttime) # starttime = time.clock() # x_test_adv = nf.generate(x_test, batch_size=10) # endtime = time.clock() # print(10, endtime - starttime) # starttime = time.clock() x_test_adv = nf.generate(x_test, batch_size=100) # endtime = time.clock() # print(100, endtime - starttime) # # starttime = time.clock() # x_test_adv = nf.generate(x_test, batch_size=1000) # endtime = time.clock() # print(1000, endtime - starttime) self.assertFalse((x_test == x_test_adv).all()) y_pred = ptc.predict(x_test) y_pred_adv = ptc.predict(x_test_adv) y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred y_pred_max = y_pred.max(axis=1) y_pred_adv_max = y_pred_adv[y_pred_bool] self.assertTrue((y_pred_max >= y_pred_adv_max).all())
def test_ptclassifier(self): """ Third test with the PyTorchClassifier. :return: """ # Build PyTorchClassifier ptc = get_classifier_pt() # Get MNIST (x_train, _), (_, _) = self.mnist x_train = np.swapaxes(x_train, 1, 3) # Attack attack_params = {"rotation_max": 22.5, "scale_min": 0.1, "scale_max": 1.0, "learning_rate": 5.0, "number_of_steps": 5, "patch_shape": (1, 28, 28), "batch_size": 10} attack_ap = AdversarialPatch(ptc) patch_adv, _ = attack_ap.generate(x_train, **attack_params) self.assertTrue(patch_adv[0, 8, 8] - (-3.1423605902784875) < 0.01) self.assertTrue(patch_adv[0, 14, 14] - 19.790434152473054 < 0.01) self.assertTrue(np.sum(patch_adv) - 383.5670772794207 < 0.01)
def test_ptclassifier(self): """ Third test with the PyTorchClassifier. :return: """ # Build PyTorchClassifier ptc = get_classifier_pt() # Get MNIST (x_train, y_train), (x_test, y_test) = self.mnist x_train = np.swapaxes(x_train, 1, 3) x_test = np.swapaxes(x_test, 1, 3) # First attack clinfm = CarliniLInfMethod(classifier=ptc, targeted=True, max_iter=10, eps=0.5) params = {'y': random_targets(y_test, ptc.nb_classes)} x_test_adv = clinfm.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1) self.assertTrue((target == y_pred_adv).any()) # Second attack clinfm = CarliniLInfMethod(classifier=ptc, targeted=False, max_iter=10, eps=0.5) x_test_adv = clinfm.generate(x_test) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1) self.assertTrue((target != y_pred_adv).any())
def test_ptclassifier(self): """ Third test with the PyTorchClassifier. :return: """ # Build PyTorchClassifier ptc = get_classifier_pt() # Get MNIST (_, _), (x_test, _) = self.mnist x_test = np.swapaxes(x_test, 1, 3) # Attack nf = NewtonFool(ptc, max_iter=5, batch_size=100) x_test_adv = nf.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) y_pred = ptc.predict(x_test) y_pred_adv = ptc.predict(x_test_adv) y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred y_pred_max = y_pred.max(axis=1) y_pred_adv_max = y_pred_adv[y_pred_bool] self.assertTrue((y_pred_max >= y_pred_adv_max).all())