def _test_backend_mnist(self, classifier): # Get MNIST (_, _), (x_test, y_test) = self.mnist x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST] df = VirtualAdversarialMethod(classifier) x_test_adv = df.generate(x_test, eps=1) self.assertFalse((x_test == x_test_adv).all()) y_pred = get_labels_np_array(classifier.predict(x_test_adv)) self.assertFalse((y_test == y_pred).all()) acc = np.sum(np.argmax(y_pred, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0] print('\nAccuracy on adversarial examples: %.2f%%' % (acc * 100))
def test_iris_pt(self): (_, _), (x_test, y_test) = self.iris classifier = get_iris_classifier_pt() attack = VirtualAdversarialMethod(classifier, eps=.1) x_test_adv = attack.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1).all()) self.assertTrue((x_test_adv >= 0).all()) preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1) self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all()) acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0] logger.info('Accuracy on Iris with VAT adversarial examples: %.2f%%', (acc * 100))
def test_iris_k_unbounded(self): (_, _), (x_test, y_test) = self.iris classifier, _ = get_iris_classifier_kr() # Recreate a classifier without clip values classifier = KerasClassifier(model=classifier._model, use_logits=False, channel_index=1) attack = VirtualAdversarialMethod(classifier, eps=1) x_test_adv = attack.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv > 1).any()) self.assertTrue((x_test_adv < 0).any()) preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1) self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all()) acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0] logger.info('Accuracy on Iris with VAT adversarial examples: %.2f%%', (acc * 100))
def test_mnist(self): session = tf.Session() k.set_session(session) comp_params = { "loss": 'categorical_crossentropy', "optimizer": 'adam', "metrics": ['accuracy'] } # get MNIST batch_size, nb_train, nb_test = 100, 1000, 100 (X_train, Y_train), (X_test, Y_test), _, _ = load_mnist() X_train, Y_train = X_train[:nb_train], Y_train[:nb_train] X_test, Y_test = X_test[:nb_test], Y_test[:nb_test] im_shape = X_train[0].shape # get classifier classifier = CNN(im_shape, act="relu") classifier.compile(comp_params) classifier.fit(X_train, Y_train, epochs=1, batch_size=batch_size, verbose=0) scores = classifier.evaluate(X_test, Y_test) print("\naccuracy on test set: %.2f%%" % (scores[1] * 100)) df = VirtualAdversarialMethod(classifier, sess=session, clip_min=0., clip_max=1.) x_test_adv = df.generate(X_test, eps=1) self.assertFalse((X_test == x_test_adv).all()) y_pred = get_labels_np_array(classifier.predict(x_test_adv)) self.assertFalse((Y_test == y_pred).all()) scores = classifier.evaluate(x_test_adv, Y_test) print('\naccuracy on adversarial examples: %.2f%%' % (scores[1] * 100))
def _test_backend_mnist(self, classifier): # Get MNIST (_, _), (x_test, y_test) = self.mnist x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST] #import time df = VirtualAdversarialMethod(classifier) # starttime = time.clock() # x_test_adv = df.generate(x_test, batch_size=1) # endtime = time.clock() # print(1, endtime - starttime) # # starttime = time.clock() # x_test_adv = df.generate(x_test, batch_size=10) # endtime = time.clock() # print(10, endtime - starttime) # # starttime = time.clock() x_test_adv = df.generate(x_test, batch_size=100) # endtime = time.clock() # print(100, endtime - starttime) # starttime = time.clock() # x_test_adv = df.generate(x_test, batch_size=1000) # endtime = time.clock() # print(1000, endtime - starttime) self.assertFalse((x_test == x_test_adv).all()) y_pred = get_labels_np_array(classifier.predict(x_test_adv)) self.assertFalse((y_test == y_pred).all()) acc = np.sum(np.argmax(y_pred, axis=1) == np.argmax( y_test, axis=1)) / y_test.shape[0] logging.info('Accuracy on adversarial examples: %.2f%%', (acc * 100))
v_print("Adversarials crafted with", args.adv_method, "on", MODEL_PATH, "will be saved in", SAVE_ADV) if args.adv_method in ['fgsm', "vat", "rnd_fgsm"]: eps_ranges = { 'fgsm': [e / 10 for e in range(1, 11)], 'rnd_fgsm': [e / 10 for e in range(1, 11)], 'vat': [1.5, 2.1, 5, 7, 10] } if args.adv_method in ["fgsm", "rnd_fgsm"]: adv_crafter = FastGradientMethod(classifier, sess=session) else: adv_crafter = VirtualAdversarialMethod(classifier, sess=session) for eps in eps_ranges[args.adv_method]: if args.adv_method == "rnd_fgsm": x_train = np.clip( X_train + alpha * np.sign(np.random.randn(*X_train.shape)), min_, max_) x_test = np.clip( X_test + alpha * np.sign(np.random.randn(*X_test.shape)), min_, max_) e = eps - alpha else: x_train = X_train x_test = X_test e = eps