def test_3_tensorflow_mnist(self): """ First test with the TensorFlowClassifier. :return: """ x_test_original = self.x_test_mnist.copy() # Build TensorFlowClassifier tfc, sess = get_image_classifier_tf() # Targeted attack zoo = ZooAttack(classifier=tfc, targeted=True, max_iter=30, binary_search_steps=8, batch_size=128, verbose=False) params = {"y": random_targets(self.y_test_mnist, tfc.nb_classes)} x_test_mnist_adv = zoo.generate(self.x_test_mnist, **params) self.assertFalse((self.x_test_mnist == x_test_mnist_adv).all()) self.assertLessEqual(np.amax(x_test_mnist_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_mnist_adv), 0.0) target = np.argmax(params["y"], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_mnist_adv), axis=1) logger.debug("ZOO target: %s", target) logger.debug("ZOO actual: %s", y_pred_adv) logger.info("ZOO success rate on MNIST: %.2f", (sum(target == y_pred_adv) / float(len(target)))) # Untargeted attack zoo = ZooAttack(classifier=tfc, targeted=False, max_iter=10, binary_search_steps=3, verbose=False) x_test_mnist_adv = zoo.generate(self.x_test_mnist) # self.assertFalse((x_test == x_test_adv).all()) self.assertLessEqual(np.amax(x_test_mnist_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_mnist_adv), 0.0) y_pred = np.argmax(tfc.predict(self.x_test_mnist), axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_mnist_adv), axis=1) logger.debug("ZOO actual: %s", y_pred_adv) logger.info("ZOO success rate on MNIST: %.2f", (sum(y_pred != y_pred_adv) / float(len(y_pred)))) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float( np.max(np.abs(x_test_original - self.x_test_mnist))), 0.0, delta=0.00001) # Check resize x_test_resized = zoo._resize_image(self.x_test_mnist, 64, 64) self.assertEqual(x_test_resized.shape, (1, 64, 64, 1)) # Clean-up session if sess is not None: sess.close()
def test_check_params(self): ptc = get_image_classifier_pt(from_logits=True) with self.assertRaises(ValueError): _ = ZooAttack(ptc, binary_search_steps=1.0) with self.assertRaises(ValueError): _ = ZooAttack(ptc, binary_search_steps=-1) with self.assertRaises(ValueError): _ = ZooAttack(ptc, max_iter=1.0) with self.assertRaises(ValueError): _ = ZooAttack(ptc, max_iter=-1) with self.assertRaises(ValueError): _ = ZooAttack(ptc, nb_parallel=1.0) with self.assertRaises(ValueError): _ = ZooAttack(ptc, nb_parallel=-1) with self.assertRaises(ValueError): _ = ZooAttack(ptc, batch_size=1.0) with self.assertRaises(ValueError): _ = ZooAttack(ptc, batch_size=-1) with self.assertRaises(ValueError): _ = ZooAttack(ptc, verbose="true")
def test_tensorflow_failure_attack(self): """ Test the corner case when attack fails. :return: """ x_test_original = self.x_test_mnist.copy() # Build TensorFlowClassifier tfc, sess = get_image_classifier_tf() # Failure attack zoo = ZooAttack(classifier=tfc, max_iter=0, binary_search_steps=0, learning_rate=0) x_test_mnist_adv = zoo.generate(self.x_test_mnist) self.assertLessEqual(np.amax(x_test_mnist_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_mnist_adv), 0.0) np.testing.assert_almost_equal(self.x_test_mnist, x_test_mnist_adv, 3) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float( np.max(np.abs(x_test_original - self.x_test_mnist))), 0.0, delta=0.00001) # Clean-up session if sess is not None: sess.close()
def _zoo(model, data, labels, attack_args): lr = attack_args.get('learning_rate', 0.01) max_iter = attack_args.get('max_iter', 100) binary_search_steps = attack_args.get('binary_search_steps', 5) confidence = attack_args.get('confidence', 0.0) targeted = attack_args.get('targeted', False) init_const = attack_args.get('init_const', 1e-3) abort_early = attack_args.get('abort_early', True) use_resize = attack_args.get('use_resize', True) use_importance = attack_args.get('use_importance', True) nb_parallel = attack_args.get('nb_parallel', 128) variable_h = attack_args.get('variable_h', 1e-4) attacker = ZooAttack(classifier=model, confidence=confidence, targeted=targeted, learning_rate=lr, max_iter=max_iter, binary_search_steps=binary_search_steps, initial_const=init_const, abort_early=abort_early, use_resize=use_resize, use_importance=use_importance, nb_parallel=nb_parallel, variable_h=variable_h) return attacker.generate(data, labels)
def test_5_keras_mnist(self): """ Second test with the KerasClassifier. :return: """ x_test_original = self.x_test_mnist.copy() # Build KerasClassifier krc = get_image_classifier_kr() # Targeted attack # zoo = ZooAttack(classifier=krc, targeted=True, batch_size=5) # params = {'y': random_targets(self.y_test, krc.nb_classes)} # x_test_adv = zoo.generate(self.x_test, **params) # # self.assertFalse((self.x_test == x_test_adv).all()) # self.assertLessEqual(np.amax(x_test_adv), 1.0) # self.assertGreaterEqual(np.amin(x_test_adv), 0.0) # target = np.argmax(params['y'], axis=1) # y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1) # logger.debug('ZOO target: %s', target) # logger.debug('ZOO actual: %s', y_pred_adv) # logger.info('ZOO success rate on MNIST: %.2f', (sum(target == y_pred_adv) / float(len(target)))) # Untargeted attack # zoo = ZooAttack(classifier=krc, targeted=False, max_iter=20) zoo = ZooAttack(classifier=krc, targeted=False, batch_size=5, max_iter=10, binary_search_steps=3) # x_test_adv = zoo.generate(x_test) params = {"y": random_targets(self.y_test_mnist, krc.nb_classes)} x_test_mnist_adv = zoo.generate(self.x_test_mnist, **params) # x_test_adv_true = [0.00000000e+00, 2.50167388e-04, 1.50529508e-04, 4.69674182e-04, # 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, # 1.67321396e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, # 0.00000000e+00, 2.08451956e-06, 0.00000000e+00, 0.00000000e+00, # 2.53360748e-01, 9.60119188e-01, 9.85227525e-01, 2.53600776e-01, # 0.00000000e+00, 0.00000000e+00, 5.23251540e-04, 0.00000000e+00, # 0.00000000e+00, 0.00000000e+00, 1.08632184e-05, 0.00000000e+00] # # for i in range(14): # self.assertAlmostEqual(x_test_adv_true[i], x_test_adv[0, 14, i, 0]) # self.assertFalse((x_test == x_test_adv).all()) self.assertLessEqual(np.amax(x_test_mnist_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_mnist_adv), 0.0) y_pred_adv = np.argmax(krc.predict(x_test_mnist_adv), axis=1) y_pred = np.argmax(krc.predict(self.x_test_mnist), axis=1) logger.debug("ZOO actual: %s", y_pred_adv) logger.info("ZOO success rate on MNIST: %.2f", (sum(y_pred != y_pred_adv) / float(len(y_pred)))) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test_mnist))), 0.0, delta=0.00001) # Clean-up k.clear_session()
def test_4_pytorch_mnist(self): """ Third test with the PyTorchClassifier. :return: """ # Build PyTorchClassifier ptc = get_image_classifier_pt() # Get MNIST x_test_mnist = np.swapaxes(self.x_test_mnist, 1, 3).astype(np.float32) x_test_original = x_test_mnist.copy() # First attack # zoo = ZooAttack(classifier=ptc, targeted=True, max_iter=10, binary_search_steps=10, verbose=False) # params = {'y': random_targets(self.y_test, ptc.nb_classes)} # x_test_adv = zoo.generate(x_test, **params) # self.assertFalse((x_test == x_test_adv).all()) # self.assertLessEqual(np.amax(x_test_adv), 1.0) # self.assertGreaterEqual(np.amin(x_test_adv), 0.0) # target = np.argmax(params['y'], axis=1) # y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1) # logger.debug('ZOO target: %s', target) # logger.debug('ZOO actual: %s', y_pred_adv) # logger.info('ZOO success rate on MNIST: %.2f', (sum(target != y_pred_adv) / float(len(target)))) # Second attack zoo = ZooAttack( classifier=ptc, targeted=False, learning_rate=1e-2, max_iter=10, binary_search_steps=3, abort_early=False, use_resize=False, use_importance=False, verbose=False, ) x_test_mnist_adv = zoo.generate(x_test_mnist) self.assertLessEqual(np.amax(x_test_mnist_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_mnist_adv), 0.0) # print(x_test[0, 0, 14, :]) # print(x_test_adv[0, 0, 14, :]) # print(np.amax(x_test - x_test_adv)) # x_test_adv_expected = [] # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float( np.max(np.abs(x_test_original - x_test_mnist))), 0.0, delta=0.00001)