def test_tensorflow_failure_attack_L2(self): """ Test the corner case when attack is failed. :return: """ (_, _), (x_test, y_test) = self.mnist x_test_original = x_test.copy() # Build TensorFlowClassifier tfc, sess = get_classifier_tf(from_logits=True) # Failure attack cl2m = CarliniL2Method(classifier=tfc, targeted=True, max_iter=0, binary_search_steps=0, learning_rate=0, initial_const=1) params = {'y': random_targets(y_test, tfc.nb_classes())} x_test_adv = cl2m.generate(x_test, **params) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) np.testing.assert_array_almost_equal(x_test, x_test_adv, decimal=3) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001) # Clean-up session sess.close()
def test_fit_generator(self): classifier, sess = get_classifier_tf() # Create TensorFlow data generator x_tensor = tf.convert_to_tensor( self.x_train.reshape(10, 100, 28, 28, 1)) y_tensor = tf.convert_to_tensor(self.y_train.reshape(10, 100, 10)) dataset = tf.data.Dataset.from_tensor_slices((x_tensor, y_tensor)) iterator = dataset.make_initializable_iterator() data_gen = TFDataGenerator(sess=sess, iterator=iterator, iterator_type='initializable', iterator_arg={}, size=1000, batch_size=100) # Test fit and predict classifier.fit_generator(data_gen, nb_epochs=2) predictions = classifier.predict(self.x_test) predictions_class = np.argmax(predictions, axis=1) true_class = np.argmax(self.y_test, axis=1) accuracy = np.sum(predictions_class == true_class) / len(true_class) logger.info( 'Accuracy after fitting TF classifier with generator: %.2f%%', (accuracy * 100)) self.assertEqual(accuracy, 0.65) tf.reset_default_graph() sess.close()
def test_loss_gradient(self): classifier, sess = get_classifier_tf() gradients = classifier.loss_gradient(self.x_test, self.y_test) self.assertEqual(gradients.shape, (NB_TEST, 28, 28, 1)) expected_gradients_1 = np.asarray([ 0.00279603, 0.00266946, 0.0032446, 0.00396258, -0.00201465, -0.00564073, 0.0009253, 0.00016253, 0.0040816, 0.00166697, 0.0015883, -0.00121023, -0.00390778, -0.00234937, 0.0053558, 0.00204322, -0.00172054, 0.00053564, -0.0021146, -0.00069308, 0.00141374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ]) np.testing.assert_array_almost_equal(gradients[0, 14, :, 0], expected_gradients_1, decimal=4) expected_gradients_2 = np.asarray([ 1.05401428e-04, 1.06959546e-04, 2.60490313e-04, 2.74000311e-04, -1.15295035e-04, 2.16038228e-04, 1.37472380e-04, 0.00000000e+00, 0.00000000e+00, -2.91720475e-03, -3.08302144e-04, 2.63109524e-03, -1.18699251e-03, 2.63655302e-03, 5.35579538e-03, 6.38693338e-03, 3.44644510e-03, 6.68899389e-04, 5.01601025e-03, 8.40547902e-04, -1.43233046e-05, -2.79442966e-03, 7.37082795e-04, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00 ]) np.testing.assert_array_almost_equal(gradients[0, :, 14, 0], expected_gradients_2, decimal=4) tf.reset_default_graph() sess.close()
def test_pickle(self): classifier, sess = get_classifier_tf() full_path = os.path.join(ART_DATA_PATH, 'my_classifier') folder = os.path.split(full_path)[0] if not os.path.exists(folder): os.makedirs(folder) pickle.dump(classifier, open(full_path, 'wb')) # Unpickle: with open(full_path, 'rb') as f: loaded = pickle.load(f) self.assertEqual(classifier._clip_values, loaded._clip_values) self.assertEqual(classifier._channel_index, loaded._channel_index) self.assertEqual(set(classifier.__dict__.keys()), set(loaded.__dict__.keys())) # Test predict predictions_1 = classifier.predict(self.x_test) accuracy_1 = np.sum( np.argmax(predictions_1, axis=1) == np.argmax( self.y_test, axis=1)) / len(self.y_test) predictions_2 = loaded.predict(self.x_test) accuracy_2 = np.sum( np.argmax(predictions_2, axis=1) == np.argmax( self.y_test, axis=1)) / len(self.y_test) self.assertEqual(accuracy_1, accuracy_2) tf.reset_default_graph() sess.close()
def test_tensorflow_mnist(self): """ First test with the TensorFlowClassifier. :return: """ (x_train, y_train), (x_test, y_test) = self.mnist x_test_original = x_test.copy() # Build TensorFlowClassifier tfc, sess = get_classifier_tf() # Attack up = UniversalPerturbation(tfc, max_iter=1, attacker="newtonfool", attacker_params={"max_iter": 5}) x_train_adv = up.generate(x_train) self.assertTrue((up.fooling_rate >= 0.2) or not up.converged) x_test_adv = x_test + up.noise self.assertFalse((x_test == x_test_adv).all()) train_y_pred = np.argmax(tfc.predict(x_train_adv), axis=1) test_y_pred = np.argmax(tfc.predict(x_test_adv), axis=1) self.assertFalse((np.argmax(y_test, axis=1) == test_y_pred).all()) self.assertFalse((np.argmax(y_train, axis=1) == train_y_pred).all()) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_tensorflow_mnist(self): """ First test with the TensorFlowClassifier. :return: """ (_, _), (x_test, _) = self.mnist x_test_original = x_test.copy() # Build TensorFlowClassifier tfc, sess = get_classifier_tf() # Attack nf = NewtonFool(tfc, max_iter=5, batch_size=100) x_test_adv = nf.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) y_pred = tfc.predict(x_test) y_pred_adv = tfc.predict(x_test_adv) y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred y_pred_max = y_pred.max(axis=1) y_pred_adv_max = y_pred_adv[y_pred_bool] self.assertTrue((y_pred_max >= .9 * y_pred_adv_max).all()) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_save(self): classifier, sess = get_classifier_tf() path = 'tmp' filename = 'model.ckpt' # Save classifier.save(filename, path=path) self.assertTrue( os.path.isfile( os.path.join(path, filename, 'variables/variables.data-00000-of-00001'))) self.assertTrue( os.path.isfile( os.path.join(path, filename, 'variables/variables.index'))) self.assertTrue( os.path.isfile(os.path.join(path, filename, 'saved_model.pb'))) # # Restore # with tf.Session(graph=tf.Graph()) as sess: # tf.saved_model.loader.load(sess, ["serve"], os.path.join(path, filename)) # graph = tf.get_default_graph() # print(graph.get_operations()) # sess.run('SavedOutput:0', feed_dict={'SavedInputPhD:0': input_batch}) # Remove saved files shutil.rmtree(os.path.join(path, filename)) tf.reset_default_graph() sess.close()
def test_tfclassifier(self): """ First test with the TensorFlowClassifier. :return: """ # Build TensorFlowClassifiers victim_tfc, sess = get_classifier_tf() # Define input and output placeholders input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) output_ph = tf.placeholder(tf.int32, shape=[None, 10]) # Define the tensorflow graph conv = tf.layers.conv2d(input_ph, 1, 7, activation=tf.nn.relu) conv = tf.layers.max_pooling2d(conv, 4, 4) flattened = tf.layers.flatten(conv) # Logits layer logits = tf.layers.dense(flattened, 10) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=output_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.001) train = optimizer.minimize(loss) # TensorFlow session and initialization sess.run(tf.global_variables_initializer()) # Create the classifier thieved_tfc = TensorFlowClassifier(clip_values=(0, 1), input_ph=input_ph, output=logits, labels_ph=output_ph, train=train, loss=loss, learning=None, sess=sess) # Create attack copycat_cnn = CopycatCNN(classifier=victim_tfc, batch_size_query=BATCH_SIZE, batch_size_fit=BATCH_SIZE, nb_epochs=NB_EPOCHS, nb_stolen=NB_STOLEN) thieved_tfc = copycat_cnn.extract(x=self.x_train, thieved_classifier=thieved_tfc) victim_preds = np.argmax(victim_tfc.predict(x=self.x_train[:100]), axis=1) thieved_preds = np.argmax(thieved_tfc.predict(x=self.x_train[:100]), axis=1) acc = np.sum(victim_preds == thieved_preds) / len(victim_preds) self.assertGreater(acc, 0.3) # Clean-up session sess.close() tf.reset_default_graph()
def test_repr(self): classifier, sess = get_classifier_tf() repr_ = repr(classifier) self.assertIn('art.classifiers.tensorflow.TensorFlowClassifier', repr_) self.assertIn('channel_index=3, clip_values=(0, 1)', repr_) self.assertIn('defences=None, preprocessing=(0, 1)', repr_) tf.reset_default_graph() sess.close()
def test_set_learning(self): classifier, sess = get_classifier_tf() self.assertEqual(classifier._feed_dict, {}) classifier.set_learning_phase(False) self.assertFalse(classifier._feed_dict[classifier._learning]) classifier.set_learning_phase(True) self.assertTrue(classifier._feed_dict[classifier._learning]) self.assertTrue(classifier.learning_phase) tf.reset_default_graph() sess.close()
def test_predict(self): classifier, sess = get_classifier_tf() predictions = classifier.predict(self.x_test) predictions_class = np.argmax(predictions, axis=1) trues_class = np.argmax(self.y_test, axis=1) accuracy = np.sum(predictions_class == trues_class) / len(trues_class) logger.info('Accuracy after fitting: %.2f%%', (accuracy * 100)) self.assertEqual(accuracy, 0.4) tf.reset_default_graph() sess.close()
def test_tensorflow_mnist(self): (x_train, y_train), (x_test, y_test) = self.mnist classifier, sess = get_classifier_tf(from_logits=True) scores = get_labels_np_array(classifier.predict(x_train)) acc = np.sum(np.argmax(scores, axis=1) == np.argmax(y_train, axis=1)) / y_train.shape[0] logger.info('[TF, MNIST] Accuracy on training set: %.2f%%', (acc * 100)) scores = get_labels_np_array(classifier.predict(x_test)) acc = np.sum(np.argmax(scores, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0] logger.info('[TF, MNIST] Accuracy on test set: %.2f%%', (acc * 100)) self._test_backend_mnist(classifier, x_test, y_test)
def test_tensorflow_mnist(self): (x_train, y_train), (x_test, y_test) = self.mnist x_test_original = x_test.copy() # Create basic CNN on MNIST using TensorFlow classifier, sess = get_classifier_tf(from_logits=True) scores = get_labels_np_array(classifier.predict(x_train)) accuracy = np.sum( np.argmax(scores, axis=1) == np.argmax(y_train, axis=1)) / y_train.shape[0] logger.info('[TF, MNIST] Accuracy on training set: %.2f%%', (accuracy * 100)) scores = get_labels_np_array(classifier.predict(x_test)) accuracy = np.sum( np.argmax(scores, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0] logger.info('[TF, MNIST] Accuracy on test set: %.2f%%', (accuracy * 100)) attack = DeepFool(classifier, max_iter=5, batch_size=11) x_train_adv = attack.generate(x_train) x_test_adv = attack.generate(x_test) self.assertFalse((x_train == x_train_adv).all()) self.assertFalse((x_test == x_test_adv).all()) train_y_pred = get_labels_np_array(classifier.predict(x_train_adv)) test_y_pred = get_labels_np_array(classifier.predict(x_test_adv)) self.assertFalse((y_train == train_y_pred).all()) self.assertFalse((y_test == test_y_pred).all()) accuracy = np.sum( np.argmax(train_y_pred, axis=1) == np.argmax( y_train, axis=1)) / y_train.shape[0] logger.info('Accuracy on adversarial train examples: %.2f%%', (accuracy * 100)) accuracy = np.sum( np.argmax(test_y_pred, axis=1) == np.argmax( y_test, axis=1)) / y_test.shape[0] logger.info('Accuracy on adversarial test examples: %.2f%%', (accuracy * 100)) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_tensorflow_mnist_L2(self): """ First test with the TensorFlowClassifier. :return: """ (_, _), (x_test, y_test) = self.mnist x_test_original = x_test.copy() # Build TensorFlowClassifier tfc, sess = get_classifier_tf() # First attack cl2m = CarliniL2Method(classifier=tfc, targeted=True, max_iter=10) params = {'y': random_targets(y_test, tfc.nb_classes())} x_test_adv = cl2m.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('CW2 Target: %s', target) logger.debug('CW2 Actual: %s', y_pred_adv) logger.info('CW2 Success Rate: %.2f', (np.sum(target == y_pred_adv) / float(len(target)))) self.assertTrue((target == y_pred_adv).any()) # Second attack, no batching cl2m = CarliniL2Method(classifier=tfc, targeted=False, max_iter=10, batch_size=1) x_test_adv = cl2m.generate(x_test) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('CW2 Target: %s', target) logger.debug('CW2 Actual: %s', y_pred_adv) logger.info('CW2 Success Rate: %.2f', (np.sum(target == y_pred_adv) / float(len(target)))) self.assertTrue((target != y_pred_adv).any()) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001) # Clean-up session sess.close()
def test_layers(self): classifier, sess = get_classifier_tf() layer_names = classifier.layer_names for i, name in enumerate(layer_names): print(self.x_test.shape) activation_i = classifier.get_activations(self.x_test, i, batch_size=5) activation_name = classifier.get_activations(self.x_test, name, batch_size=5) np.testing.assert_array_equal(activation_name, activation_i) tf.reset_default_graph() sess.close()
def test_tensorflow_mnist_LInf(self): """ First test with the TensorFlowClassifier. :return: """ (_, _), (x_test, y_test) = self.mnist # Build TensorFlowClassifier tfc, sess = get_classifier_tf(from_logits=True) # First attack clinfm = CarliniLInfMethod(classifier=tfc, targeted=True, max_iter=10, eps=0.5) params = {'y': random_targets(y_test, tfc.nb_classes())} x_test_adv = clinfm.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('CW0 Target: %s', target) logger.debug('CW0 Actual: %s', y_pred_adv) logger.info('CW0 Success Rate: %.2f', (np.sum(target == y_pred_adv) / float(len(target)))) self.assertTrue((target == y_pred_adv).any()) # Second attack, no batching clinfm = CarliniLInfMethod(classifier=tfc, targeted=False, max_iter=10, eps=0.5, batch_size=1) x_test_adv = clinfm.generate(x_test) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), -1e-6) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('CW0 Target: %s', target) logger.debug('CW0 Actual: %s', y_pred_adv) logger.info('CW0 Success Rate: %.2f', (np.sum(target != y_pred_adv) / float(len(target)))) self.assertTrue((target != y_pred_adv).any()) # Clean-up session sess.close()
def test_tensorflow_failure_attack(self): """ Test the corner case when attack fails. :return: """ (_, _), (x_test, y_test) = self.mnist # Build TensorFlowClassifier tfc, sess = get_classifier_tf() # Failure attack ead = ElasticNet(classifier=tfc, targeted=True, max_iter=0, binary_search_steps=0, learning_rate=0, initial_const=1) params = {'y': random_targets(y_test, tfc.nb_classes())} x_test_adv = ead.generate(x_test, **params) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) np.testing.assert_almost_equal(x_test, x_test_adv, 3) # Clean-up session sess.close()
def test_tfclassifier(self): """ First test with the TensorFlowClassifier. :return: """ x_test_original = self.x_test.copy() # Build TensorFlowClassifier tfc, sess = get_classifier_tf() # Attack attack_st = SpatialTransformation(tfc, max_translation=10.0, num_translations=3, max_rotation=30.0, num_rotations=3) x_train_adv = attack_st.generate(self.x_train) self.assertAlmostEqual(x_train_adv[0, 8, 13, 0], 0.49004024, delta=0.01) self.assertAlmostEqual(attack_st.fooling_rate, 0.72, delta=0.01) self.assertEqual(attack_st.attack_trans_x, 3) self.assertEqual(attack_st.attack_trans_y, 3) self.assertEqual(attack_st.attack_rot, 30.0) x_test_adv = attack_st.generate(self.x_test) self.assertAlmostEqual(x_test_adv[0, 14, 14, 0], 0.013572651, delta=0.01) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float( np.max(np.abs(x_test_original - self.x_test))), 0.0, delta=0.00001) sess.close()
def test_tensorflow_mnist(self): """ First test with the TensorFlowClassifier. :return: """ x_test_original = self.x_test.copy() # Build TensorFlowClassifier tfc, sess = get_classifier_tf() # Targeted attack zoo = ZooAttack(classifier=tfc, targeted=True, max_iter=100, binary_search_steps=10) params = {'y': random_targets(self.y_test, tfc.nb_classes())} x_test_adv = zoo.generate(self.x_test, **params) self.assertFalse((self.x_test == x_test_adv).all()) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('ZOO target: %s', target) logger.debug('ZOO actual: %s', y_pred_adv) logger.info('ZOO success rate on MNIST: %.2f', (sum(target == y_pred_adv) / float(len(target)))) # Untargeted attack zoo = ZooAttack(classifier=tfc, targeted=False) x_test_adv = zoo.generate(self.x_test) # self.assertFalse((x_test == x_test_adv).all()) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) y_pred = np.argmax(tfc.predict(self.x_test), axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('ZOO actual: %s', y_pred_adv) logger.info('ZOO success rate on MNIST: %.2f', (sum(y_pred != y_pred_adv) / float(len(y_pred)))) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test))), 0.0, delta=0.00001) # Clean-up session sess.close()
def test_tensorflow_failure_attack(self): """ Test the corner case when attack fails. :return: """ x_test_original = self.x_test.copy() # Build TensorFlowClassifier tfc, sess = get_classifier_tf() # Failure attack zoo = ZooAttack(classifier=tfc, max_iter=0, binary_search_steps=0, learning_rate=0) x_test_adv = zoo.generate(self.x_test) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) np.testing.assert_almost_equal(self.x_test, x_test_adv, 3) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test))), 0.0, delta=0.00001) # Clean-up session sess.close()
def test_tensorflow(self): """ First test with the TensorFlowClassifier. :return: """ tfc, sess = get_classifier_tf() attack_ap = AdversarialPatch(tfc, rotation_max=22.5, scale_min=0.1, scale_max=1.0, learning_rate=5.0, batch_size=10, max_iter=500) patch_adv, _ = attack_ap.generate(self.x_train) self.assertAlmostEqual(patch_adv[8, 8, 0], -3.1106631027725005, delta=0.1) self.assertAlmostEqual(patch_adv[14, 14, 0], 18.101, delta=0.1) self.assertAlmostEqual(float(np.sum(patch_adv)), 624.867, delta=0.1) sess.close()
def test_tensorflow_failure_attack_LInf(self): """ Test the corner case when attack is failed. :return: """ (_, _), (x_test, y_test) = self.mnist # Build TensorFlowClassifier tfc, sess = get_classifier_tf(from_logits=True) # Failure attack clinfm = CarliniLInfMethod(classifier=tfc, targeted=True, max_iter=0, learning_rate=0, eps=0.5) params = {'y': random_targets(y_test, tfc.nb_classes())} x_test_adv = clinfm.generate(x_test, **params) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) self.assertTrue(np.allclose(x_test, x_test_adv, atol=1e-3)) # Clean-up session sess.close()
def test_class_gradient(self): classifier, sess = get_classifier_tf(from_logits=True) # Test all gradients label = None gradients = classifier.class_gradient(self.x_test) self.assertEqual(gradients.shape, (NB_TEST, 10, 28, 28, 1)) expected_gradients_1 = np.asarray([ -0.03347399, -0.03195872, -0.02650188, 0.04111874, 0.08676253, 0.03339913, 0.06925241, 0.09387045, 0.15184258, -0.00684002, 0.05070481, 0.01409407, -0.03632583, 0.00151133, 0.05102589, 0.00766463, -0.00898967, 0.00232938, -0.00617045, -0.00201032, 0.00410065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ]) np.testing.assert_array_almost_equal(gradients[0, 5, 14, :, 0], expected_gradients_1, decimal=4) expected_gradients_2 = np.asarray([ -0.09723657, -0.00240533, 0.02445251, -0.00035474, 0.04765627, 0.04286841, 0.07209076, 0.0, 0.0, -0.07938144, -0.00142567, 0.02882954, -0.00049514, 0.04170151, 0.05102589, 0.09544909, -0.04401167, -0.06158172, 0.03359772, -0.00838454, 0.01722163, -0.13376027, 0.08206709, 0.0, 0.0, 0.0, 0.0, 0.0 ]) np.testing.assert_array_almost_equal(gradients[0, 5, :, 14, 0], expected_gradients_2, decimal=4) # Test 1 gradient label = 5 gradients = classifier.class_gradient(self.x_test, label=5) self.assertEqual(gradients.shape, (NB_TEST, 1, 28, 28, 1)) expected_gradients_1 = np.asarray([ -0.03347399, -0.03195872, -0.02650188, 0.04111874, 0.08676253, 0.03339913, 0.06925241, 0.09387045, 0.15184258, -0.00684002, 0.05070481, 0.01409407, -0.03632583, 0.00151133, 0.05102589, 0.00766463, -0.00898967, 0.00232938, -0.00617045, -0.00201032, 0.00410065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ]) np.testing.assert_array_almost_equal(gradients[0, 0, 14, :, 0], expected_gradients_1, decimal=4) expected_gradients_2 = np.asarray([ -0.09723657, -0.00240533, 0.02445251, -0.00035474, 0.04765627, 0.04286841, 0.07209076, 0.0, 0.0, -0.07938144, -0.00142567, 0.02882954, -0.00049514, 0.04170151, 0.05102589, 0.09544909, -0.04401167, -0.06158172, 0.03359772, -0.00838454, 0.01722163, -0.13376027, 0.08206709, 0.0, 0.0, 0.0, 0.0, 0.0 ]) np.testing.assert_array_almost_equal(gradients[0, 0, :, 14, 0], expected_gradients_2, decimal=4) # Test a set of gradients label = array label = np.random.randint(5, size=NB_TEST) gradients = classifier.class_gradient(self.x_test, label=label) self.assertEqual(gradients.shape, (NB_TEST, 1, 28, 28, 1)) expected_gradients_1 = np.asarray([ 0.06860766, 0.065502, 0.08539103, 0.13868105, -0.05520725, -0.18788849, 0.02264893, 0.02980516, 0.2226511, 0.11288887, -0.00678776, 0.02045561, -0.03120914, 0.00642691, 0.08449504, 0.02848018, -0.03251382, 0.00854315, -0.02354656, -0.00767687, 0.01565931, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ]) np.testing.assert_array_almost_equal(gradients[0, 0, 14, :, 0], expected_gradients_1, decimal=4) expected_gradients_2 = np.asarray([ -0.0487146, -0.0171556, -0.03161772, -0.0420007, 0.03360246, -0.01864819, 0.00315916, 0.0, 0.0, -0.07631349, -0.00374462, 0.04229517, -0.01131879, 0.05044588, 0.08449504, 0.12417868, 0.07536847, 0.03906382, 0.09467953, 0.00543209, -0.00504872, -0.03366479, -0.00385999, 0.0, 0.0, 0.0, 0.0, 0.0 ]) np.testing.assert_array_almost_equal(gradients[0, 0, :, 14, 0], expected_gradients_2, decimal=4) tf.reset_default_graph() sess.close()
def test_tensorflow_mnist(self): """ First test with the TensorFlowClassifier. :return: """ (_, _), (x_test, y_test) = self.mnist x_test_original = x_test.copy() # Build TensorFlowClassifier tfc, sess = get_classifier_tf(from_logits=True) # First attack ead = ElasticNet(classifier=tfc, targeted=True, max_iter=2) params = {'y': random_targets(y_test, tfc.nb_classes())} x_test_adv = ead.generate(x_test, **params) expected_x_test_adv = np.asarray([0.45704955, 0.43627003, 0.57238287, 1.0, 0.11541145, 0.12619308, 0.48318917, 0.3457903, 0.17863746, 0.09060935, 0.0, 0.00963121, 0.0, 0.04749763, 0.4058206, 0.17860745, 0.0, 0.9153206, 0.84564775, 0.20603634, 0.10586322, 0.00947509, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) np.testing.assert_array_almost_equal(x_test_adv[0, 14, :, 0], expected_x_test_adv, decimal=6) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('EAD target: %s', target) logger.debug('EAD actual: %s', y_pred_adv) logger.info('EAD success rate on MNIST: %.2f%%', (100 * sum(target == y_pred_adv) / len(target))) self.assertTrue((target == y_pred_adv).any()) # Second attack ead = ElasticNet(classifier=tfc, targeted=False, max_iter=2) params = {'y': random_targets(y_test, tfc.nb_classes())} x_test_adv = ead.generate(x_test, **params) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('EAD target: %s', target) logger.debug('EAD actual: %s', y_pred_adv) logger.info('EAD success rate on MNIST: %.2f%%', (100 * sum(target != y_pred_adv) / float(len(target)))) np.testing.assert_array_equal(y_pred_adv, np.asarray([7, 1, 1, 4, 4, 1, 4, 4, 4, 4])) # Third attack ead = ElasticNet(classifier=tfc, targeted=False, max_iter=2) params = {} x_test_adv = ead.generate(x_test, **params) expected_x_test_adv = np.asarray([0.22866514, 0.21826893, 0.22902338, 0.06268515, 0.0, 0.0, 0.04822975, 0.0, 0.0, 0.0, 0.05555382, 0.0, 0.0, 0.0, 0.38986346, 0.10653087, 0.32385707, 0.98043066, 0.75790393, 0.16486718, 0.16069527, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) np.testing.assert_array_almost_equal(x_test_adv[0, 14, :, 0], expected_x_test_adv, decimal=6) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) y_pred = np.argmax(tfc.predict(x_test), axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('EAD target: %s', y_pred) logger.debug('EAD actual: %s', y_pred_adv) logger.info('EAD success rate: %.2f%%', (100 * sum(y_pred != y_pred_adv) / float(len(y_pred)))) np.testing.assert_array_equal(y_pred_adv, np.asarray([0, 4, 7, 9, 0, 7, 7, 3, 0, 7])) # First attack without batching ead_wob = ElasticNet(classifier=tfc, targeted=True, max_iter=2, batch_size=1) params = {'y': random_targets(y_test, tfc.nb_classes())} x_test_adv = ead_wob.generate(x_test, **params) expected_x_test_adv = np.asarray([0.3287169, 0.31374657, 0.42853343, 0.8994576, 0.19850709, 0.11997936, 0.5622535, 0.43854535, 0.19387433, 0.12516324, 0.0, 0.10933565, 0.02162433, 0.07120894, 0.95224255, 0.3072921, 0.48966524, 1., 0.3814998, 0.15782641, 0.52283823, 0.12852049, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) np.testing.assert_array_almost_equal(x_test_adv[0, 14, :, 0], expected_x_test_adv, decimal=6) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('EAD target: %s', target) logger.debug('EAD actual: %s', y_pred_adv) logger.info('EAD success rate: %.2f%%', (100 * sum(target == y_pred_adv) / float(len(target)))) self.assertTrue((target == y_pred_adv).any()) # Second attack without batching ead_wob = ElasticNet(classifier=tfc, targeted=False, max_iter=2, batch_size=1) params = {'y': random_targets(y_test, tfc.nb_classes())} x_test_adv = ead_wob.generate(x_test, **params) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('EAD target: %s', target) logger.debug('EAD actual: %s', y_pred_adv) logger.info('EAD success rate: %.2f%%', (100 * sum(target != y_pred_adv) / float(len(target)))) np.testing.assert_array_equal(y_pred_adv, np.asarray([7, 1, 1, 4, 4, 1, 4, 4, 4, 4])) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001) # Close session sess.close()
def test_tensorflow_mnist(self): (_, _), (x_test, y_test) = self.mnist classifier, sess = get_classifier_tf() self._test_backend_mnist(classifier, x_test, y_test)
def test_tensorflow_mnist_targeted(self): (_, _), (x_test, _) = self.mnist classifier, sess = get_classifier_tf() self._test_mnist_targeted(classifier, x_test)
def test_nb_classes(self): classifier, sess = get_classifier_tf() self.assertEqual(classifier.nb_classes(), 10) tf.reset_default_graph() sess.close()
def test_input_shape(self): classifier, sess = get_classifier_tf() self.assertEqual(classifier.input_shape, (28, 28, 1)) tf.reset_default_graph() sess.close()
def tearDownClass(cls): _, sess = get_classifier_tf() tf.reset_default_graph() sess.close()
def test_tensorflow_mnist(self): """ First test with the TensorFlowClassifier. :return: """ (_, _), (x_test, y_test) = self.mnist x_test_original = x_test.copy() # Build TensorFlowClassifier tfc, sess = get_classifier_tf() # First targeted attack and norm=2 hsj = HopSkipJump(classifier=tfc, targeted=True, max_iter=2, max_eval=100, init_eval=10) params = {'y': random_targets(y_test, tfc.nb_classes())} x_test_adv = hsj.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) self.assertTrue((target == y_pred_adv).any()) # First targeted attack and norm=np.inf hsj = HopSkipJump(classifier=tfc, targeted=True, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf) params = {'y': random_targets(y_test, tfc.nb_classes())} x_test_adv = hsj.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) self.assertTrue((target == y_pred_adv).any()) # Second untargeted attack and norm=2 hsj = HopSkipJump(classifier=tfc, targeted=False, max_iter=2, max_eval=100, init_eval=10) x_test_adv = hsj.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) y_pred = np.argmax(tfc.predict(x_test), axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) self.assertTrue((y_pred != y_pred_adv).any()) # Second untargeted attack and norm=np.inf hsj = HopSkipJump(classifier=tfc, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf) x_test_adv = hsj.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) y_pred = np.argmax(tfc.predict(x_test), axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) self.assertTrue((y_pred != y_pred_adv).any()) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001) # Clean-up session sess.close()