def setUpClass(cls): k.clear_session() (x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist') cls.x_train = x_train[:NB_TRAIN] cls.y_train = y_train[:NB_TRAIN] cls.x_test = x_test[:NB_TEST] cls.y_test = y_test[:NB_TEST] # Use twice the same classifier for unittesting, in application they would be different classifier_1 = get_classifier_kr() classifier_2 = get_classifier_kr() cls.ensemble = EnsembleClassifier(classifiers=[classifier_1, classifier_2], clip_values=(0, 1))
def test_save(self): path = 'tmp' filename = 'model.h5' classifier = get_classifier_kr() classifier.save(filename, path=path) self.assertTrue(os.path.isfile(os.path.join(path, filename))) os.remove(os.path.join(path, filename))
def test_repr(self): classifier = get_classifier_kr() repr_ = repr(classifier) self.assertIn('art.classifiers.keras.KerasClassifier', repr_) self.assertIn('use_logits=False, channel_index=3', repr_) self.assertIn('clip_values=(0, 1), defences=None, preprocessing=(0, 1)', repr_) self.assertIn('input_layer=0, output_layer=0', repr_)
def test_keras_mnist(self): (x_train, y_train), (x_test, y_test) = self.mnist x_test_original = x_test.copy() # Keras classifier classifier = get_classifier_kr() scores = classifier._model.evaluate(x_train, y_train) logger.info('[Keras, MNIST] Accuracy on training set: %.2f%%', (scores[1] * 100)) scores = classifier._model.evaluate(x_test, y_test) logger.info('[Keras, MNIST] Accuracy on test set: %.2f%%', (scores[1] * 100)) # targeted # Generate random target classes nb_classes = np.unique(np.argmax(y_test, axis=1)).shape[0] targets = np.random.randint(nb_classes, size=NB_TEST) while (targets == np.argmax(y_test, axis=1)).any(): targets = np.random.randint(nb_classes, size=NB_TEST) # Perform attack df = SaliencyMapMethod(classifier, theta=1, batch_size=100) x_test_adv = df.generate(x_test, y=to_categorical(targets, nb_classes)) self.assertFalse((x_test == x_test_adv).all()) self.assertFalse((0. == x_test_adv).all()) y_pred = get_labels_np_array(classifier.predict(x_test_adv)) self.assertFalse((y_test == y_pred).all()) accuracy = np.sum( np.argmax(y_pred, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0] logger.info('Accuracy on adversarial examples: %.2f%%', (accuracy * 100)) # untargeted df = SaliencyMapMethod(classifier, theta=1, batch_size=100) x_test_adv = df.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) self.assertFalse((0. == x_test_adv).all()) y_pred = get_labels_np_array(classifier.predict(x_test_adv)) self.assertFalse((y_test == y_pred).all()) accuracy = np.sum( np.argmax(y_pred, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0] logger.info('Accuracy on adversarial examples: %.2f%%', (accuracy * 100)) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_learning_phase(self): classifier = get_classifier_kr() self.assertFalse(hasattr(classifier, '_learning_phase')) classifier.set_learning_phase(False) self.assertFalse(classifier.learning_phase) classifier.set_learning_phase(True) self.assertTrue(classifier.learning_phase) self.assertTrue(hasattr(classifier, '_learning_phase'))
def test_layers(self): classifier = get_classifier_kr() self.assertEqual(len(classifier.layer_names), 3) layer_names = classifier.layer_names for i, name in enumerate(layer_names): activation_i = classifier.get_activations(self.x_test, i, batch_size=128) activation_name = classifier.get_activations(self.x_test, name, batch_size=128) np.testing.assert_array_equal(activation_name, activation_i)
def test_keras_mnist(self): (x_train, y_train), (x_test, y_test) = self.mnist classifier = get_classifier_kr() scores = classifier._model.evaluate(x_train, y_train) logging.info('[Keras, MNIST] Accuracy on training set: %.2f%%', (scores[1] * 100)) scores = classifier._model.evaluate(x_test, y_test) logging.info('[Keras, MNIST] Accuracy on test set: %.2f%%', (scores[1] * 100)) self._test_backend_mnist(classifier, x_test, y_test)
def test_fit(self): labels = np.argmax(self.y_test, axis=1) classifier = get_classifier_kr() accuracy = np.sum(np.argmax(classifier.predict(self.x_test), axis=1) == labels) / NB_TEST logger.info('Accuracy: %.2f%%', (accuracy * 100)) classifier.fit(self.x_train, self.y_train, batch_size=BATCH_SIZE, nb_epochs=2) accuracy_2 = np.sum(np.argmax(classifier.predict(self.x_test), axis=1) == labels) / NB_TEST logger.info('Accuracy: %.2f%%', (accuracy_2 * 100)) self.assertEqual(accuracy, 0.32) self.assertAlmostEqual(accuracy_2, 0.735, delta=0.02)
def test_defences_predict(self): clip_values = (0, 1) fs = FeatureSqueezing(clip_values=clip_values, bit_depth=2) jpeg = JpegCompression(clip_values=clip_values, apply_predict=True) smooth = SpatialSmoothing() classifier_ = get_classifier_kr() classifier = KerasClassifier(clip_values=clip_values, model=classifier_._model, defences=[fs, jpeg, smooth]) self.assertEqual(len(classifier.defences), 3) predictions_classifier = classifier.predict(self.x_test) # Apply the same defences by hand x_test_defense = self.x_test x_test_defense, _ = fs(x_test_defense, self.y_test) x_test_defense, _ = jpeg(x_test_defense, self.y_test) x_test_defense, _ = smooth(x_test_defense, self.y_test) classifier = get_classifier_kr() predictions_check = classifier._model.predict(x_test_defense) # Check that the prediction results match np.testing.assert_array_almost_equal(predictions_classifier, predictions_check, decimal=4)
def test_shapes(self): classifier = get_classifier_kr() predictions = classifier.predict(self.x_test) self.assertEqual(predictions.shape, self.y_test.shape) self.assertEqual(classifier.nb_classes(), 10) class_gradients = classifier.class_gradient(self.x_test[:11]) self.assertEqual(class_gradients.shape, tuple([11, 10] + list(self.x_test[1].shape))) loss_gradients = classifier.loss_gradient(self.x_test[:11], self.y_test[:11]) self.assertEqual(loss_gradients.shape, self.x_test[:11].shape)
def setUpClass(cls): k.set_learning_phase(1) (x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist') x_train, y_train, x_test, y_test = x_train[: NB_TRAIN], y_train[: NB_TRAIN], x_test[: NB_TEST], y_test[: NB_TEST] cls.mnist = (x_train, y_train), (x_test, y_test) # Keras classifier cls.classifier_k = get_classifier_kr()
def _run_tests(_loss_name, _loss_type, _y_test_pred_expected, _class_gradient_probabilities_expected, _loss_gradient_expected, _from_logits): master_seed(1234) classifier = get_classifier_kr(loss_name=_loss_name, loss_type=_loss_type, from_logits=_from_logits) y_test_pred = np.argmax(classifier.predict(x=self.x_test), axis=1) np.testing.assert_array_equal(y_test_pred, _y_test_pred_expected) class_gradient = classifier.class_gradient(self.x_test, label=5) np.testing.assert_array_almost_equal(class_gradient[99, 0, 14, :, 0], _class_gradient_probabilities_expected) loss_gradient = classifier.loss_gradient(x=self.x_test, y=self.y_test) np.testing.assert_array_almost_equal(loss_gradient[99, 14, :, 0], _loss_gradient_expected)
def test_fit_generator(self): classifier = get_classifier_kr() labels = np.argmax(self.y_test, axis=1) accuracy = np.sum(np.argmax(classifier.predict(self.x_test), axis=1) == labels) / NB_TEST logger.info('Accuracy: %.2f%%', (accuracy * 100)) gen = generator_fit(self.x_train, self.y_train, batch_size=BATCH_SIZE) data_gen = KerasDataGenerator(generator=gen, size=NB_TRAIN, batch_size=BATCH_SIZE) classifier.fit_generator(generator=data_gen, nb_epochs=3) accuracy_2 = np.sum(np.argmax(classifier.predict(self.x_test), axis=1) == labels) / NB_TEST logger.info('Accuracy: %.2f%%', (accuracy_2 * 100)) self.assertEqual(accuracy, 0.32) self.assertAlmostEqual(accuracy_2, 0.36, delta=0.02)
def test_kera_mnist_partial_grads(self): (_, _), (x_test, y_test) = self.mnist classifier = get_classifier_kr(from_logits=True) attack = DeepFool(classifier, max_iter=2, nb_grads=3) x_test_adv = attack.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) test_y_pred = get_labels_np_array(classifier.predict(x_test_adv)) self.assertFalse((y_test == test_y_pred).all()) accuracy = np.sum( np.argmax(test_y_pred, axis=1) == np.argmax( y_test, axis=1)) / y_test.shape[0] logger.info('Accuracy on adversarial test examples: %.2f%%', (accuracy * 100))
def test_fit_kwargs(self): def get_lr(_): return 0.01 # Test a valid callback classifier = get_classifier_kr() kwargs = {'callbacks': [LearningRateScheduler(get_lr)]} classifier.fit(self.x_train, self.y_train, batch_size=BATCH_SIZE, nb_epochs=1, **kwargs) # Test failure for invalid parameters kwargs = {'epochs': 1} with self.assertRaises(TypeError) as context: classifier.fit(self.x_train, self.y_train, batch_size=BATCH_SIZE, nb_epochs=1, **kwargs) self.assertIn('multiple values for keyword argument', str(context.exception))
def test_keras_mnist(self): """ Second test with the KerasClassifier. :return: """ (_, _), (x_test, y_test) = self.mnist x_test_original = x_test.copy() # Build KerasClassifier krc = get_classifier_kr() # First attack ead = ElasticNet(classifier=krc, targeted=True, max_iter=2) y_target = to_categorical(np.asarray([6, 6, 7, 4, 9, 7, 9, 0, 1, 0]), nb_classes=10) x_test_adv = ead.generate(x_test, y=y_target) expected_x_test_adv = np.asarray([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00183569, 0.0, 0.0, 0.49765405, 1., 0.6467149, 0.0033755, 0.0052456, 0.0, 0.01104407, 0.00495547, 0.02747423, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) np.testing.assert_array_almost_equal(x_test_adv[2, 14, :, 0], expected_x_test_adv, decimal=6) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) target = np.argmax(y_target, axis=1) y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1) logger.debug('EAD target: %s', target) logger.debug('EAD actual: %s', y_pred_adv) logger.info('EAD success rate: %.2f%%', (100 * sum(target == y_pred_adv) / float(len(target)))) self.assertTrue((target == y_pred_adv).any()) # Second attack ead = ElasticNet(classifier=krc, targeted=False, max_iter=2) y_target = to_categorical(np.asarray([9, 5, 6, 7, 1, 6, 1, 5, 8, 5]), nb_classes=10) x_test_adv = ead.generate(x_test, y=y_target) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1) logger.debug('EAD target: %s', y_target) logger.debug('EAD actual: %s', y_pred_adv) logger.info('EAD success rate: %.2f', (100 * sum(target != y_pred_adv) / float(len(target)))) self.assertTrue((target != y_pred_adv).any()) np.testing.assert_array_equal(y_pred_adv, np.asarray([7, 1, 1, 4, 4, 1, 4, 4, 4, 4])) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001) k.clear_session()
def test_keras_mnist_LInf(self): """ Second test with the KerasClassifier. :return: """ (_, _), (x_test, y_test) = self.mnist # Build KerasClassifier krc = get_classifier_kr(from_logits=True) # First attack clinfm = CarliniLInfMethod(classifier=krc, targeted=True, max_iter=10, eps=0.5) params = {'y': random_targets(y_test, krc.nb_classes())} x_test_adv = clinfm.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertLessEqual(np.amax(x_test_adv), 1.000001) self.assertGreaterEqual(np.amin(x_test_adv), -1e-6) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1) logger.debug('CW0 Target: %s', target) logger.debug('CW0 Actual: %s', y_pred_adv) logger.info('CW0 Success Rate: %.2f', (np.sum(target == y_pred_adv) / float(len(target)))) self.assertTrue((target == y_pred_adv).any()) # Second attack clinfm = CarliniLInfMethod(classifier=krc, targeted=False, max_iter=10, eps=0.5) x_test_adv = clinfm.generate(x_test) self.assertLessEqual(np.amax(x_test_adv), 1.000001) self.assertGreaterEqual(np.amin(x_test_adv), -1e-6) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1) logger.debug('CW0 Target: %s', target) logger.debug('CW0 Actual: %s', y_pred_adv) logger.info('CW0 Success Rate: %.2f', (np.sum(target != y_pred_adv) / float(len(target)))) self.assertTrue((target != y_pred_adv).any()) # Clean-up k.clear_session()
def test_fit_image_generator(self): classifier = get_classifier_kr() labels_test = np.argmax(self.y_test, axis=1) accuracy = np.sum(np.argmax(classifier.predict(self.x_test), axis=1) == labels_test) / NB_TEST logger.info('Accuracy: %.2f%%', (accuracy * 100)) keras_gen = ImageDataGenerator(width_shift_range=0.075, height_shift_range=0.075, rotation_range=12, shear_range=0.075, zoom_range=0.05, fill_mode='constant', cval=0) keras_gen.fit(self.x_train) data_gen = KerasDataGenerator(generator=keras_gen.flow(self.x_train, self.y_train, batch_size=BATCH_SIZE), size=NB_TRAIN, batch_size=BATCH_SIZE) classifier.fit_generator(generator=data_gen, nb_epochs=5) accuracy_2 = np.sum(np.argmax(classifier.predict(self.x_test), axis=1) == labels_test) / NB_TEST logger.info('Accuracy: %.2f%%', (accuracy_2 * 100)) self.assertEqual(accuracy, 0.32) self.assertAlmostEqual(accuracy_2, 0.35, delta=0.04)
def test_keras_mnist_L2(self): """ Second test with the KerasClassifier. :return: """ (_, _), (x_test, y_test) = self.mnist x_test_original = x_test.copy() # Build KerasClassifier krc = get_classifier_kr(from_logits=True) # First attack cl2m = CarliniL2Method(classifier=krc, targeted=True, max_iter=10) y_target = [6, 6, 7, 4, 9, 7, 9, 0, 1, 0] x_test_adv = cl2m.generate(x_test, y=to_categorical(y_target, nb_classes=10)) self.assertFalse((x_test == x_test_adv).all()) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1) logger.debug('CW2 Target: %s', y_target) logger.debug('CW2 Actual: %s', y_pred_adv) logger.info('CW2 Success Rate: %.2f', (np.sum(y_target == y_pred_adv) / float(len(y_target)))) self.assertTrue((y_target == y_pred_adv).any()) # Second attack cl2m = CarliniL2Method(classifier=krc, targeted=False, max_iter=10) x_test_adv = cl2m.generate(x_test) self.assertLessEqual(np.amax(x_test_adv), 1.0) self.assertGreaterEqual(np.amin(x_test_adv), 0.0) y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1) logger.debug('CW2 Target: %s', y_target) logger.debug('CW2 Actual: %s', y_pred_adv) logger.info('CW2 Success Rate: %.2f', (np.sum(y_target != y_pred_adv) / float(len(y_target)))) self.assertTrue((y_target != y_pred_adv).any()) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001) # Clean-up k.clear_session()
def test_keras_mnist(self): """ Second test with the KerasClassifier. :return: """ (_, _), (x_test, y_test) = self.mnist x_test_original = x_test.copy() # Build KerasClassifier krc = get_classifier_kr() # First targeted attack boundary = BoundaryAttack(classifier=krc, targeted=True, max_iter=20) params = {'y': random_targets(y_test, krc.nb_classes())} x_test_adv = boundary.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1) self.assertTrue((target == y_pred_adv).any()) # Second untargeted attack boundary = BoundaryAttack(classifier=krc, targeted=False, max_iter=20) x_test_adv = boundary.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) y_pred = np.argmax(krc.predict(x_test), axis=1) y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1) self.assertTrue((y_pred != y_pred_adv).any()) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001) # Clean-up session k.clear_session()
def test_keras_mnist(self): (x_train, y_train), (x_test, y_test) = self.mnist x_test_original = x_test.copy() # Keras classifier classifier = get_classifier_kr(from_logits=True) scores = classifier._model.evaluate(x_train, y_train) logger.info('[Keras, MNIST] Accuracy on training set: %.2f%%', (scores[1] * 100)) scores = classifier._model.evaluate(x_test, y_test) logger.info('[Keras, MNIST] Accuracy on test set: %.2f%%', (scores[1] * 100)) attack = DeepFool(classifier, max_iter=5, batch_size=11) x_train_adv = attack.generate(x_train) x_test_adv = attack.generate(x_test) self.assertFalse((x_train == x_train_adv).all()) self.assertFalse((x_test == x_test_adv).all()) train_y_pred = get_labels_np_array(classifier.predict(x_train_adv)) test_y_pred = get_labels_np_array(classifier.predict(x_test_adv)) self.assertFalse((y_train == train_y_pred).all()) self.assertFalse((y_test == test_y_pred).all()) accuracy = np.sum( np.argmax(train_y_pred, axis=1) == np.argmax( y_train, axis=1)) / y_train.shape[0] logger.info('Accuracy on adversarial train examples: %.2f%%', (accuracy * 100)) accuracy = np.sum( np.argmax(test_y_pred, axis=1) == np.argmax( y_test, axis=1)) / y_test.shape[0] logger.info('Accuracy on adversarial test examples: %.2f%%', (accuracy * 100)) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_keras(self): """ Second test with the KerasClassifier. :return: """ krc = get_classifier_kr() attack_ap = AdversarialPatch(krc, rotation_max=22.5, scale_min=0.1, scale_max=1.0, learning_rate=5.0, batch_size=10, max_iter=500) master_seed(1234) patch_adv, _ = attack_ap.generate(self.x_train) self.assertAlmostEqual(patch_adv[8, 8, 0], -3.494, delta=0.2) self.assertAlmostEqual(patch_adv[14, 14, 0], 18.402, delta=0.2) self.assertAlmostEqual(float(np.sum(patch_adv)), 1099.293, delta=50)