def test_two_attacks(self): (x_train, y_train), (x_test, y_test) = self.mnist x_test_original = x_test.copy() attack1 = FastGradientMethod(classifier=self.classifier, batch_size=16) attack2 = DeepFool(classifier=self.classifier, max_iter=5, batch_size=16) x_test_adv = attack1.generate(x_test) predictions = np.argmax(self.classifier.predict(x_test_adv), axis=1) accuracy = np.sum(predictions == np.argmax(y_test, axis=1)) / NB_TEST adv_trainer = AdversarialTrainer(self.classifier, attacks=[attack1, attack2]) adv_trainer.fit(x_train, y_train, nb_epochs=2, batch_size=16) predictions_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1) accuracy_new = np.sum( predictions_new == np.argmax(y_test, axis=1)) / NB_TEST self.assertEqual(accuracy_new, 0.36) self.assertEqual(accuracy, 0.13) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_transfer(self): (x_train, y_train), (x_test, y_test) = self.mnist attack = DeepFool(self.classifier_tf) x_test_adv = attack.generate(x_test) preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1) acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST adv_trainer = AdversarialTrainer(self.classifier_k, attack) adv_trainer.fit(x_train, y_train, nb_epochs=2, batch_size=6) preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1) acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST self.assertGreaterEqual(acc_new, acc * ACCURACY_DROP) logger.info('Accuracy before adversarial training: %.2f%%', (acc * 100)) logger.info('Accuracy after adversarial training: %.2f%%', (acc_new * 100))
def test_fit_predict(self): (x_train, y_train), (x_test, y_test) = self.mnist attack = FastGradientMethod(self.classifier_k) x_test_adv = attack.generate(x_test) preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1) acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST adv_trainer = AdversarialTrainer(self.classifier_k, attack) adv_trainer.fit(x_train, y_train, nb_epochs=5, batch_size=128) preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1) acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST self.assertGreaterEqual(acc_new, acc * accuracy_drop) print('\nAccuracy before adversarial training: %.2f%%' % (acc * 100)) print('\nAccuracy after adversarial training: %.2f%%' % (acc_new * 100))
def test_two_attacks(self): (x_train, y_train), (x_test, y_test) = self.mnist attack1 = FastGradientMethod(self.classifier_k) attack2 = DeepFool(self.classifier_tf) x_test_adv = attack1.generate(x_test) preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1) acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST adv_trainer = AdversarialTrainer(self.classifier_k, attacks=[attack1, attack2]) adv_trainer.fit(x_train, y_train, nb_epochs=5, batch_size=128) preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1) acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST # No reason to assert the newer accuracy is higher. It might go down slightly self.assertGreaterEqual(acc_new, acc * ACCURACY_DROP) logger.info('Accuracy before adversarial training: %.2f%%', (acc * 100)) logger.info('\nAccuracy after adversarial training: %.2f%%', (acc_new * 100))
def test_transfer(self): (x_train, y_train), (x_test, y_test) = self.mnist x_test_original = x_test.copy() attack = DeepFool(self.classifier_tf) x_test_adv = attack.generate(x_test) preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1) acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST adv_trainer = AdversarialTrainer(self.classifier_k, attack) adv_trainer.fit(x_train, y_train, nb_epochs=2, batch_size=6) preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1) acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST self.assertGreaterEqual(acc_new, acc * ACCURACY_DROP) logger.info('Accuracy before adversarial training: %.2f%%', (acc * 100)) logger.info('Accuracy after adversarial training: %.2f%%', (acc_new * 100)) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_two_attacks(self): (x_train, y_train), (x_test, y_test) = self.mnist x_test_original = x_test.copy() attack1 = FastGradientMethod(self.classifier_k) attack2 = DeepFool(self.classifier_tf) x_test_adv = attack1.generate(x_test) preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1) acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST adv_trainer = AdversarialTrainer(self.classifier_k, attacks=[attack1, attack2]) adv_trainer.fit(x_train, y_train, nb_epochs=5, batch_size=128) preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1) acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST # No reason to assert the newer accuracy is higher. It might go down slightly self.assertGreaterEqual(acc_new, acc * ACCURACY_DROP) logger.info('Accuracy before adversarial training: %.2f%%', (acc * 100)) logger.info('\nAccuracy after adversarial training: %.2f%%', (acc_new * 100)) # Check that x_test has not been modified by attack and classifier self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
attack = ProjectedGradientDescent(classifier, eps=8 / 255, eps_step=1 / 255, max_iter=20, batch_size=512) x_test_pgd = attack.generate(x_test, y_test) # np.save('./data/' + dataset + '_data/model/' + model_name + '_y_' + attack_name + '.npy', x_test_pgd) # Evaluate the benign trained model on adv test set labels_pgd = np.argmax(classifier.predict(x_test_pgd), axis=1) print('Accuracy on original PGD adversarial samples: %.2f%%' % (np.sum(labels_pgd == labels_true) / x_test.shape[0] * 100)) trainer = AdversarialTrainer(classifier, attack, ratio=1.0) trainer.fit(x_train, y_train, nb_epochs=60, batch_size=1024) classifier.save(filename='adv_' + model_name + '.h5', path='../data/' + dataset + '_data/model/') # Evaluate the adversarially trained model on clean test set labels_true = np.argmax(y_test, axis=1) labels_test = np.argmax(classifier.predict(x_test), axis=1) print('Accuracy test set: %.2f%%' % (np.sum(labels_test == labels_true) / x_test.shape[0] * 100)) # Evaluate the adversarially trained model on original adversarial samples labels_pgd = np.argmax(classifier.predict(x_test_pgd), axis=1) print('Accuracy on original PGD adversarial samples: %.2f%%' % (np.sum(labels_pgd == labels_true) / x_test.shape[0] * 100))
loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=labels_ph)) optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) # Step 3: Create the ART classifier classifier = TFClassifier(clip_values=(min_pixel_value, max_pixel_value), input_ph=input_ph, output=logits, labels_ph=labels_ph, train=train, loss=loss, learning=None, sess=sess) # Step 4: Train the ART classifier attack=art.attacks.CarliniLInfMethod(classifier) trainer = AdversarialTrainer(classifier,attack, ratio=1.0) trainer.fit(x_train, y_train, batch_size=128, nb_epochs=3) # Step 5: Evaluate the ART classifier on benign test examples ##hello predictions = classifier.predict(x_test) accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test) print('Accuracy on benign test examples: {}%'.format(accuracy * 100)) # Step 6: Generate adversarial test examples attack = art.attacks.SaliencyMapMethod(classifier=classifier) x_test_adv = attack.generate(x=x_test) # Step 7: Evaluate the ART classifier on adversarial test examples predictions = classifier.predict(x_test_adv) accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)
from art.defences import AdversarialTrainer # get a new untrained model and warp it new_model = mnist_cnn_model(x_train, y_train, x_test, y_test, epochs=0) defended_model = KerasClassifier(clip_values=(0, 1), model=new_model) # define the attack we are using fgsm = FastGradientMethod(defended_model) # Create the `AdversarialTrainer` instance. # Train the model and evaluate it on the test data. # In[ ]: # define the adversarial trainer and train the new network adversarial_tranier = AdversarialTrainer(defended_model, fgsm) adversarial_tranier.fit(x_train, y_train, batch_size=100, nb_epochs=2) # evaluate how good our model is defended_model._model.evaluate(x_test, y_test) # Calculate the `empirical robustness` for our now hopfully more robust model # In[ ]: # calculate the empiracal robustness print('robustness of the defended model', empirical_robustness(defended_model, x_test[0:], 'fgsm', {})) x_adv = fgsm.generate(x_test[0].reshape((1, 28, 28, 1))) print('class prediction for the adversarial sample:', clf.predict(x_adv.reshape((1, 28, 28, 1)))) plt.imshow(x_adv.reshape(28, 28), cmap="gray_r")