def test_two_attacks_with_generator(self):
(x_train, y_train), (x_test, y_test) = self.mnist
x_train_original = x_train.copy()
class MyDataGenerator(DataGenerator):
def __init__(self, x, y, size, batch_size):
self.x = x
self.y = y
self.size = size
self.batch_size = batch_size
def get_batch(self):
ids = np.random.choice(self.size, size=min(self.size, self.batch_size), replace=False)
return self.x[ids], self.y[ids]
generator = MyDataGenerator(x_train, y_train, x_train.shape[0], 128)
attack1 = FastGradientMethod(self.classifier_k)
attack2 = DeepFool(self.classifier_tf)
x_test_adv = attack1.generate(x_test)
preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1)
acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST
adv_trainer = AdversarialTrainer(self.classifier_k, attacks=[attack1, attack2])
adv_trainer.fit_generator(generator, nb_epochs=5)
preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1)
acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST
# No reason to assert the newer accuracy is higher. It might go down slightly
self.assertGreaterEqual(acc_new, acc * ACCURACY_DROP)
logger.info('Accuracy before adversarial training: %.2f%%', (acc * 100))
logger.info('\nAccuracy after adversarial training: %.2f%%', (acc_new * 100))
# Finally assert that the original training data hasn't changed:
self.assertTrue((x_train == x_train_original).all())
def test_two_attacks(self):
(x_train, y_train), (x_test, y_test) = self.mnist
x_test_original = x_test.copy()
attack1 = FastGradientMethod(classifier=self.classifier, batch_size=16)
attack2 = DeepFool(classifier=self.classifier,
max_iter=5,
batch_size=16)
x_test_adv = attack1.generate(x_test)
predictions = np.argmax(self.classifier.predict(x_test_adv), axis=1)
accuracy = np.sum(predictions == np.argmax(y_test, axis=1)) / NB_TEST
adv_trainer = AdversarialTrainer(self.classifier,
attacks=[attack1, attack2])
adv_trainer.fit(x_train, y_train, nb_epochs=2, batch_size=16)
predictions_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1)
accuracy_new = np.sum(
predictions_new == np.argmax(y_test, axis=1)) / NB_TEST
self.assertEqual(accuracy_new, 0.36)
self.assertEqual(accuracy, 0.13)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))),
0.0,
delta=0.00001)
def test_two_attacks_with_generator(self):
(x_train, y_train), (x_test, y_test) = self.mnist
x_train_original = x_train.copy()
x_test_original = x_test.copy()
class MyDataGenerator(DataGenerator):
def __init__(self, x, y, size, batch_size):
super().__init__(size=size, batch_size=batch_size)
self.x = x
self.y = y
self._size = size
self._batch_size = batch_size
def get_batch(self):
ids = np.random.choice(self.size,
size=min(self.size, self.batch_size),
replace=False)
return self.x[ids], self.y[ids]
generator = MyDataGenerator(x_train,
y_train,
size=x_train.shape[0],
batch_size=16)
attack1 = FastGradientMethod(classifier=self.classifier, batch_size=16)
attack2 = DeepFool(classifier=self.classifier,
max_iter=5,
batch_size=16)
x_test_adv = attack1.generate(x_test)
predictions = np.argmax(self.classifier.predict(x_test_adv), axis=1)
accuracy = np.sum(predictions == np.argmax(y_test, axis=1)) / NB_TEST
adv_trainer = AdversarialTrainer(self.classifier,
attacks=[attack1, attack2])
adv_trainer.fit_generator(generator, nb_epochs=3)
predictions_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1)
accuracy_new = np.sum(
predictions_new == np.argmax(y_test, axis=1)) / NB_TEST
self.assertAlmostEqual(accuracy_new, 0.25, delta=0.02)
self.assertAlmostEqual(accuracy, 0.11, delta=0.0)
# Check that x_train and x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_train_original -
x_train))),
0.0,
delta=0.00001)
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))),
0.0,
delta=0.00001)
def test_transfer(self):
(x_train, y_train), (x_test, y_test) = self.mnist
attack = DeepFool(self.classifier_tf)
x_test_adv = attack.generate(x_test)
preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1)
acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST
adv_trainer = AdversarialTrainer(self.classifier_k, attack)
adv_trainer.fit(x_train, y_train, nb_epochs=2, batch_size=6)
preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1)
acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST
self.assertGreaterEqual(acc_new, acc * ACCURACY_DROP)
logger.info('Accuracy before adversarial training: %.2f%%', (acc * 100))
logger.info('Accuracy after adversarial training: %.2f%%', (acc_new * 100))
def test_fit_predict(self):
(x_train, y_train), (x_test, y_test) = self.mnist
attack = FastGradientMethod(self.classifier_k)
x_test_adv = attack.generate(x_test)
preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1)
acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST
adv_trainer = AdversarialTrainer(self.classifier_k, attack)
adv_trainer.fit(x_train, y_train, nb_epochs=5, batch_size=128)
preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1)
acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST
self.assertGreaterEqual(acc_new, acc * accuracy_drop)
print('\nAccuracy before adversarial training: %.2f%%' % (acc * 100))
print('\nAccuracy after adversarial training: %.2f%%' %
(acc_new * 100))
def test_two_attacks(self):
(x_train, y_train), (x_test, y_test) = self.mnist
attack1 = FastGradientMethod(self.classifier_k)
attack2 = DeepFool(self.classifier_tf)
x_test_adv = attack1.generate(x_test)
preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1)
acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST
adv_trainer = AdversarialTrainer(self.classifier_k, attacks=[attack1, attack2])
adv_trainer.fit(x_train, y_train, nb_epochs=5, batch_size=128)
preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1)
acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST
# No reason to assert the newer accuracy is higher. It might go down slightly
self.assertGreaterEqual(acc_new, acc * ACCURACY_DROP)
logger.info('Accuracy before adversarial training: %.2f%%', (acc * 100))
logger.info('\nAccuracy after adversarial training: %.2f%%', (acc_new * 100))
def test_transfer(self):
(x_train, y_train), (x_test, y_test) = self.mnist
x_test_original = x_test.copy()
attack = DeepFool(self.classifier_tf)
x_test_adv = attack.generate(x_test)
preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1)
acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST
adv_trainer = AdversarialTrainer(self.classifier_k, attack)
adv_trainer.fit(x_train, y_train, nb_epochs=2, batch_size=6)
preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1)
acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST
self.assertGreaterEqual(acc_new, acc * ACCURACY_DROP)
logger.info('Accuracy before adversarial training: %.2f%%', (acc * 100))
logger.info('Accuracy after adversarial training: %.2f%%', (acc_new * 100))
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_two_attacks(self):
(x_train, y_train), (x_test, y_test) = self.mnist
x_test_original = x_test.copy()
attack1 = FastGradientMethod(self.classifier_k)
attack2 = DeepFool(self.classifier_tf)
x_test_adv = attack1.generate(x_test)
preds = np.argmax(self.classifier_k.predict(x_test_adv), axis=1)
acc = np.sum(preds == np.argmax(y_test, axis=1)) / NB_TEST
adv_trainer = AdversarialTrainer(self.classifier_k, attacks=[attack1, attack2])
adv_trainer.fit(x_train, y_train, nb_epochs=5, batch_size=128)
preds_new = np.argmax(adv_trainer.predict(x_test_adv), axis=1)
acc_new = np.sum(preds_new == np.argmax(y_test, axis=1)) / NB_TEST
# No reason to assert the newer accuracy is higher. It might go down slightly
self.assertGreaterEqual(acc_new, acc * ACCURACY_DROP)
logger.info('Accuracy before adversarial training: %.2f%%', (acc * 100))
logger.info('\nAccuracy after adversarial training: %.2f%%', (acc_new * 100))
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
