def _test_backend_mnist(self, classifier):
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
df = VirtualAdversarialMethod(classifier)
x_test_adv = df.generate(x_test, eps=1)
self.assertFalse((x_test == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((y_test == y_pred).all())
acc = np.sum(np.argmax(y_pred, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0]
print('\nAccuracy on adversarial examples: %.2f%%' % (acc * 100))
def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
attack = VirtualAdversarialMethod(classifier, eps=.1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with VAT adversarial examples: %.2f%%',
(acc * 100))
def test_iris_k_unbounded(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model,
use_logits=False,
channel_index=1)
attack = VirtualAdversarialMethod(classifier, eps=1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv > 1).any())
self.assertTrue((x_test_adv < 0).any())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with VAT adversarial examples: %.2f%%',
(acc * 100))
def test_mnist(self):
session = tf.Session()
k.set_session(session)
comp_params = {
"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']
}
# get MNIST
batch_size, nb_train, nb_test = 100, 1000, 100
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train = X_train[:nb_train], Y_train[:nb_train]
X_test, Y_test = X_test[:nb_test], Y_test[:nb_test]
im_shape = X_train[0].shape
# get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train,
Y_train,
epochs=1,
batch_size=batch_size,
verbose=0)
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
df = VirtualAdversarialMethod(classifier,
sess=session,
clip_min=0.,
clip_max=1.)
x_test_adv = df.generate(X_test, eps=1)
self.assertFalse((X_test == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((Y_test == y_pred).all())
scores = classifier.evaluate(x_test_adv, Y_test)
print('\naccuracy on adversarial examples: %.2f%%' % (scores[1] * 100))
def _test_backend_mnist(self, classifier):
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
#import time
df = VirtualAdversarialMethod(classifier)
# starttime = time.clock()
# x_test_adv = df.generate(x_test, batch_size=1)
# endtime = time.clock()
# print(1, endtime - starttime)
#
# starttime = time.clock()
# x_test_adv = df.generate(x_test, batch_size=10)
# endtime = time.clock()
# print(10, endtime - starttime)
#
# starttime = time.clock()
x_test_adv = df.generate(x_test, batch_size=100)
# endtime = time.clock()
# print(100, endtime - starttime)
# starttime = time.clock()
# x_test_adv = df.generate(x_test, batch_size=1000)
# endtime = time.clock()
# print(1000, endtime - starttime)
self.assertFalse((x_test == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((y_test == y_pred).all())
acc = np.sum(np.argmax(y_pred, axis=1) == np.argmax(
y_test, axis=1)) / y_test.shape[0]
logging.info('Accuracy on adversarial examples: %.2f%%', (acc * 100))
v_print("Adversarials crafted with", args.adv_method, "on", MODEL_PATH,
"will be saved in", SAVE_ADV)
if args.adv_method in ['fgsm', "vat", "rnd_fgsm"]:
eps_ranges = {
'fgsm': [e / 10 for e in range(1, 11)],
'rnd_fgsm': [e / 10 for e in range(1, 11)],
'vat': [1.5, 2.1, 5, 7, 10]
}
if args.adv_method in ["fgsm", "rnd_fgsm"]:
adv_crafter = FastGradientMethod(classifier, sess=session)
else:
adv_crafter = VirtualAdversarialMethod(classifier, sess=session)
for eps in eps_ranges[args.adv_method]:
if args.adv_method == "rnd_fgsm":
x_train = np.clip(
X_train + alpha * np.sign(np.random.randn(*X_train.shape)),
min_, max_)
x_test = np.clip(
X_test + alpha * np.sign(np.random.randn(*X_test.shape)), min_,
max_)
e = eps - alpha
else:
x_train = X_train
x_test = X_test
e = eps
