def _test_mnist_targeted(self, classifier):
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
# Generate random target classes
import numpy as np
nb_classes = np.unique(np.argmax(y_test, axis=1)).shape[0]
targets = np.random.randint(nb_classes, size=NB_TEST)
while (targets == np.argmax(y_test, axis=1)).any():
targets = np.random.randint(nb_classes, size=NB_TEST)
# Perform attack
df = SaliencyMapMethod(classifier, theta=1)
x_test_adv = df.generate(x_test, y=to_categorical(targets, nb_classes))
self.assertFalse((x_test == x_test_adv).all())
self.assertFalse((0. == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((y_test == y_pred).all())
acc = np.sum(np.argmax(y_pred, axis=1) == np.argmax(
y_test, axis=1)) / y_test.shape[0]
print('\nAccuracy on adversarial examples: %.2f%%' % (acc * 100))
def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
attack = SaliencyMapMethod(classifier, theta=1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with JSMA adversarial examples: %.2f%%', (acc * 100))
def test_iris_k_unbounded(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model, use_logits=False, channel_index=1)
attack = SaliencyMapMethod(classifier, theta=1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with JSMA adversarial examples: %.2f%%', (acc * 100))
def test_mnist_targeted(self):
session = tf.Session()
k.set_session(session)
comp_params = {
"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']
}
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
im_shape = x_train[0].shape
# Get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(x_train,
y_train,
epochs=1,
batch_size=batch_size,
verbose=0)
scores = classifier.evaluate(x_test, y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
# Generate random target classes
import numpy as np
nb_classes = np.unique(np.argmax(y_test, axis=1)).shape[0]
targets = np.random.randint(nb_classes, size=nb_test)
while (targets == np.argmax(y_test, axis=1)).any():
targets = np.random.randint(nb_classes, size=nb_test)
# Perform attack
df = SaliencyMapMethod(classifier,
sess=session,
clip_min=0,
clip_max=1,
theta=1)
x_test_adv = df.generate(x_test, y_val=targets)
self.assertFalse((x_test == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((y_test == y_pred).all())
scores = classifier.evaluate(x_test_adv, y_test)
print('\naccuracy on adversarial examples: %.2f%%' % (scores[1] * 100))
def _test_mnist_untargeted(self, classifier):
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
df = SaliencyMapMethod(classifier, theta=1)
x_test_adv = df.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertFalse((0. == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((y_test == y_pred).all())
acc = np.sum(np.argmax(y_pred, axis=1) == np.argmax(y_test, axis=1)) / y_test.shape[0]
print('\nAccuracy on adversarial examples: %.2f%%' % (acc * 100))
def _test_mnist_targeted(self, classifier):
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
# Generate random target classes
nb_classes = np.unique(np.argmax(y_test, axis=1)).shape[0]
targets = np.random.randint(nb_classes, size=NB_TEST)
while (targets == np.argmax(y_test, axis=1)).any():
targets = np.random.randint(nb_classes, size=NB_TEST)
# Perform attack
# import time
df = SaliencyMapMethod(classifier, theta=1)
# starttime = time.clock()
# x_test_adv = df.generate(x_test, y=to_categorical(targets, nb_classes), batch_size=1)
# endtime = time.clock()
# print(1, endtime - starttime)
#
# starttime = time.clock()
# x_test_adv = df.generate(x_test, y=to_categorical(targets, nb_classes), batch_size=10)
# endtime = time.clock()
# print(10, endtime - starttime)
#
# starttime = time.clock()
x_test_adv = df.generate(x_test,
y=to_categorical(targets, nb_classes),
batch_size=100)
# endtime = time.clock()
# print(100, endtime - starttime)
# starttime = time.clock()
# x_test_adv = df.generate(x_test, y=to_categorical(targets, nb_classes), batch_size=1000)
# endtime = time.clock()
# print(1000, endtime - starttime)
self.assertFalse((x_test == x_test_adv).all())
self.assertFalse((0. == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((y_test == y_pred).all())
acc = np.sum(np.argmax(y_pred, axis=1) == np.argmax(
y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on adversarial examples: %.2f%%', (acc * 100))
def test_mnist_untargeted(self):
session = tf.Session()
k.set_session(session)
comp_params = {
"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']
}
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train = X_train[:nb_train], Y_train[:nb_train]
X_test, Y_test = X_test[:nb_test], Y_test[:nb_test]
im_shape = X_train[0].shape
# Get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train,
Y_train,
epochs=1,
batch_size=batch_size,
verbose=0)
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
# Perform attack
df = SaliencyMapMethod(classifier, sess=session)
df.set_params(clip_min=0, clip_max=1, theta=1)
x_test_adv = df.generate(X_test)
self.assertFalse((X_test == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((Y_test == y_pred).all())
scores = classifier.evaluate(x_test_adv, Y_test)
print('\naccuracy on adversarial examples: %.2f%%' % (scores[1] * 100))
def _test_mnist_untargeted(self, classifier):
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
# import time
df = SaliencyMapMethod(classifier, theta=1)
# starttime = time.clock()
# x_test_adv = df.generate(x_test, batch_size=1)
# endtime = time.clock()
# print(1, endtime - starttime)
#
# starttime = time.clock()
# x_test_adv = df.generate(x_test, batch_size=10)
# endtime = time.clock()
# print(10, endtime - starttime)
#
# starttime = time.clock()
x_test_adv = df.generate(x_test, batch_size=100)
# endtime = time.clock()
# print(100, endtime - starttime)
# starttime = time.clock()
# x_test_adv = df.generate(x_test, batch_size=1000)
# endtime = time.clock()
# print(1000, endtime - starttime)
self.assertFalse((x_test == x_test_adv).all())
self.assertFalse((0. == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((y_test == y_pred).all())
acc = np.sum(np.argmax(y_pred, axis=1) == np.argmax(
y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on adversarial examples: %.2f%%', (acc * 100))