def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build a TFClassifier
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(fc, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Train the classifier
tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss,
None, sess)
tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=2)
# Attack
# TODO Launch with all possible attacks
attack_params = {
"attacker": "newtonfool",
"attacker_params": {
"max_iter": 5
}
}
up = UniversalPerturbation(tfc)
x_train_adv = up.generate(x_train, **attack_params)
self.assertTrue((up.fooling_rate >= 0.2) or not up.converged)
x_test_adv = x_test + up.v
self.assertFalse((x_test == x_test_adv).all())
train_y_pred = np.argmax(tfc.predict(x_train_adv), axis=1)
test_y_pred = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == test_y_pred).all())
self.assertFalse((np.argmax(y_train, axis=1) == train_y_pred).all())
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build a TFClassifier
# Define input and output placeholders
self._input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
self._output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(self._input_ph, 4, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
self._logits = tf.layers.dense(fc, 10)
# Train operator
self._loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(logits=self._logits, onehot_labels=self._output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
self._train = optimizer.minimize(self._loss)
# Tensorflow session and initialization
self._sess = tf.Session()
self._sess.run(tf.global_variables_initializer())
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
# Train the classifier
tfc = TFClassifier((0, 1), self._input_ph, self._logits, self._output_ph, self._train, self._loss, None,
self._sess)
tfc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=2)
# Attack
nf = NewtonFool(tfc)
nf.set_params(max_iter=5)
x_test_adv = nf.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
y_pred = tfc.predict(x_test)
y_pred_adv = tfc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= y_pred_adv_max).all())
def test_fit_predict(self):
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN]
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
# Test fit and predict
tfc = TFClassifier(None, self._input_ph, self._logits, self._output_ph,
self._train, self._loss, None, self._sess)
tfc.fit(x_train, y_train, batch_size=100, nb_epochs=1)
preds = tfc.predict(x_test)
preds_class = np.argmax(preds, axis=1)
trues_class = np.argmax(y_test, axis=1)
acc = np.sum(preds_class == trues_class) / len(trues_class)
print("\nAccuracy: %.2f%%" % (acc * 100))
self.assertGreater(acc, 0.1)
class TestTFClassifier(unittest.TestCase):
"""
This class tests the functionalities of the Tensorflow-based classifier.
"""
@classmethod
def setUpClass(cls):
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN]
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
cls.mnist = (x_train, y_train), (x_test, y_test)
def setUp(self):
# Set master seed
master_seed(1234)
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
learning = tf.placeholder(tf.bool)
# Define the tensorflow graph
conv = tf.layers.conv2d(input_ph, 16, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(fc, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
# Create classifier and fit
self.classifier = TFClassifier((0, 1), input_ph, logits, output_ph,
train, loss, learning, self.sess)
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
self.classifier.fit(x_train, y_train, batch_size=100, nb_epochs=3)
def tearDown(self):
self.sess.close()
def test_fit_predict(self):
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Test fit and predict
preds = self.classifier.predict(x_test)
preds_class = np.argmax(preds, axis=1)
trues_class = np.argmax(y_test, axis=1)
acc = np.sum(preds_class == trues_class) / len(trues_class)
logger.info('Accuracy after fitting: %.2f%%', (acc * 100))
self.assertGreater(acc, 0.1)
tf.reset_default_graph()
def test_fit_generator(self):
from art.data_generators import TFDataGenerator
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Create Tensorflow data generator
x_tensor = tf.convert_to_tensor(x_train.reshape(10, 100, 28, 28, 1))
y_tensor = tf.convert_to_tensor(y_train.reshape(10, 100, 10))
dataset = tf.data.Dataset.from_tensor_slices((x_tensor, y_tensor))
iterator = dataset.make_initializable_iterator()
data_gen = TFDataGenerator(sess=self.sess,
iterator=iterator,
iterator_type='initializable',
iterator_arg={},
size=1000,
batch_size=100)
# Test fit and predict
self.classifier.fit_generator(data_gen, nb_epochs=1)
preds = self.classifier.predict(x_test)
preds_class = np.argmax(preds, axis=1)
trues_class = np.argmax(y_test, axis=1)
acc = np.sum(preds_class == trues_class) / len(trues_class)
logger.info('Accuracy after fitting: %.2f%%', (acc * 100))
self.assertGreater(acc, 0.1)
tf.reset_default_graph()
def test_fit_generator(self):
from art.classifiers.keras import generator_fit
from art.data_generators import KerasDataGenerator
labels = np.argmax(self.mnist[1][1], axis=1)
acc = np.sum(
np.argmax(self.classifier.predict(self.mnist[1][0]), axis=1) ==
labels) / NB_TEST
logger.info('Accuracy: %.2f%%', (acc * 100))
gen = generator_fit(self.mnist[0][0], self.mnist[0][1], batch_size=100)
data_gen = KerasDataGenerator(generator=gen,
size=NB_TRAIN,
batch_size=100)
self.classifier.fit_generator(generator=data_gen, nb_epochs=2)
acc2 = np.sum(
np.argmax(self.classifier.predict(self.mnist[1][0]), axis=1) ==
labels) / NB_TEST
logger.info('Accuracy: %.2f%%', (acc2 * 100))
self.assertTrue(acc2 >= .8 * acc)
def test_nb_classes(self):
# Start to test
self.assertTrue(self.classifier.nb_classes == 10)
tf.reset_default_graph()
def test_input_shape(self):
# Start to test
self.assertTrue(
np.array(self.classifier.input_shape == (28, 28, 1)).all())
tf.reset_default_graph()
def test_class_gradient(self):
# Get MNIST
(_, _), (x_test, _) = self.mnist
# Test all gradients label = None
grads = self.classifier.class_gradient(x_test)
self.assertTrue(
np.array(grads.shape == (NB_TEST, 10, 28, 28, 1)).all())
self.assertTrue(np.sum(grads) != 0)
# Test 1 gradient label = 5
grads = self.classifier.class_gradient(x_test, label=5)
self.assertTrue(np.array(grads.shape == (NB_TEST, 1, 28, 28, 1)).all())
self.assertTrue(np.sum(grads) != 0)
# Test a set of gradients label = array
label = np.random.randint(5, size=NB_TEST)
grads = self.classifier.class_gradient(x_test, label=label)
self.assertTrue(np.array(grads.shape == (NB_TEST, 1, 28, 28, 1)).all())
self.assertTrue(np.sum(grads) != 0)
def test_loss_gradient(self):
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# Test gradient
grads = self.classifier.loss_gradient(x_test, y_test)
self.assertTrue(np.array(grads.shape == (NB_TEST, 28, 28, 1)).all())
self.assertTrue(np.sum(grads) != 0)
tf.reset_default_graph()
def test_layers(self):
# Get MNIST
(_, _), (x_test, _) = self.mnist
# Test and get layers
layer_names = self.classifier.layer_names
logger.debug(layer_names)
self.assertTrue(layer_names == [
'conv2d/Relu:0', 'max_pooling2d/MaxPool:0',
'Flatten/flatten/Reshape:0', 'dense/BiasAdd:0'
])
for i, name in enumerate(layer_names):
act_i = self.classifier.get_activations(x_test, i)
act_name = self.classifier.get_activations(x_test, name)
self.assertAlmostEqual(np.sum(act_name - act_i), 0)
self.assertTrue(
self.classifier.get_activations(x_test, 0).shape == (20, 24, 24,
16))
self.assertTrue(
self.classifier.get_activations(x_test, 1).shape == (20, 12, 12,
16))
self.assertTrue(
self.classifier.get_activations(x_test, 2).shape == (20, 2304))
self.assertTrue(
self.classifier.get_activations(x_test, 3).shape == (20, 10))
tf.reset_default_graph()
def test_save(self):
import os
import re
path = 'tmp'
filename = 'model.ckpt'
self.classifier.save(filename, path=path)
self.assertTrue(os.path.isfile(os.path.join(path, filename + '.meta')))
self.assertTrue(os.path.isfile(os.path.join(path,
filename + '.index')))
# Remove saved files
for f in os.listdir(path):
if re.search(filename, f):
os.remove(os.path.join(path, f))
def test_set_learning(self):
tfc = self.classifier
self.assertTrue(tfc._feed_dict == {})
tfc.set_learning_phase(False)
self.assertFalse(tfc._feed_dict[tfc._learning])
tfc.set_learning_phase(True)
self.assertTrue(tfc._feed_dict[tfc._learning])
self.assertTrue(tfc.learning_phase)
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build a TFClassifier
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(fc, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Train the classifier
tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss,
None, sess)
tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=10)
# First attack
clinfm = CarliniLInfMethod(classifier=tfc,
targeted=True,
max_iter=10,
eps=0.5)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = clinfm.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', target)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target == y_pred_adv).any())
# Second attack
clinfm = CarliniLInfMethod(classifier=tfc,
targeted=False,
max_iter=10,
eps=0.5)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = clinfm.generate(x_test, **params)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', target)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(sum(target != y_pred_adv) / float(len(target))))
self.assertTrue((target != y_pred_adv).any())
# Third attack
clinfm = CarliniLInfMethod(classifier=tfc,
targeted=False,
max_iter=10,
eps=0.5)
params = {}
x_test_adv = clinfm.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(x_test), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', y_pred)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(sum(y_pred != y_pred_adv) / float(len(y_pred))))
self.assertTrue((y_pred != y_pred_adv).any())
# First attack without batching
clinfmwob = CarliniLInfMethod(classifier=tfc,
targeted=True,
max_iter=10,
eps=0.5,
batch_size=1)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = clinfmwob.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', target)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target == y_pred_adv).any())
# Second attack without batching
clinfmwob = CarliniLInfMethod(classifier=tfc,
targeted=False,
max_iter=10,
eps=0.5,
batch_size=1)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = clinfmwob.generate(x_test, **params)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', target)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(sum(target != y_pred_adv) / float(len(target))))
self.assertTrue((target != y_pred_adv).any())
# Third attack without batching
clinfmwob = CarliniLInfMethod(classifier=tfc,
targeted=False,
max_iter=10,
eps=0.5,
batch_size=1)
params = {}
x_test_adv = clinfmwob.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(x_test), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', y_pred)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(sum(y_pred != y_pred_adv) / float(len(y_pred))))
self.assertTrue((y_pred != y_pred_adv).any())
logits = tf.layers.dense(x, 10)
loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=labels_ph))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
#print('success')
# Step 3: Create the ART classifier
classifier = TFClassifier(clip_values=(min_pixel_value, max_pixel_value), input_ph=input_ph, output=logits,
labels_ph=labels_ph, train=train, loss=loss, learning=None, sess=sess)
# Step 4: Train the ART classifier
classifier.fit(x_train, y_train, batch_size=64, nb_epochs=3)
print('successful')
# Step 5: Evaluate the ART classifier on benign test examples
predictions = classifier.predict(x_test)
accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)
print('Accuracy on benign test examples: {}%'.format(accuracy * 100))
# Step 6: Generate adversarial test examples
attack = FastGradientMethod(classifier=classifier, eps=0.2)
x_test_adv = attack.generate(x=x_test)
# Step 7: Evaluate the ART classifier on adversarial test examples
predictions = classifier.predict(x_test_adv)
accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)
print('Accuracy on adversarial test examples: {}%'.format(accuracy * 100))
class TestTFClassifier(unittest.TestCase):
"""
This class tests the functionalities of the Tensorflow-based classifier.
"""
@classmethod
def setUpClass(cls):
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN]
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
cls.mnist = (x_train, y_train), (x_test, y_test)
def setUp(self):
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(input_ph, 16, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(fc, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
# Create classifier
self.classifier = TFClassifier((0, 1), input_ph, logits, output_ph,
train, loss, None, self.sess)
def tearDown(self):
self.sess.close()
def test_fit_predict(self):
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Test fit and predict
self.classifier.fit(x_train, y_train, batch_size=100, nb_epochs=1)
preds = self.classifier.predict(x_test)
preds_class = np.argmax(preds, axis=1)
trues_class = np.argmax(y_test, axis=1)
acc = np.sum(preds_class == trues_class) / len(trues_class)
logger.info('Accuracy after fitting: %.2f%%', (acc * 100))
self.assertGreater(acc, 0.1)
tf.reset_default_graph()
def test_nb_classes(self):
# Start to test
self.assertTrue(self.classifier.nb_classes == 10)
tf.reset_default_graph()
def test_input_shape(self):
# Start to test
self.assertTrue(
np.array(self.classifier.input_shape == (28, 28, 1)).all())
tf.reset_default_graph()
def test_class_gradient(self):
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# Test gradient
grads = self.classifier.class_gradient(x_test)
self.assertTrue(
np.array(grads.shape == (NB_TEST, 10, 28, 28, 1)).all())
self.assertTrue(np.sum(grads) != 0)
tf.reset_default_graph()
def test_loss_gradient(self):
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# Test gradient
grads = self.classifier.loss_gradient(x_test, y_test)
self.assertTrue(np.array(grads.shape == (NB_TEST, 28, 28, 1)).all())
self.assertTrue(np.sum(grads) != 0)
tf.reset_default_graph()
def test_layers(self):
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# Test and get layers
layer_names = self.classifier.layer_names
logger.debug(layer_names)
self.assertTrue(layer_names == [
'conv2d/Relu:0', 'max_pooling2d/MaxPool:0',
'Flatten/flatten/Reshape:0', 'dense/BiasAdd:0'
])
for i, name in enumerate(layer_names):
act_i = self.classifier.get_activations(x_test, i)
act_name = self.classifier.get_activations(x_test, name)
self.assertAlmostEqual(np.sum(act_name - act_i), 0)
self.assertTrue(
self.classifier.get_activations(x_test, 0).shape == (20, 24, 24,
16))
self.assertTrue(
self.classifier.get_activations(x_test, 1).shape == (20, 12, 12,
16))
self.assertTrue(
self.classifier.get_activations(x_test, 2).shape == (20, 2304))
self.assertTrue(
self.classifier.get_activations(x_test, 3).shape == (20, 10))
tf.reset_default_graph()
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build a TFClassifier
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(fc, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Get MNIST
(x_train, y_train), (x_test, _) = self.mnist
# Train the classifier
tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss,
None, sess)
tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=2)
# Attack
# import time
nf = NewtonFool(tfc, max_iter=5)
# print("Test Tensorflow....")
# starttime = time.clock()
# x_test_adv = nf.generate(x_test, batch_size=1)
# self.assertFalse((x_test == x_test_adv).all())
# endtime = time.clock()
# print(1, endtime - starttime)
# starttime = time.clock()
# x_test_adv = nf.generate(x_test, batch_size=10)
# endtime = time.clock()
# print(10, endtime - starttime)
# starttime = time.clock()
x_test_adv = nf.generate(x_test, batch_size=100)
# endtime = time.clock()
# print(100, endtime - starttime)
#
# starttime = time.clock()
# x_test_adv = nf.generate(x_test, batch_size=1000)
# endtime = time.clock()
# print(1000, endtime - starttime)
self.assertFalse((x_test == x_test_adv).all())
y_pred = tfc.predict(x_test)
y_pred_adv = tfc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= y_pred_adv_max).all())
