def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build TFClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(x_train, _), (x_test, _) = self.mnist
# Attack
attack_st = SpatialTransformation(tfc,
max_translation=10.0,
num_translations=3,
max_rotation=30.0,
num_rotations=3)
x_train_adv = attack_st.generate(x_train)
self.assertLessEqual(abs(x_train_adv[0, 8, 13, 0] - 0.49004024), 0.01)
self.assertLessEqual(abs(attack_st.fooling_rate - 0.707), 0.01)
self.assertEqual(attack_st.attack_trans_x, 3)
self.assertEqual(attack_st.attack_trans_y, 3)
self.assertEqual(attack_st.attack_rot, 30.0)
x_test_adv = attack_st.generate(x_test)
self.assertLessEqual(abs(x_test_adv[0, 14, 14, 0] - 0.013572651), 0.01)
sess.close()
tf.reset_default_graph()
def test_loss_gradient(self):
classifier, sess = get_classifier_tf()
gradients = classifier.loss_gradient(self.x_test, self.y_test)
self.assertEqual(gradients.shape, (NB_TEST, 28, 28, 1))
expected_gradients_1 = np.asarray([
0.00279603, 0.00266946, 0.0032446, 0.00396258, -0.00201465,
-0.00564073, 0.0009253, 0.00016253, 0.0040816, 0.00166697,
0.0015883, -0.00121023, -0.00390778, -0.00234937, 0.0053558,
0.00204322, -0.00172054, 0.00053564, -0.0021146, -0.00069308,
0.00141374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
])
np.testing.assert_array_almost_equal(gradients[0, 14, :, 0],
expected_gradients_1,
decimal=4)
expected_gradients_2 = np.asarray([
1.05401428e-04, 1.06959546e-04, 2.60490313e-04, 2.74000311e-04,
-1.15295035e-04, 2.16038228e-04, 1.37472380e-04, 0.00000000e+00,
0.00000000e+00, -2.91720475e-03, -3.08302144e-04, 2.63109524e-03,
-1.18699251e-03, 2.63655302e-03, 5.35579538e-03, 6.38693338e-03,
3.44644510e-03, 6.68899389e-04, 5.01601025e-03, 8.40547902e-04,
-1.43233046e-05, -2.79442966e-03, 7.37082795e-04, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00
])
np.testing.assert_array_almost_equal(gradients[0, :, 14, 0],
expected_gradients_2,
decimal=4)
tf.reset_default_graph()
sess.close()
def test_save(self):
classifier, sess = get_classifier_tf()
path = 'tmp'
filename = 'model.ckpt'
# Save
classifier.save(filename, path=path)
self.assertTrue(
os.path.isfile(
os.path.join(path, filename,
'variables/variables.data-00000-of-00001')))
self.assertTrue(
os.path.isfile(
os.path.join(path, filename, 'variables/variables.index')))
self.assertTrue(
os.path.isfile(os.path.join(path, filename, 'saved_model.pb')))
# # Restore
# with tf.Session(graph=tf.Graph()) as sess:
# tf.saved_model.loader.load(sess, ["serve"], os.path.join(path, filename))
# graph = tf.get_default_graph()
# print(graph.get_operations())
# sess.run('SavedOutput:0', feed_dict={'SavedInputPhD:0': input_batch})
# Remove saved files
shutil.rmtree(os.path.join(path, filename))
tf.reset_default_graph()
sess.close()
def test_pickle(self):
classifier, sess = get_classifier_tf()
full_path = os.path.join(DATA_PATH, 'my_classifier')
folder = os.path.split(full_path)[0]
if not os.path.exists(folder):
os.makedirs(folder)
pickle.dump(classifier, open(full_path, 'wb'))
# Unpickle:
with open(full_path, 'rb') as f:
loaded = pickle.load(f)
self.assertEqual(classifier._clip_values, loaded._clip_values)
self.assertEqual(classifier._channel_index, loaded._channel_index)
self.assertEqual(set(classifier.__dict__.keys()),
set(loaded.__dict__.keys()))
# Test predict
predictions_1 = classifier.predict(self.x_test)
accuracy_1 = np.sum(
np.argmax(predictions_1, axis=1) == np.argmax(
self.y_test, axis=1)) / len(self.y_test)
predictions_2 = loaded.predict(self.x_test)
accuracy_2 = np.sum(
np.argmax(predictions_2, axis=1) == np.argmax(
self.y_test, axis=1)) / len(self.y_test)
self.assertEqual(accuracy_1, accuracy_2)
tf.reset_default_graph()
sess.close()
def test_tfclassifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# Attack
attack_st = SpatialTransformation(tfc,
max_translation=10.0,
num_translations=3,
max_rotation=30.0,
num_rotations=3)
x_train_adv = attack_st.generate(self.x_train)
self.assertAlmostEqual(x_train_adv[0, 8, 13, 0],
0.49004024,
delta=0.01)
self.assertAlmostEqual(attack_st.fooling_rate, 0.72, delta=0.01)
self.assertEqual(attack_st.attack_trans_x, 3)
self.assertEqual(attack_st.attack_trans_y, 3)
self.assertEqual(attack_st.attack_rot, 30.0)
x_test_adv = attack_st.generate(self.x_test)
self.assertAlmostEqual(x_test_adv[0, 14, 14, 0],
0.013572651,
delta=0.01)
sess.close()
def test_fit_generator(self):
classifier, sess = get_classifier_tf()
# Create TensorFlow data generator
x_tensor = tf.convert_to_tensor(
self.x_train.reshape(10, 100, 28, 28, 1))
y_tensor = tf.convert_to_tensor(self.y_train.reshape(10, 100, 10))
dataset = tf.data.Dataset.from_tensor_slices((x_tensor, y_tensor))
iterator = dataset.make_initializable_iterator()
data_gen = TFDataGenerator(sess=sess,
iterator=iterator,
iterator_type='initializable',
iterator_arg={},
size=1000,
batch_size=100)
# Test fit and predict
classifier.fit_generator(data_gen, nb_epochs=2)
predictions = classifier.predict(self.x_test)
predictions_class = np.argmax(predictions, axis=1)
true_class = np.argmax(self.y_test, axis=1)
accuracy = np.sum(predictions_class == true_class) / len(true_class)
logger.info(
'Accuracy after fitting TF classifier with generator: %.2f%%',
(accuracy * 100))
self.assertEqual(accuracy, 0.65)
tf.reset_default_graph()
sess.close()
def test_failure_attack(self):
"""
Test the corner case when attack is failed.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# Failure attack
cl2m = CarliniL2Method(classifier=tfc,
targeted=True,
max_iter=0,
binary_search_steps=0,
learning_rate=0,
initial_const=1)
params = {'y': random_targets(self.y_test, tfc.nb_classes())}
x_test_adv = cl2m.generate(self.x_test, **params)
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
np.testing.assert_array_almost_equal(self.x_test,
x_test_adv,
decimal=3)
# Clean-up session
sess.close()
def test_tfclassifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Attack
up = UniversalPerturbation(tfc,
max_iter=1,
attacker="newtonfool",
attacker_params={"max_iter": 5})
x_train_adv = up.generate(x_train)
self.assertTrue((up.fooling_rate >= 0.2) or not up.converged)
x_test_adv = x_test + up.noise
self.assertFalse((x_test == x_test_adv).all())
train_y_pred = np.argmax(tfc.predict(x_train_adv), axis=1)
test_y_pred = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == test_y_pred).all())
self.assertFalse((np.argmax(y_train, axis=1) == train_y_pred).all())
def test_failure_attack(self):
"""
Test the corner case when attack fails.
:return:
"""
# Build TFClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# Failure attack
ead = ElasticNet(classifier=tfc,
targeted=True,
max_iter=0,
binary_search_steps=0,
learning_rate=0,
initial_const=1)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = ead.generate(x_test, **params)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
np.testing.assert_almost_equal(x_test, x_test_adv, 3)
# Kill TF
sess.close()
tf.reset_default_graph()
def setUpClass(cls):
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN]
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
cls.mnist = (x_train, y_train), (x_test, y_test)
cls.classifier, cls.sess = get_classifier_tf()
def test_repr(self):
classifier, sess = get_classifier_tf()
repr_ = repr(classifier)
self.assertIn('art.classifiers.tensorflow.TensorFlowClassifier', repr_)
self.assertIn('channel_index=3, clip_values=(0, 1)', repr_)
self.assertIn('defences=None, preprocessing=(0, 1)', repr_)
tf.reset_default_graph()
sess.close()
def test_set_learning(self):
classifier, sess = get_classifier_tf()
self.assertEqual(classifier._feed_dict, {})
classifier.set_learning_phase(False)
self.assertFalse(classifier._feed_dict[classifier._learning])
classifier.set_learning_phase(True)
self.assertTrue(classifier._feed_dict[classifier._learning])
self.assertTrue(classifier.learning_phase)
tf.reset_default_graph()
sess.close()
def setUpClass(cls):
(x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist')
cls.x_train = x_train[:NB_TRAIN]
cls.y_train = y_train[:NB_TRAIN]
cls.x_test = x_test[:NB_TEST]
cls.y_test = y_test[:NB_TEST]
# Keras classifier
cls.classifier_k = get_classifier_kr()
scores = cls.classifier_k._model.evaluate(x_train, y_train)
logger.info('[Keras, MNIST] Accuracy on training set: %.2f%%',
(scores[1] * 100))
scores = cls.classifier_k._model.evaluate(x_test, y_test)
logger.info('[Keras, MNIST] Accuracy on test set: %.2f%%',
(scores[1] * 100))
# Create basic CNN on MNIST using TensorFlow
cls.classifier_tf, sess = get_classifier_tf()
scores = get_labels_np_array(cls.classifier_tf.predict(x_train))
accuracy = np.sum(
np.argmax(scores, axis=1) == np.argmax(y_train,
axis=1)) / y_train.shape[0]
logger.info('[TF, MNIST] Accuracy on training set: %.2f%%',
(accuracy * 100))
scores = get_labels_np_array(cls.classifier_tf.predict(x_test))
accuracy = np.sum(
np.argmax(scores, axis=1) == np.argmax(y_test,
axis=1)) / y_test.shape[0]
logger.info('[TF, MNIST] Accuracy on test set: %.2f%%',
(accuracy * 100))
# Create basic PyTorch model
cls.classifier_py = get_classifier_pt()
x_train, x_test = np.swapaxes(x_train, 1, 3), np.swapaxes(x_test, 1, 3)
scores = get_labels_np_array(
cls.classifier_py.predict(x_train.astype(np.float32)))
accuracy = np.sum(
np.argmax(scores, axis=1) == np.argmax(y_train,
axis=1)) / y_train.shape[0]
logger.info('[PyTorch, MNIST] Accuracy on training set: %.2f%%',
(accuracy * 100))
scores = get_labels_np_array(
cls.classifier_py.predict(x_test.astype(np.float32)))
accuracy = np.sum(
np.argmax(scores, axis=1) == np.argmax(y_test,
axis=1)) / y_test.shape[0]
logger.info('[PyTorch, MNIST] Accuracy on test set: %.2f%%',
(accuracy * 100))
def test_predict(self):
classifier, sess = get_classifier_tf()
predictions = classifier.predict(self.x_test)
predictions_class = np.argmax(predictions, axis=1)
trues_class = np.argmax(self.y_test, axis=1)
accuracy = np.sum(predictions_class == trues_class) / len(trues_class)
logger.info('Accuracy after fitting: %.2f%%', (accuracy * 100))
self.assertEqual(accuracy, 0.4)
tf.reset_default_graph()
sess.close()
def test_tfclassifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
tfc, sess = get_classifier_tf()
attack_ap = AdversarialPatch(tfc, rotation_max=22.5, scale_min=0.1, scale_max=1.0, learning_rate=5.0,
batch_size=10, max_iter=500)
patch_adv, _ = attack_ap.generate(self.x_train)
self.assertAlmostEqual(patch_adv[8, 8, 0], -3.1106631027725005, delta=0.1)
self.assertAlmostEqual(patch_adv[14, 14, 0], 18.101, delta=0.1)
self.assertAlmostEqual(float(np.sum(patch_adv)), 624.867, delta=0.1)
sess.close()
def test_layers(self):
classifier, sess = get_classifier_tf()
layer_names = classifier.layer_names
for i, name in enumerate(layer_names):
print(self.x_test.shape)
activation_i = classifier.get_activations(self.x_test,
i,
batch_size=5)
activation_name = classifier.get_activations(self.x_test,
name,
batch_size=5)
np.testing.assert_array_equal(activation_name, activation_i)
tf.reset_default_graph()
sess.close()
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc, sess = get_classifier_tf()
# First attack
clinfm = CarliniLInfMethod(classifier=krc,
targeted=True,
max_iter=10,
eps=0.5)
params = {'y': random_targets(self.y_test, krc.nb_classes())}
x_test_adv = clinfm.generate(self.x_test, **params)
self.assertFalse((self.x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', target)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(np.sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target == y_pred_adv).any())
# Second attack
clinfm = CarliniLInfMethod(classifier=krc,
targeted=False,
max_iter=10,
eps=0.5)
x_test_adv = clinfm.generate(self.x_test)
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), -1e-6)
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', target)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(np.sum(target != y_pred_adv) / float(len(target))))
self.assertTrue((target != y_pred_adv).any())
# Clean-up
k.clear_session()
sess.close()
def test_failure_attack(self):
"""
Test the corner case when attack fails.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# Failure attack
zoo = ZooAttack(classifier=tfc,
max_iter=0,
binary_search_steps=0,
learning_rate=0)
x_test_adv = zoo.generate(self.x_test)
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
np.testing.assert_almost_equal(self.x_test, x_test_adv, 3)
# Clean-up session
sess.close()
def test_tfclassifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# First attack
cl2m = CarliniL2Method(classifier=tfc, targeted=True, max_iter=10)
params = {'y': random_targets(self.y_test, tfc.nb_classes())}
x_test_adv = cl2m.generate(self.x_test, **params)
self.assertFalse((self.x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW2 Target: %s', target)
logger.debug('CW2 Actual: %s', y_pred_adv)
logger.info('CW2 Success Rate: %.2f',
(np.sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target == y_pred_adv).any())
# Second attack, no batching
cl2m = CarliniL2Method(classifier=tfc,
targeted=False,
max_iter=10,
batch_size=1)
x_test_adv = cl2m.generate(self.x_test)
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW2 Target: %s', target)
logger.debug('CW2 Actual: %s', y_pred_adv)
logger.info('CW2 Success Rate: %.2f',
(np.sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target != y_pred_adv).any())
# Clean-up session
sess.close()
def test_failure_attack(self):
"""
Test the corner case when attack fails.
:return:
"""
# Build TFClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
x_test, _ = self.mnist
# Failure attack
zoo = ZooAttack(classifier=tfc, max_iter=0, binary_search_steps=0, learning_rate=0)
x_test_adv = zoo.generate(x_test)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
np.testing.assert_almost_equal(x_test, x_test_adv, 3)
# Clean-up session
sess.close()
tf.reset_default_graph()
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build TFClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# First attack
cl2m = CarliniL2Method(classifier=tfc, targeted=True, max_iter=10)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW2 Target: %s', target)
logger.debug('CW2 Actual: %s', y_pred_adv)
logger.info('CW2 Success Rate: %.2f', (np.sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target == y_pred_adv).any())
# Second attack, no batching
cl2m = CarliniL2Method(classifier=tfc, targeted=False, max_iter=10, batch_size=1)
x_test_adv = cl2m.generate(x_test)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW2 Target: %s', target)
logger.debug('CW2 Actual: %s', y_pred_adv)
logger.info('CW2 Success Rate: %.2f', (np.sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target != y_pred_adv).any())
# Clean-up session
sess.close()
tf.reset_default_graph()
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build TFClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(x_train, _), (_, _) = self.mnist
# Attack
attack_ap = AdversarialPatch(tfc, rotation_max=22.5, scale_min=0.1, scale_max=1.0, learning_rate=5.0,
batch_size=10, max_iter=500)
patch_adv, _ = attack_ap.generate(x_train)
self.assertLess(patch_adv[8, 8, 0] - (-3.1106631027725005), 0.01)
self.assertLess(patch_adv[14, 14, 0] - 18.954278294246386, 0.01)
self.assertLess(np.sum(patch_adv) - 794.2447019737851, 0.01)
sess.close()
tf.reset_default_graph()
def test_failure_attack(self):
"""
Test the corner case when attack is failed.
:return:
"""
# Build TFClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# Failure attack
clinfm = CarliniLInfMethod(classifier=tfc, targeted=True, max_iter=0, learning_rate=0, eps=0.5)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = clinfm.generate(x_test, **params)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
self.assertTrue(np.allclose(x_test, x_test_adv, atol=1e-3))
# Clean-up session
sess.close()
tf.reset_default_graph()
def test_tfclassifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# Targeted attack
zoo = ZooAttack(classifier=tfc,
targeted=True,
max_iter=100,
binary_search_steps=10)
params = {'y': random_targets(self.y_test, tfc.nb_classes())}
x_test_adv = zoo.generate(self.x_test, **params)
self.assertFalse((self.x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('ZOO target: %s', target)
logger.debug('ZOO actual: %s', y_pred_adv)
logger.info('ZOO success rate on MNIST: %.2f',
(sum(target == y_pred_adv) / float(len(target))))
# Untargeted attack
zoo = ZooAttack(classifier=tfc, targeted=False)
x_test_adv = zoo.generate(self.x_test)
# self.assertFalse((x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
y_pred = np.argmax(tfc.predict(self.x_test), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('ZOO actual: %s', y_pred_adv)
logger.info('ZOO success rate on MNIST: %.2f',
(sum(y_pred != y_pred_adv) / float(len(y_pred))))
# Clean-up session
sess.close()
def test_failure_attack(self):
"""
Test the corner case when attack is failed.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# Failure attack
clinfm = CarliniLInfMethod(classifier=tfc,
targeted=True,
max_iter=0,
learning_rate=0,
eps=0.5)
params = {'y': random_targets(self.y_test, tfc.nb_classes())}
x_test_adv = clinfm.generate(self.x_test, **params)
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
self.assertTrue(np.allclose(self.x_test, x_test_adv, atol=1e-3))
# Clean-up session
sess.close()
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build TFClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
x_test, y_test = self.mnist
# Targeted attack
zoo = ZooAttack(classifier=tfc, targeted=True)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = zoo.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('ZOO target: %s', target)
logger.debug('ZOO actual: %s', y_pred_adv)
logger.info('ZOO success rate on MNIST: %.2f', (sum(target == y_pred_adv) / float(len(target))))
# Untargeted attack
zoo = ZooAttack(classifier=tfc, targeted=False)
x_test_adv = zoo.generate(x_test)
# self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(x_test), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('ZOO actual: %s', y_pred_adv)
logger.info('ZOO success rate on MNIST: %.2f', (sum(y_pred != y_pred_adv) / float(len(y_pred))))
# Clean-up session
sess.close()
tf.reset_default_graph()
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build TFClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(_, _), (x_test, _) = self.mnist
# Attack
nf = NewtonFool(tfc, max_iter=5, batch_size=100)
x_test_adv = nf.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
y_pred = tfc.predict(x_test)
y_pred_adv = tfc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= y_pred_adv_max).all())
def tearDownClass(cls):
_, sess = get_classifier_tf()
tf.reset_default_graph()
sess.close()
def test_tfclassifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# First targeted attack
boundary = BoundaryAttack(classifier=tfc,
targeted=True,
max_iter=200,
delta=0.5)
params = {'y': random_targets(self.y_test, tfc.nb_classes())}
x_test_adv = boundary.generate(self.x_test, **params)
expected_x_test_adv_1 = np.asarray([
0.42622495, 0.0, 0.0, 0.33005068, 0.2277837, 0.0, 0.18348512,
0.42622495, 0.27452883, 0.0, 0.0, 0.0, 0.1653487, 0.70523715,
0.7367977, 0.7974912, 0.28579983, 0.0, 0.36499417, 0.0, 0.0, 0.0,
0.42622495, 0.0, 0.26680174, 0.42622495, 0.0, 0.19260764
])
expected_x_test_adv_2 = np.asarray([
0.0459, 0., 0., 0.0756, 0.2048, 0.037, 0., 0., 0.0126, 0.4338,
0.1566, 0.3061, 0., 0.296, 0.8318, 0.7267, 0.2252, 0.074, 0.,
0.1208, 0.4362, 0., 0., 0., 0., 0.0359, 0., 0.1191
])
try:
np.testing.assert_array_almost_equal(x_test_adv[2, 14, :, 0],
expected_x_test_adv_1,
decimal=4)
except AssertionError:
np.testing.assert_array_almost_equal(x_test_adv[2, 14, :, 0],
expected_x_test_adv_2,
decimal=4)
self.assertLessEqual(np.max(x_test_adv), 1.0)
self.assertGreaterEqual(np.min(x_test_adv), 0.0)
y_pred_adv = tfc.predict(x_test_adv)
y_pred_adv_expected = np.asarray([
1.57103419e-01, -7.31061280e-01, -4.03979905e-02, -4.79048371e-01,
9.37852338e-02, -8.01057637e-01, -4.77534801e-01, 1.08687377e+00,
-3.06577891e-01, -5.74976981e-01
])
np.testing.assert_array_almost_equal(y_pred_adv[0],
y_pred_adv_expected,
decimal=4)
# Second untargeted attack
boundary = BoundaryAttack(classifier=tfc, targeted=False, max_iter=20)
x_test_adv = boundary.generate(self.x_test)
self.assertFalse((self.x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(self.x_test), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Clean-up session
sess.close()
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build TFClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# First attack
ead = ElasticNet(classifier=tfc, targeted=True, max_iter=2)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = ead.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('EAD target: %s', target)
logger.debug('EAD actual: %s', y_pred_adv)
logger.info('EAD success rate on MNIST: %.2f%%',
(100 * sum(target == y_pred_adv) / len(target)))
self.assertTrue((target == y_pred_adv).any())
# Second attack
ead = ElasticNet(classifier=tfc, targeted=False, max_iter=2)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = ead.generate(x_test, **params)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('EAD target: %s', target)
logger.debug('EAD actual: %s', y_pred_adv)
logger.info('EAD success rate on MNIST: %.2f%%',
(100 * sum(target != y_pred_adv) / float(len(target))))
self.assertTrue((target != y_pred_adv).any())
# Third attack
ead = ElasticNet(classifier=tfc, targeted=False, max_iter=2)
params = {}
x_test_adv = ead.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(x_test), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('EAD target: %s', y_pred)
logger.debug('EAD actual: %s', y_pred_adv)
logger.info('EAD success rate: %.2f%%',
(100 * sum(y_pred != y_pred_adv) / float(len(y_pred))))
self.assertTrue((y_pred != y_pred_adv).any())
# First attack without batching
ead_wob = ElasticNet(classifier=tfc,
targeted=True,
max_iter=2,
batch_size=1)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = ead_wob.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('EAD target: %s', target)
logger.debug('EAD actual: %s', y_pred_adv)
logger.info('EAD success rate: %.2f%%',
(100 * sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target == y_pred_adv).any())
# Second attack without batching
ead_wob = ElasticNet(classifier=tfc,
targeted=False,
max_iter=2,
batch_size=1)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = ead_wob.generate(x_test, **params)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('EAD target: %s', target)
logger.debug('EAD actual: %s', y_pred_adv)
logger.info('EAD success rate: %.2f%%',
(100 * sum(target != y_pred_adv) / float(len(target))))
self.assertTrue((target != y_pred_adv).any())
# Kill TF
sess.close()
tf.reset_default_graph()
