def test_tensorflow_classifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
x_test_original = self.x_test_mnist.copy()
# Build TensorFlowClassifier
tfc, sess = get_image_classifier_tf()
# Attack
attack_st = SpatialTransformation(
tfc, max_translation=10.0, num_translations=3, max_rotation=30.0, num_rotations=3
)
x_train_adv = attack_st.generate(self.x_train_mnist)
self.assertAlmostEqual(x_train_adv[0, 8, 13, 0], 0.49004024, delta=0.01)
self.assertAlmostEqual(attack_st.fooling_rate, 0.71, delta=0.02)
self.assertEqual(attack_st.attack_trans_x, 3)
self.assertEqual(attack_st.attack_trans_y, 3)
self.assertEqual(attack_st.attack_rot, 30.0)
x_test_adv = attack_st.generate(self.x_test_mnist)
self.assertAlmostEqual(x_test_adv[0, 14, 14, 0], 0.013572651, delta=0.01)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test_mnist))), 0.0, delta=0.00001)
if sess is not None:
sess.close()
def test_keras_classifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
x_test_original = self.x_test_mnist.copy()
# Build KerasClassifier
krc = get_image_classifier_kr()
# Attack
attack_st = SpatialTransformation(
krc, max_translation=10.0, num_translations=3, max_rotation=30.0, num_rotations=3
)
x_train_adv = attack_st.generate(self.x_train_mnist)
self.assertAlmostEqual(x_train_adv[0, 8, 13, 0], 0.49004024, delta=0.01)
self.assertAlmostEqual(attack_st.fooling_rate, 0.71, delta=0.02)
self.assertEqual(attack_st.attack_trans_x, 3)
self.assertEqual(attack_st.attack_trans_y, 3)
self.assertEqual(attack_st.attack_rot, 30.0)
x_test_adv = attack_st.generate(self.x_test_mnist)
self.assertAlmostEqual(x_test_adv[0, 14, 14, 0], 0.013572651, delta=0.01)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test_mnist))), 0.0, delta=0.00001)
k.clear_session()
def test_pytorch_classifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
x_train_mnist = np.reshape(self.x_train_mnist, (self.x_train_mnist.shape[0], 1, 28, 28)).astype(np.float32)
x_test_mnist = np.reshape(self.x_test_mnist, (self.x_test_mnist.shape[0], 1, 28, 28)).astype(np.float32)
x_test_original = x_test_mnist.copy()
# Build PyTorchClassifier
ptc = get_image_classifier_pt(from_logits=True)
# Attack
attack_st = SpatialTransformation(
ptc, max_translation=10.0, num_translations=3, max_rotation=30.0, num_rotations=3
)
x_train__mnistadv = attack_st.generate(x_train_mnist)
self.assertAlmostEqual(x_train__mnistadv[0, 0, 13, 18], 0.627451, delta=0.01)
self.assertAlmostEqual(attack_st.fooling_rate, 0.57, delta=0.03)
self.assertEqual(attack_st.attack_trans_x, 0)
self.assertEqual(attack_st.attack_trans_y, 3)
self.assertEqual(attack_st.attack_rot, 0.0)
x_test_adv = attack_st.generate(x_test_mnist)
self.assertLessEqual(abs(x_test_adv[0, 0, 14, 14] - 0.008591662), 0.01)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test_mnist))), 0.0, delta=0.00001)
def _spatial(model, data, labels, attack_args):
max_translation = attack_args.get('max_translation', 0.2)
num_translations = attack_args.get('num_translations', 1)
max_rotation = attack_args.get('max_rotation', 10)
num_rotations = attack_args.get('num_rotations', 1)
print('>>> Generating SpatialTransformation examples.')
attacker = SpatialTransformation(classifier=model,
max_translation=max_translation,
num_translations=num_translations,
max_rotation=max_rotation,
num_rotations=num_rotations)
return attacker.generate(data, labels)
def test_check_params(self):
ptc = get_image_classifier_pt(from_logits=True)
with self.assertRaises(ValueError):
_ = SpatialTransformation(ptc, max_translation=-1)
with self.assertRaises(ValueError):
_ = SpatialTransformation(ptc, num_translations=-1)
with self.assertRaises(ValueError):
_ = SpatialTransformation(ptc, max_rotation=-1)
with self.assertRaises(ValueError):
_ = SpatialTransformation(ptc, verbose="False")
def _spatial(model, data, labels, attack_args):
max_translation = attack_args.get('max_translation', 0.2)
num_translations = attack_args.get('num_translations', 10)
max_rotation = attack_args.get('max_rotation', 15)
num_rotations = attack_args.get('num_rotations', 10)
if num_rotations <= 0:
num_rotations = 1
if num_translations <= 0:
num_translations = 1
attacker = SpatialTransformation(classifier=model,
max_translation=max_translation,
num_translations=num_translations,
max_rotation=max_rotation,
num_rotations=num_rotations)
return attacker.generate(data, labels)
def test_failure_feature_vectors(self):
attack_params = {"max_translation": 10.0, "num_translations": 3, "max_rotation": 30.0, "num_rotations": 3}
classifier = get_tabular_classifier_kr()
attack = SpatialTransformation(classifier=classifier)
attack.set_params(**attack_params)
data = np.random.rand(10, 4)
# Assert that value error is raised for feature vectors
with self.assertRaises(ValueError) as context:
attack.generate(data)
self.assertIn("Feature vectors detected.", str(context.exception))
