def test_defences_predict(self):
clip_values = (0, 1)
fs = FeatureSqueezing(clip_values=clip_values, bit_depth=2)
jpeg = JpegCompression(clip_values=clip_values, apply_predict=True)
smooth = SpatialSmoothing()
classifier_ = get_classifier_kr()
classifier = KerasClassifier(clip_values=clip_values,
model=classifier_._model,
defences=[fs, jpeg, smooth])
self.assertEqual(len(classifier.defences), 3)
predictions_classifier = classifier.predict(self.x_test)
# Apply the same defences by hand
x_test_defense = self.x_test
x_test_defense, _ = fs(x_test_defense, self.y_test)
x_test_defense, _ = jpeg(x_test_defense, self.y_test)
x_test_defense, _ = smooth(x_test_defense, self.y_test)
classifier = get_classifier_kr()
predictions_check = classifier._model.predict(x_test_defense)
# Check that the prediction results match
np.testing.assert_array_almost_equal(predictions_classifier,
predictions_check,
decimal=4)
def setUpClass(cls):
k.clear_session()
(x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist')
cls.x_train = x_train[:NB_TRAIN]
cls.y_train = y_train[:NB_TRAIN]
cls.x_test = x_test[:NB_TEST]
cls.y_test = y_test[:NB_TEST]
# Use twice the same classifier for unittesting, in application they would be different
classifier_1 = get_classifier_kr()
classifier_2 = get_classifier_kr()
cls.ensemble = EnsembleClassifier(
classifiers=[classifier_1, classifier_2], clip_values=(0, 1))
def setUpClass(cls):
k.clear_session()
k.set_learning_phase(1)
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train, x_test, y_test = x_train[:
NB_TRAIN], y_train[:
NB_TRAIN], x_test[:
NB_TEST], y_test[:
NB_TEST]
cls.mnist = (x_train, y_train), (x_test, y_test)
# Load small Keras model
cls.model_mnist, _ = get_classifier_kr()
cls.functional_model = cls.functional_model()
import requests
import tempfile
import os
# Temporary folder for tests
cls.test_dir = tempfile.mkdtemp()
# Download one ImageNet pic for tests
url = 'http://farm1.static.flickr.com/163/381342603_81db58bea4.jpg'
result = requests.get(url, stream=True)
if result.status_code == 200:
image = result.raw.read()
f = open(os.path.join(cls.test_dir, 'test.jpg'), 'wb')
f.write(image)
f.close()
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc, sess = get_classifier_kr()
# Get MNIST
(x_train, _), (x_test, _) = self.mnist
# Attack
attack_st = SpatialTransformation(krc,
max_translation=10.0,
num_translations=3,
max_rotation=30.0,
num_rotations=3)
x_train_adv = attack_st.generate(x_train)
self.assertLessEqual(abs(x_train_adv[0, 8, 13, 0] - 0.49004024), 0.01)
self.assertLessEqual(abs(attack_st.fooling_rate - 0.707), 0.01)
self.assertEqual(attack_st.attack_trans_x, 3)
self.assertEqual(attack_st.attack_trans_y, 3)
self.assertEqual(attack_st.attack_rot, 30.0)
x_test_adv = attack_st.generate(x_test)
self.assertLessEqual(abs(x_test_adv[0, 14, 14, 0] - 0.013572651), 0.01)
k.clear_session()
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc = get_classifier_kr()
# Attack
attack_st = SpatialTransformation(krc,
max_translation=10.0,
num_translations=3,
max_rotation=30.0,
num_rotations=3)
x_train_adv = attack_st.generate(self.x_train)
self.assertAlmostEqual(x_train_adv[0, 8, 13, 0],
0.49004024,
delta=0.01)
self.assertAlmostEqual(attack_st.fooling_rate, 0.72, delta=0.01)
self.assertEqual(attack_st.attack_trans_x, 3)
self.assertEqual(attack_st.attack_trans_y, 3)
self.assertEqual(attack_st.attack_rot, 30.0)
x_test_adv = attack_st.generate(self.x_test)
self.assertAlmostEqual(x_test_adv[0, 14, 14, 0],
0.013572651,
delta=0.01)
k.clear_session()
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc = get_classifier_kr()
# First targeted attack
boundary = BoundaryAttack(classifier=krc, targeted=True, max_iter=20)
params = {'y': random_targets(self.y_test, krc.nb_classes())}
x_test_adv = boundary.generate(self.x_test, **params)
self.assertFalse((self.x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack
boundary = BoundaryAttack(classifier=krc, targeted=False, max_iter=20)
x_test_adv = boundary.generate(self.x_test)
self.assertFalse((self.x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(krc.predict(self.x_test), axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Clean-up session
k.clear_session()
def test_krclassifier(self):
"""
Test with a KerasClassifier.
:return:
"""
# Build KerasClassifier
krc, sess = get_classifier_kr()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# First attack (without EoT):
fgsm = FastGradientMethod(classifier=krc, targeted=True)
params = {'y': random_targets(y_test, krc.nb_classes)}
x_test_adv = fgsm.generate(x_test, **params)
# Second attack (with EoT):
def t(x):
return x
def transformation():
while True:
yield t
eot = ExpectationOverTransformations(classifier=krc,
sample_size=1,
transformation=transformation)
fgsm_with_eot = FastGradientMethod(classifier=eot, targeted=True)
x_test_adv_with_eot = fgsm_with_eot.generate(x_test, **params)
self.assertTrue(
(np.abs(x_test_adv - x_test_adv_with_eot) < 0.001).all())
def test_loss_gradient(self):
classifier = get_classifier_kr()
# Test gradient
gradients = classifier.loss_gradient(self.x_test, self.y_test)
self.assertEqual(gradients.shape, (NB_TEST, 28, 28, 1))
expected_gradients_1 = np.asarray([
0.0559206, 0.05338925, 0.0648919, 0.07925165, -0.04029291,
-0.11281465, 0.01850601, 0.00325054, 0.08163195, 0.03333949,
0.031766, -0.02420463, -0.07815556, -0.04698735, 0.10711591,
0.04086434, -0.03441073, 0.01071284, -0.04229195, -0.01386157,
0.02827487, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
])
np.testing.assert_array_almost_equal(gradients[0, 14, :, 0],
expected_gradients_1,
decimal=4)
expected_gradients_2 = np.asarray([
0.00210803, 0.00213919, 0.00520981, 0.00548001, -0.0023059,
0.00432077, 0.00274945, 0.0, 0.0, -0.0583441, -0.00616604,
0.0526219, -0.02373985, 0.05273106, 0.10711591, 0.12773865,
0.0689289, 0.01337799, 0.10032021, 0.01681096, -0.00028647,
-0.05588859, 0.01474165, 0.0, 0.0, 0.0, 0.0, 0.0
])
np.testing.assert_array_almost_equal(gradients[0, :, 14, 0],
expected_gradients_2,
decimal=4)
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc = get_classifier_kr()
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Attack
up = UniversalPerturbation(krc,
max_iter=1,
attacker="ead",
attacker_params={
"max_iter": 5,
"targeted": False
})
x_train_adv = up.generate(x_train)
self.assertTrue((up.fooling_rate >= 0.2) or not up.converged)
x_test_adv = x_test + up.noise
self.assertFalse((x_test == x_test_adv).all())
train_y_pred = np.argmax(krc.predict(x_train_adv), axis=1)
test_y_pred = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == test_y_pred).all())
self.assertFalse((np.argmax(y_train, axis=1) == train_y_pred).all())
def test_fit_image_generator(self):
classifier = get_classifier_kr()
labels_test = np.argmax(self.y_test, axis=1)
accuracy = np.sum(
np.argmax(classifier.predict(self.x_test), axis=1) ==
labels_test) / NB_TEST
logger.info('Accuracy: %.2f%%', (accuracy * 100))
keras_gen = ImageDataGenerator(width_shift_range=0.075,
height_shift_range=0.075,
rotation_range=12,
shear_range=0.075,
zoom_range=0.05,
fill_mode='constant',
cval=0)
keras_gen.fit(self.x_train)
data_gen = KerasDataGenerator(generator=keras_gen.flow(
self.x_train, self.y_train, batch_size=BATCH_SIZE),
size=NB_TRAIN,
batch_size=BATCH_SIZE)
classifier.fit_generator(generator=data_gen, nb_epochs=2)
accuracy_2 = np.sum(
np.argmax(classifier.predict(self.x_test), axis=1) ==
labels_test) / NB_TEST
logger.info('Accuracy: %.2f%%', (accuracy_2 * 100))
self.assertEqual(accuracy, 0.32)
self.assertAlmostEqual(accuracy_2, 0.35, delta=0.02)
def test_save(self):
path = 'tmp'
filename = 'model.h5'
classifier = get_classifier_kr()
classifier.save(filename, path=path)
self.assertTrue(os.path.isfile(os.path.join(path, filename)))
os.remove(os.path.join(path, filename))
def test_subsetscan_detector(self):
(x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist')
x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN]
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
# Keras classifier
classifier, _ = get_classifier_kr()
# Generate adversarial samples:
attacker = FastGradientMethod(classifier, eps=0.5)
x_train_adv = attacker.generate(x_train)
x_test_adv = attacker.generate(x_test)
# Compile training data for detector:
x_train_detector = np.concatenate((x_train, x_train_adv), axis=0)
bgd = x_train
clean = x_test
anom = x_test_adv
detector = SubsetScanningDetector(classifier, bgd, layer=1)
_, _, dpwr = detector.scan(clean, clean)
self.assertAlmostEqual(dpwr, 0.5)
_, _, dpwr = detector.scan(clean, anom)
self.assertGreater(dpwr, 0.5)
_, _, dpwr = detector.scan(clean, x_train_detector, 85, 15)
self.assertGreater(dpwr, 0.5)
def test_learning_phase(self):
classifier = get_classifier_kr()
self.assertFalse(hasattr(classifier, '_learning_phase'))
classifier.set_learning_phase(False)
self.assertFalse(classifier.learning_phase)
classifier.set_learning_phase(True)
self.assertTrue(classifier.learning_phase)
self.assertTrue(hasattr(classifier, '_learning_phase'))
def test_repr(self):
classifier = get_classifier_kr()
repr_ = repr(classifier)
self.assertIn('art.classifiers.keras.KerasClassifier', repr_)
self.assertIn('use_logits=False, channel_index=3', repr_)
self.assertIn(
'clip_values=(0, 1), defences=None, preprocessing=(0, 1)', repr_)
self.assertIn('input_layer=0, output_layer=0', repr_)
def test_layers(self):
classifier = get_classifier_kr()
self.assertEqual(len(classifier.layer_names), 3)
layer_names = classifier.layer_names
for i, name in enumerate(layer_names):
activation_i = classifier.get_activations(self.x_test, i)
activation_name = classifier.get_activations(self.x_test, name)
np.testing.assert_array_equal(activation_name, activation_i)
def setUpClass(cls):
k.set_learning_phase(1)
(x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist')
x_train, y_train, x_test, y_test = x_train[:NB_TRAIN], y_train[:NB_TRAIN], x_test[:NB_TEST], y_test[:NB_TEST]
cls.mnist = (x_train, y_train), (x_test, y_test)
# Keras classifier
cls.classifier_k = get_classifier_kr()
def test_krclassifier(self):
"""
Test with a KerasClassifier.
:return:
"""
# Build KerasClassifier
krc = get_classifier_kr()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# First FGSM attack:
fgsm = FastGradientMethod(classifier=krc, targeted=True)
params = {'y': random_targets(y_test, krc.nb_classes())}
x_test_adv = fgsm.generate(x_test, **params)
# Initialize RS object and attack with FGSM
rs = RandomizedSmoothing(classifier=krc,
sample_size=100,
scale=0.01,
alpha=0.001)
fgsm_with_rs = FastGradientMethod(classifier=rs, targeted=True)
x_test_adv_with_rs = fgsm_with_rs.generate(x_test, **params)
# Compare results
# check shapes are equal and values are within a certain range
self.assertEqual(x_test_adv.shape, x_test_adv_with_rs.shape)
self.assertTrue((np.abs(x_test_adv - x_test_adv_with_rs) < 0.75).all())
# Check basic functionality of RS object
# check predict
y_test_smooth = rs.predict(x=x_test)
y_test_base = krc.predict(x=x_test)
self.assertEqual(y_test_smooth.shape, y_test.shape)
self.assertTrue((np.sum(y_test_smooth, axis=1) <= np.ones(
(NB_TEST, ))).all())
self.assertTrue((np.argmax(y_test_smooth,
axis=1) == np.argmax(y_test_base,
axis=1)).all())
# check gradients
grad_smooth1 = rs.loss_gradient(x=x_test, y=y_test)
grad_smooth2 = rs.class_gradient(x=x_test, label=None)
grad_smooth3 = rs.class_gradient(x=x_test,
label=np.argmax(y_test, axis=1))
self.assertEqual(grad_smooth1.shape, x_test_adv.shape)
self.assertEqual(grad_smooth2.shape[0], NB_TEST)
self.assertEqual(grad_smooth3.shape[0], NB_TEST)
# check certification
pred, radius = rs.certify(x=x_test, n=250)
self.assertEqual(len(pred), NB_TEST)
self.assertEqual(len(radius), NB_TEST)
self.assertTrue((radius <= 1).all())
self.assertTrue((pred < y_test.shape[1]).all())
def setUpClass(cls):
(x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist')
cls.x_train = x_train[:NB_TRAIN]
cls.y_train = y_train[:NB_TRAIN]
cls.x_test = x_test[:NB_TEST]
cls.y_test = y_test[:NB_TEST]
# Keras classifier
cls.classifier_k = get_classifier_kr()
scores = cls.classifier_k._model.evaluate(x_train, y_train)
logger.info('[Keras, MNIST] Accuracy on training set: %.2f%%',
(scores[1] * 100))
scores = cls.classifier_k._model.evaluate(x_test, y_test)
logger.info('[Keras, MNIST] Accuracy on test set: %.2f%%',
(scores[1] * 100))
# Create basic CNN on MNIST using TensorFlow
cls.classifier_tf, sess = get_classifier_tf()
scores = get_labels_np_array(cls.classifier_tf.predict(x_train))
accuracy = np.sum(
np.argmax(scores, axis=1) == np.argmax(y_train,
axis=1)) / y_train.shape[0]
logger.info('[TF, MNIST] Accuracy on training set: %.2f%%',
(accuracy * 100))
scores = get_labels_np_array(cls.classifier_tf.predict(x_test))
accuracy = np.sum(
np.argmax(scores, axis=1) == np.argmax(y_test,
axis=1)) / y_test.shape[0]
logger.info('[TF, MNIST] Accuracy on test set: %.2f%%',
(accuracy * 100))
# Create basic PyTorch model
cls.classifier_py = get_classifier_pt()
x_train, x_test = np.swapaxes(x_train, 1, 3), np.swapaxes(x_test, 1, 3)
scores = get_labels_np_array(
cls.classifier_py.predict(x_train.astype(np.float32)))
accuracy = np.sum(
np.argmax(scores, axis=1) == np.argmax(y_train,
axis=1)) / y_train.shape[0]
logger.info('[PyTorch, MNIST] Accuracy on training set: %.2f%%',
(accuracy * 100))
scores = get_labels_np_array(
cls.classifier_py.predict(x_test.astype(np.float32)))
accuracy = np.sum(
np.argmax(scores, axis=1) == np.argmax(y_test,
axis=1)) / y_test.shape[0]
logger.info('[PyTorch, MNIST] Accuracy on test set: %.2f%%',
(accuracy * 100))
def test_loss_functions(self):
loss_names = [
'categorical_hinge', 'categorical_crossentropy',
'sparse_categorical_crossentropy', 'binary_crossentropy',
'kullback_leibler_divergence', 'cosine_proximity'
]
for loss_name in loss_names:
classifier = get_classifier_kr(loss_name=loss_name)
classifier.fit(x=self.x_test, y=self.y_test, nb_epochs=1)
classifier.predict(x=self.x_test)
classifier.class_gradient(self.x_test, label=5)
classifier.loss_gradient(x=self.x_test, y=self.y_test)
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc = get_classifier_kr()
# Targeted attack
# zoo = ZooAttack(classifier=krc, targeted=True, batch_size=5)
# params = {'y': random_targets(self.y_test, krc.nb_classes())}
# x_test_adv = zoo.generate(self.x_test, **params)
#
# self.assertFalse((self.x_test == x_test_adv).all())
# self.assertLessEqual(np.amax(x_test_adv), 1.0)
# self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
# target = np.argmax(params['y'], axis=1)
# y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
# logger.debug('ZOO target: %s', target)
# logger.debug('ZOO actual: %s', y_pred_adv)
# logger.info('ZOO success rate on MNIST: %.2f', (sum(target == y_pred_adv) / float(len(target))))
# Untargeted attack
# zoo = ZooAttack(classifier=krc, targeted=False, max_iter=20)
zoo = ZooAttack(classifier=krc, targeted=False, batch_size=5)
# x_test_adv = zoo.generate(x_test)
params = {'y': random_targets(self.y_test, krc.nb_classes())}
x_test_adv = zoo.generate(self.x_test, **params)
# x_test_adv_true = [0.00000000e+00, 2.50167388e-04, 1.50529508e-04, 4.69674182e-04,
# 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
# 1.67321396e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
# 0.00000000e+00, 2.08451956e-06, 0.00000000e+00, 0.00000000e+00,
# 2.53360748e-01, 9.60119188e-01, 9.85227525e-01, 2.53600776e-01,
# 0.00000000e+00, 0.00000000e+00, 5.23251540e-04, 0.00000000e+00,
# 0.00000000e+00, 0.00000000e+00, 1.08632184e-05, 0.00000000e+00]
#
# for i in range(14):
# self.assertAlmostEqual(x_test_adv_true[i], x_test_adv[0, 14, i, 0])
# self.assertFalse((x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
y_pred = np.argmax(krc.predict(self.x_test), axis=1)
logger.debug('ZOO actual: %s', y_pred_adv)
logger.info('ZOO success rate on MNIST: %.2f',
(sum(y_pred != y_pred_adv) / float(len(y_pred))))
# Clean-up
k.clear_session()
def test_binary_activation_detector(self):
"""
Test the binary activation detector end-to-end.
:return:
"""
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN]
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
# Keras classifier
classifier, _ = get_classifier_kr()
# Generate adversarial samples:
attacker = FastGradientMethod(classifier, eps=0.1)
x_train_adv = attacker.generate(x_train[:NB_TRAIN])
x_test_adv = attacker.generate(x_test[:NB_TRAIN])
# Compile training data for detector:
x_train_detector = np.concatenate((x_train[:NB_TRAIN], x_train_adv), axis=0)
y_train_detector = np.concatenate((np.array([[1, 0]] * NB_TRAIN), np.array([[0, 1]] * NB_TRAIN)), axis=0)
# Create a simple CNN for the detector
activation_shape = classifier.get_activations(x_test[:1], 0).shape[1:]
number_outputs = 2
model = Sequential()
model.add(MaxPooling2D(pool_size=(2, 2), input_shape=activation_shape))
model.add(Flatten())
model.add(Dense(number_outputs, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adam(lr=0.01),
metrics=['accuracy'])
# Create detector and train it.
# Detector consider activations at layer=0:
detector = BinaryActivationDetector(classifier=classifier,
detector=KerasClassifier(model=model, clip_values=(0, 1), use_logits=False),
layer=0)
detector.fit(x_train_detector, y_train_detector, nb_epochs=2, batch_size=128)
# Apply detector on clean and adversarial test data:
test_detection = np.argmax(detector.predict(x_test), axis=1)
test_adv_detection = np.argmax(detector.predict(x_test_adv), axis=1)
# Assert there is at least one true positive and negative
nb_true_positives = len(np.where(test_adv_detection == 1)[0])
nb_true_negatives = len(np.where(test_detection == 0)[0])
logger.debug('Number of true positives detected: %i', nb_true_positives)
logger.debug('Number of true negatives detected: %i', nb_true_negatives)
self.assertGreater(nb_true_positives, 0)
self.assertGreater(nb_true_negatives, 0)
def setUpClass(cls):
k.clear_session()
k.set_learning_phase(1)
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train, x_test, y_test = x_train[:
NB_TRAIN], y_train[:
NB_TRAIN], x_test[:
NB_TEST], y_test[:
NB_TEST]
cls.mnist = (x_train, y_train), (x_test, y_test)
# Load small Keras model
cls.model_mnist = get_classifier_kr()
def test_shapes(self):
classifier = get_classifier_kr()
predictions = classifier.predict(self.x_test)
self.assertEqual(predictions.shape, self.y_test.shape)
self.assertEqual(classifier.nb_classes(), 10)
class_gradients = classifier.class_gradient(self.x_test[:11])
self.assertEqual(class_gradients.shape,
tuple([11, 10] + list(self.x_test[1].shape)))
loss_gradients = classifier.loss_gradient(self.x_test[:11],
self.y_test[:11])
self.assertEqual(loss_gradients.shape, self.x_test[:11].shape)
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
krc = get_classifier_kr()
attack_ap = AdversarialPatch(krc, rotation_max=22.5, scale_min=0.1, scale_max=1.0, learning_rate=5.0,
batch_size=10, max_iter=500)
patch_adv, _ = attack_ap.generate(self.x_train)
self.assertAlmostEqual(patch_adv[8, 8, 0], -3.336, delta=0.1)
self.assertAlmostEqual(patch_adv[14, 14, 0], 18.574, delta=0.1)
self.assertAlmostEqual(float(np.sum(patch_adv)), 1054.587, delta=0.1)
k.clear_session()
def test_fit(self):
labels = np.argmax(self.y_test, axis=1)
classifier = get_classifier_kr()
accuracy = np.sum(
np.argmax(classifier.predict(self.x_test), axis=1) ==
labels) / NB_TEST
logger.info('Accuracy: %.2f%%', (accuracy * 100))
classifier.fit(self.x_train,
self.y_train,
batch_size=BATCH_SIZE,
nb_epochs=2)
accuracy_2 = np.sum(
np.argmax(classifier.predict(self.x_test), axis=1) ==
labels) / NB_TEST
logger.info('Accuracy: %.2f%%', (accuracy_2 * 100))
self.assertEqual(accuracy, 0.32)
self.assertEqual(accuracy_2, 0.73)
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc, sess = get_classifier_kr()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# First attack
ead = ElasticNet(classifier=krc, targeted=True, max_iter=2)
params = {'y': random_targets(y_test, krc.nb_classes)}
x_test_adv = ead.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
logger.debug('EAD target: %s', target)
logger.debug('EAD actual: %s', y_pred_adv)
logger.info('EAD success rate: %.2f%%',
(100 * sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target == y_pred_adv).any())
# Second attack
ead = ElasticNet(classifier=krc, targeted=False, max_iter=2)
params = {'y': random_targets(y_test, krc.nb_classes)}
x_test_adv = ead.generate(x_test, **params)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
logger.debug('EAD target: %s', target)
logger.debug('EAD actual: %s', y_pred_adv)
logger.info('EAD success rate: %.2f',
(100 * sum(target != y_pred_adv) / float(len(target))))
self.assertTrue((target != y_pred_adv).any())
# Kill Keras
k.clear_session()
def test_fit_generator(self):
classifier = get_classifier_kr()
labels = np.argmax(self.y_test, axis=1)
accuracy = np.sum(
np.argmax(classifier.predict(self.x_test), axis=1) ==
labels) / NB_TEST
logger.info('Accuracy: %.2f%%', (accuracy * 100))
gen = generator_fit(self.x_train, self.y_train, batch_size=BATCH_SIZE)
data_gen = KerasDataGenerator(generator=gen,
size=NB_TRAIN,
batch_size=BATCH_SIZE)
classifier.fit_generator(generator=data_gen, nb_epochs=2)
accuracy_2 = np.sum(
np.argmax(classifier.predict(self.x_test), axis=1) ==
labels) / NB_TEST
logger.info('Accuracy: %.2f%%', (accuracy_2 * 100))
self.assertEqual(accuracy, 0.32)
self.assertEqual(accuracy_2, 0.36)
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc, _ = get_classifier_kr()
# Get MNIST
(x_train, _), (_, _) = self.mnist
# Attack
attack_ap = AdversarialPatch(krc, rotation_max=22.5, scale_min=0.1, scale_max=1.0, learning_rate=5.0,
batch_size=10, max_iter=500)
patch_adv, _ = attack_ap.generate(x_train)
self.assertLess(patch_adv[8, 8, 0] - (-3.2501425017774923), 0.01)
self.assertLess(patch_adv[14, 14, 0] - 20.48400094881169, 0.01)
self.assertLess(np.sum(patch_adv) - 1764.7681744376168, 0.01)
k.clear_session()
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc, sess = get_classifier_kr()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# First attack
cl2m = CarliniL2Method(classifier=krc, targeted=True, max_iter=10)
params = {'y': random_targets(y_test, krc.nb_classes)}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
logger.debug('CW2 Target: %s', target)
logger.debug('CW2 Actual: %s', y_pred_adv)
logger.info('CW2 Success Rate: %.2f', (np.sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target == y_pred_adv).any())
# Second attack
cl2m = CarliniL2Method(classifier=krc, targeted=False, max_iter=10)
x_test_adv = cl2m.generate(x_test)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
logger.debug('CW2 Target: %s', target)
logger.debug('CW2 Actual: %s', y_pred_adv)
logger.info('CW2 Success Rate: %.2f', (np.sum(target != y_pred_adv) / float(len(target))))
self.assertTrue((target != y_pred_adv).any())
# Clean-up
k.clear_session()
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc, sess = get_classifier_kr()
# Get MNIST
(_, _), (x_test, _) = self.mnist
# Attack
nf = NewtonFool(krc, max_iter=5, batch_size=100)
x_test_adv = nf.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
y_pred = krc.predict(x_test)
y_pred_adv = krc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= y_pred_adv_max).all())
