def test_loss_gradient(self):
classifier = get_classifier_pt()
gradients = classifier.loss_gradient(self.x_test, self.y_test)
self.assertEqual(gradients.shape, (NB_TEST, 1, 28, 28))
expected_gradients_1 = np.asarray([
1.15053124e-04, 1.09845030e-04, 1.15488816e-04, 3.37422716e-05,
1.84526041e-04, 1.60381125e-04, 3.05866299e-04, 1.81207652e-04,
6.27528992e-04, -8.84818073e-05, 4.38439834e-04, 9.20038365e-05,
-4.22611454e-04, -8.74137331e-05, 1.77365995e-03, 6.17997837e-04,
-1.99291739e-04, 1.21479861e-04, -8.62729270e-04, -2.84300098e-04,
5.79916057e-04, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00
])
np.testing.assert_array_almost_equal(gradients[0, 0, :, 14],
expected_gradients_1,
decimal=4)
expected_gradients_2 = np.asarray([
-6.5131334e-04, 3.0298581e-04, 1.0413901e-03, 8.3430990e-04,
1.2461779e-04, 1.0898571e-03, 1.1422117e-03, 0.0000000e+00,
0.0000000e+00, -6.2288583e-04, -1.2216593e-04, 9.0777961e-04,
-5.1075104e-04, 8.0109172e-04, 1.7736600e-03, 1.5165898e-03,
1.3210204e-03, 6.4311037e-04, 1.2632636e-03, 2.8421549e-04,
-1.2425387e-04, 7.5472635e-05, -3.4554966e-04, 0.0000000e+00,
0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00
])
np.testing.assert_array_almost_equal(gradients[0, 0, 14, :],
expected_gradients_2,
decimal=4)
def setUpClass(cls):
(x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist')
x_train = np.swapaxes(x_train, 1, 3).astype(np.float32)
x_test = np.swapaxes(x_test, 1, 3).astype(np.float32)
cls.x_train = x_train[:NB_TRAIN]
cls.y_train = y_train[:NB_TRAIN]
cls.x_test = x_test[:NB_TEST]
cls.y_test = y_test[:NB_TEST]
# Define the internal classifier
classifier = get_classifier_pt()
# Define the internal detector
conv = nn.Conv2d(1, 16, 5)
linear = nn.Linear(2304, 1)
torch.nn.init.xavier_uniform_(conv.weight)
torch.nn.init.xavier_uniform_(linear.weight)
model = nn.Sequential(conv, nn.ReLU(), nn.MaxPool2d(2, 2), Flatten(),
linear)
model = Model(model)
loss_fn = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)
detector = PyTorchClassifier(model=model,
loss=loss_fn,
optimizer=optimizer,
input_shape=(1, 28, 28),
nb_classes=1,
clip_values=(0, 1))
# Define the detector-classifier
cls.detector_classifier = DetectorClassifier(classifier=classifier,
detector=detector)
def test_class_gradient_target(self):
classifier = get_classifier_pt()
gradients = classifier.class_gradient(self.x_test, label=3)
self.assertEqual(gradients.shape, (NB_TEST, 1, 1, 28, 28))
expected_gradients_1 = np.asarray([
-2.4956216e-03, -2.3826526e-03, -2.5130073e-03, -7.8883994e-04,
5.7082525e-03, 5.8527971e-03, 3.7764946e-03, 4.5487015e-03,
5.7944758e-03, -2.0162186e-03, 2.8324679e-03, 1.8695745e-03,
-3.1888916e-04, 1.4913176e-03, 6.9031012e-03, 1.4145866e-03,
1.5333943e-03, 2.6929809e-05, -3.3573490e-03, -1.1134071e-03,
2.2711302e-03, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,
0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00
])
np.testing.assert_array_almost_equal(gradients[0, 0, 0, :, 14],
expected_gradients_1,
decimal=4)
expected_gradients_2 = np.asarray([
-0.003264, 0.00116614, 0.00425005, 0.00325745, 0.00083933,
0.00457561, 0.00499626, 0.00000000, 0.00000000, -0.00372264,
-0.00048806, 0.0039387, -0.00194692, 0.00372874, 0.0069031,
0.00504249, 0.00077568, -0.0002209, 0.00204011, 0.00011565,
0.00013972, -0.00219941, 0.00097462, 0.00000000, 0.00000000,
0.00000000, 0.00000000, 0.00000000
])
np.testing.assert_array_almost_equal(gradients[0, 0, 0, 14, :],
expected_gradients_2,
decimal=4)
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
x_test = np.swapaxes(x_test, 1, 3)
# First attack
ead = ElasticNet(classifier=ptc, targeted=True, max_iter=2)
params = {'y': random_targets(y_test, ptc.nb_classes)}
x_test_adv = ead.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second attack
ead = ElasticNet(classifier=ptc, targeted=False, max_iter=2)
params = {'y': random_targets(y_test, ptc.nb_classes)}
x_test_adv = ead.generate(x_test, **params)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target != y_pred_adv).any())
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
x_train = np.swapaxes(x_train, 1, 3).astype(np.float32)
x_test = np.swapaxes(x_test, 1, 3).astype(np.float32)
# Attack
up = UniversalPerturbation(ptc,
max_iter=1,
attacker="newtonfool",
attacker_params={"max_iter": 5})
x_train_adv = up.generate(x_train)
self.assertTrue((up.fooling_rate >= 0.2) or not up.converged)
x_test_adv = x_test + up.noise
self.assertFalse((x_test == x_test_adv).all())
train_y_pred = np.argmax(ptc.predict(x_train_adv), axis=1)
test_y_pred = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == test_y_pred).all())
self.assertFalse((np.argmax(y_train, axis=1) == train_y_pred).all())
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
x_test = np.swapaxes(x_test, 1, 3)
# First attack
cl2m = CarliniL2Method(classifier=ptc, targeted=True, max_iter=10)
params = {'y': random_targets(y_test, ptc.nb_classes)}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
logger.info('CW2 Success Rate: %.2f', (sum(target == y_pred_adv) / float(len(target))))
# Second attack
cl2m = CarliniL2Method(classifier=ptc, targeted=False, max_iter=10)
x_test_adv = cl2m.generate(x_test)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target != y_pred_adv).any())
logger.info('CW2 Success Rate: %.2f', (sum(target != y_pred_adv) / float(len(target))))
def test_fit_generator(self):
classifier = get_classifier_pt()
accuracy = np.sum(
np.argmax(classifier.predict(self.x_test), axis=1) == np.argmax(
self.y_test, axis=1)) / NB_TEST
logger.info('Accuracy: %.2f%%', (accuracy * 100))
# Create tensors from data
x_train_tens = torch.from_numpy(self.x_train)
x_train_tens = x_train_tens.float()
y_train_tens = torch.from_numpy(self.y_train)
# Create PyTorch dataset and loader
dataset = torch.utils.data.TensorDataset(x_train_tens, y_train_tens)
data_loader = DataLoader(dataset=dataset, batch_size=5, shuffle=True)
data_gen = PyTorchDataGenerator(data_loader,
size=NB_TRAIN,
batch_size=5)
# Fit model with generator
classifier.fit_generator(data_gen, nb_epochs=2)
accuracy_2 = np.sum(
np.argmax(classifier.predict(self.x_test), axis=1) == np.argmax(
self.y_test, axis=1)) / NB_TEST
logger.info('Accuracy: %.2f%%', (accuracy_2 * 100))
self.assertEqual(accuracy, 0.3)
self.assertAlmostEqual(accuracy_2, 0.7, delta=0.1)
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
x_test = np.swapaxes(self.x_test, 1, 3).astype(np.float32)
# First targeted attack
boundary = BoundaryAttack(classifier=ptc, targeted=True, max_iter=20)
params = {'y': random_targets(self.y_test, ptc.nb_classes())}
x_test_adv = boundary.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack
boundary = BoundaryAttack(classifier=ptc, targeted=False, max_iter=20)
x_test_adv = boundary.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(ptc.predict(x_test), axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Get MNIST
(x_train, _), (x_test, _) = self.mnist
x_train = np.swapaxes(x_train, 1, 3)
x_test = np.swapaxes(x_test, 1, 3)
# Attack
attack_st = SpatialTransformation(ptc,
max_translation=10.0,
num_translations=3,
max_rotation=30.0,
num_rotations=3)
x_train_adv = attack_st.generate(x_train)
self.assertLessEqual(abs(x_train_adv[0, 0, 13, 5] - 0.374206543), 0.01)
self.assertLessEqual(abs(attack_st.fooling_rate - 0.361), 0.01)
self.assertEqual(attack_st.attack_trans_x, 0)
self.assertEqual(attack_st.attack_trans_y, -3)
self.assertEqual(attack_st.attack_rot, 30.0)
x_test_adv = attack_st.generate(x_test)
self.assertLessEqual(abs(x_test_adv[0, 0, 14, 14] - 0.008591662), 0.01)
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
x_train = np.swapaxes(self.x_train, 1, 3).astype(np.float32)
x_test = np.swapaxes(self.x_test, 1, 3).astype(np.float32)
# Attack
attack_st = SpatialTransformation(ptc,
max_translation=10.0,
num_translations=3,
max_rotation=30.0,
num_rotations=3)
x_train_adv = attack_st.generate(x_train)
print('abs(x_train_adv[0, 0, 13, :]', abs(x_train[0, 0, 13, :]))
print('abs(x_train_adv[0, 0, 13, :]', abs(x_train_adv[0, 0, 13, :]))
self.assertAlmostEqual(x_train_adv[0, 0, 13, 7], 0.287, delta=0.01)
self.assertAlmostEqual(attack_st.fooling_rate, 0.82, delta=0.01)
self.assertEqual(attack_st.attack_trans_x, 0)
self.assertEqual(attack_st.attack_trans_y, 3)
self.assertEqual(attack_st.attack_rot, -30.0)
x_test_adv = attack_st.generate(x_test)
self.assertLessEqual(abs(x_test_adv[0, 0, 14, 14] - 0.008591662), 0.01)
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Get MNIST
x_test, y_test = self.mnist
x_test = np.swapaxes(x_test, 1, 3)
# First attack
zoo = ZooAttack(classifier=ptc, targeted=True, max_iter=10)
params = {'y': random_targets(y_test, ptc.nb_classes)}
x_test_adv = zoo.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
logger.debug('ZOO target: %s', target)
logger.debug('ZOO actual: %s', y_pred_adv)
logger.info('ZOO success rate on MNIST: %.2f', (sum(target != y_pred_adv) / float(len(target))))
# Second attack
zoo = ZooAttack(classifier=ptc, targeted=False, max_iter=10)
x_test_adv = zoo.generate(x_test)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
y_pred = np.argmax(ptc.predict(x_test), axis=1)
logger.debug('ZOO actual: %s', y_pred_adv)
logger.info('ZOO success rate on MNIST: %.2f', (sum(y_pred != y_pred_adv) / float(len(y_pred))))
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
x_test = np.swapaxes(self.x_test, 1, 3).astype(np.float32)
# First attack
cl2m = CarliniL2Method(classifier=ptc, targeted=True, max_iter=10)
params = {'y': random_targets(self.y_test, ptc.nb_classes())}
x_test_adv = cl2m.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
logger.info('CW2 Success Rate: %.2f',
(sum(target == y_pred_adv) / float(len(target))))
# Second attack
cl2m = CarliniL2Method(classifier=ptc, targeted=False, max_iter=10)
x_test_adv = cl2m.generate(x_test)
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target != y_pred_adv).any())
logger.info('CW2 Success Rate: %.2f',
(sum(target != y_pred_adv) / float(len(target))))
def test_fit_predict(self):
classifier = get_classifier_pt()
predictions = classifier.predict(self.x_test)
accuracy = np.sum(
np.argmax(predictions, axis=1) == np.argmax(self.y_test,
axis=1)) / NB_TEST
logger.info('Accuracy after fitting: %.2f%%', (accuracy * 100))
self.assertEqual(accuracy, 0.3)
def setUpClass(cls):
(x_train, y_train), (x_test, y_test), _, _ = load_dataset('mnist')
cls.x_train = x_train[:NB_TRAIN]
cls.y_train = y_train[:NB_TRAIN]
cls.x_test = x_test[:NB_TEST]
cls.y_test = y_test[:NB_TEST]
# Keras classifier
cls.classifier_k = get_classifier_kr()
scores = cls.classifier_k._model.evaluate(x_train, y_train)
logger.info('[Keras, MNIST] Accuracy on training set: %.2f%%',
(scores[1] * 100))
scores = cls.classifier_k._model.evaluate(x_test, y_test)
logger.info('[Keras, MNIST] Accuracy on test set: %.2f%%',
(scores[1] * 100))
# Create basic CNN on MNIST using TensorFlow
cls.classifier_tf, sess = get_classifier_tf()
scores = get_labels_np_array(cls.classifier_tf.predict(x_train))
accuracy = np.sum(
np.argmax(scores, axis=1) == np.argmax(y_train,
axis=1)) / y_train.shape[0]
logger.info('[TF, MNIST] Accuracy on training set: %.2f%%',
(accuracy * 100))
scores = get_labels_np_array(cls.classifier_tf.predict(x_test))
accuracy = np.sum(
np.argmax(scores, axis=1) == np.argmax(y_test,
axis=1)) / y_test.shape[0]
logger.info('[TF, MNIST] Accuracy on test set: %.2f%%',
(accuracy * 100))
# Create basic PyTorch model
cls.classifier_py = get_classifier_pt()
x_train, x_test = np.swapaxes(x_train, 1, 3), np.swapaxes(x_test, 1, 3)
scores = get_labels_np_array(
cls.classifier_py.predict(x_train.astype(np.float32)))
accuracy = np.sum(
np.argmax(scores, axis=1) == np.argmax(y_train,
axis=1)) / y_train.shape[0]
logger.info('[PyTorch, MNIST] Accuracy on training set: %.2f%%',
(accuracy * 100))
scores = get_labels_np_array(
cls.classifier_py.predict(x_test.astype(np.float32)))
accuracy = np.sum(
np.argmax(scores, axis=1) == np.argmax(y_test,
axis=1)) / y_test.shape[0]
logger.info('[PyTorch, MNIST] Accuracy on test set: %.2f%%',
(accuracy * 100))
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
ptc = get_classifier_pt()
x_train = np.swapaxes(self.x_train, 1, 3).astype(np.float32)
attack_ap = AdversarialPatch(ptc, rotation_max=22.5, scale_min=0.1, scale_max=1.0, learning_rate=5.0,
batch_size=10, max_iter=500)
patch_adv, _ = attack_ap.generate(x_train)
self.assertAlmostEqual(patch_adv[0, 8, 8], -3.143605902784875, delta=0.1)
self.assertAlmostEqual(patch_adv[0, 14, 14], 19.790434152473054, delta=0.1)
self.assertAlmostEqual(float(np.sum(patch_adv)), 394.905, delta=0.1)
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Get MNIST
(x_train, _), (_, _) = self.mnist
x_train = np.swapaxes(x_train, 1, 3)
# Attack
attack_ap = AdversarialPatch(ptc, rotation_max=22.5, scale_min=0.1, scale_max=1.0, learning_rate=5.0,
batch_size=10, max_iter=500)
patch_adv, _ = attack_ap.generate(x_train)
self.assertLess(patch_adv[0, 8, 8] - (-3.1423605902784875), 0.01)
self.assertLess(patch_adv[0, 14, 14] - 19.790434152473054, 0.01)
self.assertLess(np.sum(patch_adv) - 383.5670772794207, 0.01)
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Get MNIST
x_test = np.swapaxes(self.x_test, 1, 3).astype(np.float32)
# First attack
# zoo = ZooAttack(classifier=ptc, targeted=True, max_iter=10, binary_search_steps=10)
# params = {'y': random_targets(self.y_test, ptc.nb_classes())}
# x_test_adv = zoo.generate(x_test, **params)
# self.assertFalse((x_test == x_test_adv).all())
# self.assertLessEqual(np.amax(x_test_adv), 1.0)
# self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
# target = np.argmax(params['y'], axis=1)
# y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
# logger.debug('ZOO target: %s', target)
# logger.debug('ZOO actual: %s', y_pred_adv)
# logger.info('ZOO success rate on MNIST: %.2f', (sum(target != y_pred_adv) / float(len(target))))
# Second attack
zoo = ZooAttack(classifier=ptc,
targeted=False,
learning_rate=1e-2,
max_iter=15,
binary_search_steps=10,
abort_early=False,
use_resize=False,
use_importance=False)
x_test_adv = zoo.generate(x_test)
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
# print(x_test[0, 0, 14, :])
# print(x_test_adv[0, 0, 14, :])
# print(np.amax(x_test - x_test_adv))
x_test_adv_expected = []
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
x_test = np.swapaxes(self.x_test, 1, 3).astype(np.float32)
# First attack
clinfm = CarliniLInfMethod(classifier=ptc,
targeted=True,
max_iter=10,
eps=0.5)
params = {'y': random_targets(self.y_test, ptc.nb_classes())}
x_test_adv = clinfm.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0 + 1e-6)
self.assertGreaterEqual(np.amin(x_test_adv), -1e-6)
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second attack
clinfm = CarliniLInfMethod(classifier=ptc,
targeted=False,
max_iter=10,
eps=0.5)
x_test_adv = clinfm.generate(x_test)
self.assertLessEqual(np.amax(x_test_adv), 1.0 + 1e-6)
self.assertGreaterEqual(np.amin(x_test_adv), -1e-6)
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target != y_pred_adv).any())
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Get MNIST
(_, _), (x_test, _) = self.mnist
x_test = np.swapaxes(x_test, 1, 3)
# Attack
nf = NewtonFool(ptc, max_iter=5, batch_size=100)
x_test_adv = nf.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
y_pred = ptc.predict(x_test)
y_pred_adv = ptc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= y_pred_adv_max).all())
def test_nb_classes(self):
classifier = get_classifier_pt()
self.assertEqual(classifier.nb_classes(), 10)
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
x_test = np.swapaxes(x_test, 1, 3).astype(np.float32)
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=ptc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10)
params = {'y': random_targets(y_test, ptc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=ptc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
params = {'y': random_targets(y_test, ptc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=ptc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(ptc.predict(x_test), axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=ptc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(ptc.predict(x_test), axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
def test_input_shape(self):
classifier = get_classifier_pt()
self.assertEqual(classifier.input_shape, (1, 28, 28))
def test_class_gradient(self):
classifier = get_classifier_pt()
# Test all gradients label = None
gradients = classifier.class_gradient(self.x_test)
self.assertEqual(gradients.shape, (NB_TEST, 10, 1, 28, 28))
expected_gradients_1 = np.asarray([
-0.00150054, -0.00143262, -0.00084528, 0.00430235, 0.00461788,
-0.00041574, 0.00425031, 0.00541148, 0.00873945, -0.00144859,
0.00644154, -0.00305683, -0.00861008, -0.005047, 0.01267533,
0.00141767, 0.00625692, -0.00056882, -0.00664478, -0.00221345,
0.00451501, 0.00000000, 0.00000000, 0.00000000, 0.00000000,
0.00000000, 0.00000000, 0.00000000
])
np.testing.assert_array_almost_equal(gradients[0, 5, 0, :, 14],
expected_gradients_1,
decimal=4)
expected_gradients_2 = np.asarray([
-0.00451381, -0.00108492, -0.00154162, -0.0025684, 0.00280577,
-0.00045833, 0.00133553, 0.00000000, 0.00000000, -0.00713912,
-0.00096829, 0.00775198, -0.00388135, 0.00725403, 0.01267533,
0.00680012, 0.00754666, 0.00996057, 0.00415468, -0.00202466,
-0.00184498, 0.00614046, -0.00652217, 0.00000000, 0.00000000,
0.00000000, 0.00000000, 0.00000000
])
np.testing.assert_array_almost_equal(gradients[0, 5, 0, 14, :],
expected_gradients_2,
decimal=4)
# Test 1 gradient label = 5
gradients = classifier.class_gradient(self.x_test, label=5)
self.assertEqual(gradients.shape, (NB_TEST, 1, 1, 28, 28))
expected_gradients_1 = np.asarray([
-0.00150054, -0.00143262, -0.00084528, 0.00430235, 0.00461788,
-0.00041574, 0.00425031, 0.00541148, 0.00873945, -0.00144859,
0.00644154, -0.00305683, -0.00861008, -0.005047, 0.01267533,
0.00141767, 0.00625692, -0.00056882, -0.00664478, -0.00221345,
0.00451501, 0.00000000, 0.00000000, 0.00000000, 0.00000000,
0.00000000, 0.00000000, 0.00000000
])
np.testing.assert_array_almost_equal(gradients[0, 0, 0, :, 14],
expected_gradients_1,
decimal=4)
expected_gradients_2 = np.asarray([
-0.00451381, -0.00108492, -0.00154162, -0.0025684, 0.00280577,
-0.00045833, 0.00133553, 0.00000000, 0.00000000, -0.00713912,
-0.00096829, 0.00775198, -0.00388135, 0.00725403, 0.01267533,
0.00680012, 0.00754666, 0.00996057, 0.00415468, -0.00202466,
-0.00184498, 0.00614046, -0.00652217, 0.00000000, 0.00000000,
0.00000000, 0.00000000, 0.00000000
])
np.testing.assert_array_almost_equal(gradients[0, 0, 0, 14, :],
expected_gradients_2,
decimal=4)
# Test a set of gradients label = array
label = np.random.randint(5, size=NB_TEST)
gradients = classifier.class_gradient(self.x_test, label=label)
self.assertEqual(gradients.shape, (NB_TEST, 1, 1, 28, 28))
expected_gradients_1 = np.asarray([
-2.4956216e-03, -2.3826526e-03, -2.5130073e-03, -7.8883994e-04,
5.7082525e-03, 5.8527971e-03, 3.7764946e-03, 4.5487015e-03,
5.7944758e-03, -2.0162186e-03, 2.8324679e-03, 1.8695745e-03,
-3.1888916e-04, 1.4913176e-03, 6.9031012e-03, 1.4145866e-03,
1.5333943e-03, 2.6929809e-05, -3.3573490e-03, -1.1134071e-03,
2.2711302e-03, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,
0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00
])
np.testing.assert_array_almost_equal(gradients[0, 0, 0, :, 14],
expected_gradients_1,
decimal=4)
expected_gradients_2 = np.asarray([
-0.003264, 0.00116614, 0.00425005, 0.00325745, 0.00083933,
0.00457561, 0.00499626, 0.00000000, 0.00000000, -0.00372264,
-0.00048806, 0.0039387, -0.00194692, 0.00372874, 0.0069031,
0.00504249, 0.00077568, -0.0002209, 0.00204011, 0.00011565,
0.00013972, -0.00219941, 0.00097462, 0.00000000, 0.00000000,
0.00000000, 0.00000000, 0.00000000
])
np.testing.assert_array_almost_equal(gradients[0, 0, 0, 14, :],
expected_gradients_2,
decimal=4)
