def test_clever_pt(self):
"""
Test with pytorch.
:return:
"""
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
x_train = np.swapaxes(x_train, 1, 3)
x_test = np.swapaxes(x_test, 1, 3)
# Get the classifier
ptc = self._create_ptclassifier()
ptc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=1)
# Test targeted clever
res0 = clever_t(ptc, x_test[-1], 2, 10, 5, r_l1, norm=1, pool_factor=3)
res1 = clever_t(ptc, x_test[-1], 2, 10, 5, r_l2, norm=2, pool_factor=3)
res2 = clever_t(ptc, x_test[-1], 2, 10, 5, r_li, norm=np.inf, pool_factor=3)
print("Target pt: ", res0, res1, res2)
self.assertFalse(res0 == res1)
self.assertFalse(res1 == res2)
self.assertFalse(res2 == res0)
# Test untargeted clever
res0 = clever_u(ptc, x_test[-1], 10, 5, r_l1, norm=1, pool_factor=3)
res1 = clever_u(ptc, x_test[-1], 10, 5, r_l2, norm=2, pool_factor=3)
res2 = clever_u(ptc, x_test[-1], 10, 5, r_li, norm=np.inf, pool_factor=3)
print("Untarget pt: ", res0, res1, res2)
self.assertFalse(res0 == res1)
self.assertFalse(res1 == res2)
self.assertFalse(res2 == res0)
def test_clever_kr(self):
"""
Test with keras.
:return:
"""
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
# Get the classifier
krc = self._create_krclassifier()
krc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=1)
# Test targeted clever
res0 = clever_t(krc, x_test[-1], 2, 10, 5, r_l1, norm=1, pool_factor=3)
res1 = clever_t(krc, x_test[-1], 2, 10, 5, r_l2, norm=2, pool_factor=3)
res2 = clever_t(krc, x_test[-1], 2, 10, 5, r_li, norm=np.inf, pool_factor=3)
print("Target kr: ", res0, res1, res2)
self.assertNotEqual(res0, res1)
self.assertNotEqual(res1, res2)
self.assertNotEqual(res2, res0)
# Test untargeted clever
res0 = clever_u(krc, x_test[-1], 10, 5, r_l1, norm=1, pool_factor=3)
res1 = clever_u(krc, x_test[-1], 10, 5, r_l2, norm=2, pool_factor=3)
res2 = clever_u(krc, x_test[-1], 10, 5, r_li, norm=np.inf, pool_factor=3)
print("Untarget kr: ", res0, res1, res2)
self.assertNotEqual(res0, res1)
self.assertNotEqual(res1, res2)
self.assertNotEqual(res2, res0)
def test_clever_tf(self):
"""
Test with tensorflow.
:return:
"""
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
# Get the classifier
tfc = self._create_tfclassifier()
tfc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=1)
# TODO Need to configure r
# Test targeted clever
res0 = clever_t(tfc, x_test[-1], 2, 10, 5, r_l1, norm=1, pool_factor=3)
res1 = clever_t(tfc, x_test[-1], 2, 10, 5, r_l2, norm=2, pool_factor=3)
res2 = clever_t(tfc, x_test[-1], 2, 10, 5, r_li, norm=np.inf, pool_factor=3)
print("Target tf: ", res0, res1, res2)
self.assertFalse(res0 == res1)
self.assertFalse(res1 == res2)
self.assertFalse(res2 == res0)
# Test untargeted clever
res0 = clever_u(tfc, x_test[-1], 10, 5, r_l1, norm=1, pool_factor=3)
res1 = clever_u(tfc, x_test[-1], 10, 5, r_l2, norm=2, pool_factor=3)
res2 = clever_u(tfc, x_test[-1], 10, 5, r_li, norm=np.inf, pool_factor=3)
print("Untarget tf: ", res0, res1, res2)
self.assertFalse(res0 == res1)
self.assertFalse(res1 == res2)
self.assertFalse(res2 == res0)
def test_clever_u(self):
"""
Test the untargeted version.
:return:
"""
print("Test if the untargeted version works on a true classifier/data")
# Define session & params
session = tf.Session()
k.set_session(session)
comp_params = {
"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']
}
# Get MNIST
(X_train, Y_train), (_, _), _, _ = load_mnist()
X_train, Y_train = X_train[:NB_TRAIN], Y_train[:NB_TRAIN]
im_shape = X_train[0].shape
# Get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train,
Y_train,
epochs=1,
batch_size=BATCH_SIZE,
verbose=0)
res = clever_u(X_train[-1], classifier, 2, 10, 5, session)
self.assertGreater(res[0], res[1])
self.assertGreater(res[1], res[2])
def test_clever_u_unit(self):
"""
Test the untargeted version with simplified data.
:return:
"""
print("Unit test for the untargeted version with simplified data.")
# Define session & params
session = tf.Session()
k.set_session(session)
# Get classifier
classifier = TestClassifier()
# Compute scores
res = clever_u(np.array([1, 0]), classifier, 20, 10, 1, session)
# Test
self.assertAlmostEqual(res[0], 0.9999999999999998, delta=0.00001)
self.assertAlmostEqual(res[1], 0.7071067811865474, delta=0.00001)
self.assertAlmostEqual(res[2], 0.4999999999999999, delta=0.00001)
# Test targeted clever
# res0 = clever_t(classifier, x_test[-1], 2, 10, 5, R_L1, norm=1, pool_factor=3)
# res1 = clever_t(classifier, x_test[-1], 2, 10, 5, R_L2, norm=2, pool_factor=3)
# res2 = clever_t(classifier, x_test[-1], 2, 10, 5, R_LI, norm=np.inf, pool_factor=3)
# print("Targeted PyTorch: %f %f %f", res0, res1, res2)
# Test untargeted clever
# res0 = clever_u(classifier, x_test[-1], 10, 5, R_L1, norm=1, pool_factor=3)
# res1 = clever_u(classifier, x_test[-1], 10, 5, R_L2, norm=2, pool_factor=3)
total = 0
for i in range(10):
res = clever_u(classifier,
x_test[-i],
50,
10,
R_LI,
norm=np.inf,
pool_factor=3)
total += res
print("Untargeted PyTorch: i: %d, res:%f", i, res)
print('==============================\n', total / 10)
total = 0
for i in range(10):
res = clever_u(classifier,
x_test[i],
50,
10,
R_LI,
norm=np.inf,
pool_factor=3)
