def test_clever_kr(self):
"""
Test with keras.
:return:
"""
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
# Get the classifier
krc = self._create_krclassifier()
krc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=1, verbose=0)
# Test targeted clever
res0 = clever_t(krc, x_test[-1], 2, 10, 5, R_L1, norm=1, pool_factor=3)
res1 = clever_t(krc, x_test[-1], 2, 10, 5, R_L2, norm=2, pool_factor=3)
res2 = clever_t(krc, x_test[-1], 2, 10, 5, R_LI, norm=np.inf, pool_factor=3)
logger.info("Targeted Keras: %f %f %f", res0, res1, res2)
self.assertNotEqual(res0, res1)
self.assertNotEqual(res1, res2)
self.assertNotEqual(res2, res0)
# Test untargeted clever
res0 = clever_u(krc, x_test[-1], 10, 5, R_L1, norm=1, pool_factor=3)
res1 = clever_u(krc, x_test[-1], 10, 5, R_L2, norm=2, pool_factor=3)
res2 = clever_u(krc, x_test[-1], 10, 5, R_LI, norm=np.inf, pool_factor=3)
logger.info("Untargeted Keras: %f %f %f", res0, res1, res2)
self.assertNotEqual(res0, res1)
self.assertNotEqual(res1, res2)
self.assertNotEqual(res2, res0)
def test_3_clever_pt(self):
"""
Test with pytorch.
:return:
"""
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
x_train = np.swapaxes(x_train, 1, 3).astype(np.float32)
x_test = np.swapaxes(x_test, 1, 3).astype(np.float32)
# Get the classifier
ptc = self._create_ptclassifier()
ptc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=1)
# Test targeted clever
res0 = clever_t(ptc, x_test[-1], 2, 10, 5, R_L1, norm=1, pool_factor=3)
res1 = clever_t(ptc, x_test[-1], 2, 10, 5, R_L2, norm=2, pool_factor=3)
res2 = clever_t(ptc, x_test[-1], 2, 10, 5, R_LI, norm=np.inf, pool_factor=3)
logger.info("Targeted PyTorch: %f %f %f", res0, res1, res2)
self.assertNotEqual(res0, res1)
self.assertNotEqual(res1, res2)
self.assertNotEqual(res2, res0)
# Test untargeted clever
res0 = clever_u(ptc, x_test[-1], 10, 5, R_L1, norm=1, pool_factor=3)
res1 = clever_u(ptc, x_test[-1], 10, 5, R_L2, norm=2, pool_factor=3)
res2 = clever_u(ptc, x_test[-1], 10, 5, R_LI, norm=np.inf, pool_factor=3)
logger.info("Untargeted PyTorch: %f %f %f", res0, res1, res2)
self.assertNotEqual(res0, res1)
self.assertNotEqual(res1, res2)
self.assertNotEqual(res2, res0)
def test_2_clever_tf(self):
"""
Test with TensorFlow.
:return:
"""
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
# Get the classifier
tfc = self._create_tfclassifier()
tfc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=1)
# TODO Need to configure r
# Test targeted clever
res0 = clever_t(tfc, x_test[-1], 2, 10, 5, R_L1, norm=1, pool_factor=3)
res1 = clever_t(tfc, x_test[-1], 2, 10, 5, R_L2, norm=2, pool_factor=3)
res2 = clever_t(tfc, x_test[-1], 2, 10, 5, R_LI, norm=np.inf, pool_factor=3)
logger.info("Targeted TensorFlow: %f %f %f", res0, res1, res2)
self.assertNotEqual(res0, res1)
self.assertNotEqual(res1, res2)
self.assertNotEqual(res2, res0)
# Test untargeted clever
res0 = clever_u(tfc, x_test[-1], 10, 5, R_L1, norm=1, pool_factor=3)
res1 = clever_u(tfc, x_test[-1], 10, 5, R_L2, norm=2, pool_factor=3)
res2 = clever_u(tfc, x_test[-1], 10, 5, R_LI, norm=np.inf, pool_factor=3)
logger.info("Untargeted TensorFlow: %f %f %f", res0, res1, res2)
self.assertNotEqual(res0, res1)
self.assertNotEqual(res1, res2)
self.assertNotEqual(res2, res0)
