def test_clever_kr(self): """ Test with keras. :return: """ # Get MNIST batch_size, nb_train, nb_test = 100, 1000, 10 (x_train, y_train), (x_test, y_test), _, _ = load_mnist() x_train, y_train = x_train[:nb_train], y_train[:nb_train] x_test, y_test = x_test[:nb_test], y_test[:nb_test] # Get the classifier krc = self._create_krclassifier() krc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=1, verbose=0) # Test targeted clever res0 = clever_t(krc, x_test[-1], 2, 10, 5, R_L1, norm=1, pool_factor=3) res1 = clever_t(krc, x_test[-1], 2, 10, 5, R_L2, norm=2, pool_factor=3) res2 = clever_t(krc, x_test[-1], 2, 10, 5, R_LI, norm=np.inf, pool_factor=3) logger.info("Targeted Keras: %f %f %f", res0, res1, res2) self.assertNotEqual(res0, res1) self.assertNotEqual(res1, res2) self.assertNotEqual(res2, res0) # Test untargeted clever res0 = clever_u(krc, x_test[-1], 10, 5, R_L1, norm=1, pool_factor=3) res1 = clever_u(krc, x_test[-1], 10, 5, R_L2, norm=2, pool_factor=3) res2 = clever_u(krc, x_test[-1], 10, 5, R_LI, norm=np.inf, pool_factor=3) logger.info("Untargeted Keras: %f %f %f", res0, res1, res2) self.assertNotEqual(res0, res1) self.assertNotEqual(res1, res2) self.assertNotEqual(res2, res0)
def test_3_clever_pt(self): """ Test with pytorch. :return: """ # Get MNIST batch_size, nb_train, nb_test = 100, 1000, 10 (x_train, y_train), (x_test, y_test), _, _ = load_mnist() x_train, y_train = x_train[:nb_train], y_train[:nb_train] x_test, y_test = x_test[:nb_test], y_test[:nb_test] x_train = np.swapaxes(x_train, 1, 3).astype(np.float32) x_test = np.swapaxes(x_test, 1, 3).astype(np.float32) # Get the classifier ptc = self._create_ptclassifier() ptc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=1) # Test targeted clever res0 = clever_t(ptc, x_test[-1], 2, 10, 5, R_L1, norm=1, pool_factor=3) res1 = clever_t(ptc, x_test[-1], 2, 10, 5, R_L2, norm=2, pool_factor=3) res2 = clever_t(ptc, x_test[-1], 2, 10, 5, R_LI, norm=np.inf, pool_factor=3) logger.info("Targeted PyTorch: %f %f %f", res0, res1, res2) self.assertNotEqual(res0, res1) self.assertNotEqual(res1, res2) self.assertNotEqual(res2, res0) # Test untargeted clever res0 = clever_u(ptc, x_test[-1], 10, 5, R_L1, norm=1, pool_factor=3) res1 = clever_u(ptc, x_test[-1], 10, 5, R_L2, norm=2, pool_factor=3) res2 = clever_u(ptc, x_test[-1], 10, 5, R_LI, norm=np.inf, pool_factor=3) logger.info("Untargeted PyTorch: %f %f %f", res0, res1, res2) self.assertNotEqual(res0, res1) self.assertNotEqual(res1, res2) self.assertNotEqual(res2, res0)
def test_2_clever_tf(self): """ Test with TensorFlow. :return: """ # Get MNIST batch_size, nb_train, nb_test = 100, 1000, 10 (x_train, y_train), (x_test, y_test), _, _ = load_mnist() x_train, y_train = x_train[:nb_train], y_train[:nb_train] x_test, y_test = x_test[:nb_test], y_test[:nb_test] # Get the classifier tfc = self._create_tfclassifier() tfc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=1) # TODO Need to configure r # Test targeted clever res0 = clever_t(tfc, x_test[-1], 2, 10, 5, R_L1, norm=1, pool_factor=3) res1 = clever_t(tfc, x_test[-1], 2, 10, 5, R_L2, norm=2, pool_factor=3) res2 = clever_t(tfc, x_test[-1], 2, 10, 5, R_LI, norm=np.inf, pool_factor=3) logger.info("Targeted TensorFlow: %f %f %f", res0, res1, res2) self.assertNotEqual(res0, res1) self.assertNotEqual(res1, res2) self.assertNotEqual(res2, res0) # Test untargeted clever res0 = clever_u(tfc, x_test[-1], 10, 5, R_L1, norm=1, pool_factor=3) res1 = clever_u(tfc, x_test[-1], 10, 5, R_L2, norm=2, pool_factor=3) res2 = clever_u(tfc, x_test[-1], 10, 5, R_LI, norm=np.inf, pool_factor=3) logger.info("Untargeted TensorFlow: %f %f %f", res0, res1, res2) self.assertNotEqual(res0, res1) self.assertNotEqual(res1, res2) self.assertNotEqual(res2, res0)