def test_clever_pt(self): """ Test with pytorch. :return: """ # Get MNIST batch_size, nb_train, nb_test = 100, 1000, 10 (x_train, y_train), (x_test, y_test), _, _ = load_mnist() x_train, y_train = x_train[:nb_train], y_train[:nb_train] x_test, y_test = x_test[:nb_test], y_test[:nb_test] x_train = np.swapaxes(x_train, 1, 3) x_test = np.swapaxes(x_test, 1, 3) # Get the classifier ptc = self._create_ptclassifier() ptc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=1) # Test targeted clever res0 = clever_t(ptc, x_test[-1], 2, 10, 5, r_l1, norm=1, pool_factor=3) res1 = clever_t(ptc, x_test[-1], 2, 10, 5, r_l2, norm=2, pool_factor=3) res2 = clever_t(ptc, x_test[-1], 2, 10, 5, r_li, norm=np.inf, pool_factor=3) print("Target pt: ", res0, res1, res2) self.assertFalse(res0 == res1) self.assertFalse(res1 == res2) self.assertFalse(res2 == res0) # Test untargeted clever res0 = clever_u(ptc, x_test[-1], 10, 5, r_l1, norm=1, pool_factor=3) res1 = clever_u(ptc, x_test[-1], 10, 5, r_l2, norm=2, pool_factor=3) res2 = clever_u(ptc, x_test[-1], 10, 5, r_li, norm=np.inf, pool_factor=3) print("Untarget pt: ", res0, res1, res2) self.assertFalse(res0 == res1) self.assertFalse(res1 == res2) self.assertFalse(res2 == res0)
def test_clever_kr(self): """ Test with keras. :return: """ # Get MNIST batch_size, nb_train, nb_test = 100, 1000, 10 (x_train, y_train), (x_test, y_test), _, _ = load_mnist() x_train, y_train = x_train[:nb_train], y_train[:nb_train] x_test, y_test = x_test[:nb_test], y_test[:nb_test] # Get the classifier krc = self._create_krclassifier() krc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=1) # Test targeted clever res0 = clever_t(krc, x_test[-1], 2, 10, 5, r_l1, norm=1, pool_factor=3) res1 = clever_t(krc, x_test[-1], 2, 10, 5, r_l2, norm=2, pool_factor=3) res2 = clever_t(krc, x_test[-1], 2, 10, 5, r_li, norm=np.inf, pool_factor=3) print("Target kr: ", res0, res1, res2) self.assertNotEqual(res0, res1) self.assertNotEqual(res1, res2) self.assertNotEqual(res2, res0) # Test untargeted clever res0 = clever_u(krc, x_test[-1], 10, 5, r_l1, norm=1, pool_factor=3) res1 = clever_u(krc, x_test[-1], 10, 5, r_l2, norm=2, pool_factor=3) res2 = clever_u(krc, x_test[-1], 10, 5, r_li, norm=np.inf, pool_factor=3) print("Untarget kr: ", res0, res1, res2) self.assertNotEqual(res0, res1) self.assertNotEqual(res1, res2) self.assertNotEqual(res2, res0)
def test_clever_tf(self): """ Test with tensorflow. :return: """ # Get MNIST batch_size, nb_train, nb_test = 100, 1000, 10 (x_train, y_train), (x_test, y_test), _, _ = load_mnist() x_train, y_train = x_train[:nb_train], y_train[:nb_train] x_test, y_test = x_test[:nb_test], y_test[:nb_test] # Get the classifier tfc = self._create_tfclassifier() tfc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=1) # TODO Need to configure r # Test targeted clever res0 = clever_t(tfc, x_test[-1], 2, 10, 5, r_l1, norm=1, pool_factor=3) res1 = clever_t(tfc, x_test[-1], 2, 10, 5, r_l2, norm=2, pool_factor=3) res2 = clever_t(tfc, x_test[-1], 2, 10, 5, r_li, norm=np.inf, pool_factor=3) print("Target tf: ", res0, res1, res2) self.assertFalse(res0 == res1) self.assertFalse(res1 == res2) self.assertFalse(res2 == res0) # Test untargeted clever res0 = clever_u(tfc, x_test[-1], 10, 5, r_l1, norm=1, pool_factor=3) res1 = clever_u(tfc, x_test[-1], 10, 5, r_l2, norm=2, pool_factor=3) res2 = clever_u(tfc, x_test[-1], 10, 5, r_li, norm=np.inf, pool_factor=3) print("Untarget tf: ", res0, res1, res2) self.assertFalse(res0 == res1) self.assertFalse(res1 == res2) self.assertFalse(res2 == res0)
def test_clever_u(self): """ Test the untargeted version. :return: """ print("Test if the untargeted version works on a true classifier/data") # Define session & params session = tf.Session() k.set_session(session) comp_params = { "loss": 'categorical_crossentropy', "optimizer": 'adam', "metrics": ['accuracy'] } # Get MNIST (X_train, Y_train), (_, _), _, _ = load_mnist() X_train, Y_train = X_train[:NB_TRAIN], Y_train[:NB_TRAIN] im_shape = X_train[0].shape # Get classifier classifier = CNN(im_shape, act="relu") classifier.compile(comp_params) classifier.fit(X_train, Y_train, epochs=1, batch_size=BATCH_SIZE, verbose=0) res = clever_u(X_train[-1], classifier, 2, 10, 5, session) self.assertGreater(res[0], res[1]) self.assertGreater(res[1], res[2])
def test_clever_u_unit(self): """ Test the untargeted version with simplified data. :return: """ print("Unit test for the untargeted version with simplified data.") # Define session & params session = tf.Session() k.set_session(session) # Get classifier classifier = TestClassifier() # Compute scores res = clever_u(np.array([1, 0]), classifier, 20, 10, 1, session) # Test self.assertAlmostEqual(res[0], 0.9999999999999998, delta=0.00001) self.assertAlmostEqual(res[1], 0.7071067811865474, delta=0.00001) self.assertAlmostEqual(res[2], 0.4999999999999999, delta=0.00001)
# Test targeted clever # res0 = clever_t(classifier, x_test[-1], 2, 10, 5, R_L1, norm=1, pool_factor=3) # res1 = clever_t(classifier, x_test[-1], 2, 10, 5, R_L2, norm=2, pool_factor=3) # res2 = clever_t(classifier, x_test[-1], 2, 10, 5, R_LI, norm=np.inf, pool_factor=3) # print("Targeted PyTorch: %f %f %f", res0, res1, res2) # Test untargeted clever # res0 = clever_u(classifier, x_test[-1], 10, 5, R_L1, norm=1, pool_factor=3) # res1 = clever_u(classifier, x_test[-1], 10, 5, R_L2, norm=2, pool_factor=3) total = 0 for i in range(10): res = clever_u(classifier, x_test[-i], 50, 10, R_LI, norm=np.inf, pool_factor=3) total += res print("Untargeted PyTorch: i: %d, res:%f", i, res) print('==============================\n', total / 10) total = 0 for i in range(10): res = clever_u(classifier, x_test[i], 50, 10, R_LI, norm=np.inf, pool_factor=3)