def test_get_range(self): x = np.array([[2, 2], [0, 0], [1, 1]], dtype=np.float32) out = get_range(x) e = ([0., 0.], [2., 2.]) r = [all(xx == ee) for xx, ee in zip(out, e)] self.assertTrue(all(r)) master_seed(SEED) x = np.random.rand(10, 1, 28, 28) out = np.array(list(get_range(x, is_image=True))) np.testing.assert_almost_equal(out, [0., 0.9998], decimal=4)
def test_load(self): x_range = get_range(self.mc.data_container.x_train) # Do not load pretrained parameters squeezer = FeatureSqueezing( self.mc, clip_values=x_range, smoothing_methods=SQUEEZING_METHODS, bit_depth=8, sigma=0.1, kernel_size=3, pretrained=False, ) squeezer.load(os.path.join('save', SQUEEZER_FILE)) mc_binary = squeezer.get_def_model_container(SQUEEZING_METHODS[0]) mc_median = squeezer.get_def_model_container(SQUEEZING_METHODS[1]) mc_normal = squeezer.get_def_model_container(SQUEEZING_METHODS[2]) x_test = squeezer.model_container.data_container.x_test y_test = squeezer.model_container.data_container.y_test acc_bin_after = mc_binary.evaluate(x_test, y_test) logger.info('For binary squeezer, accuracy after load parameters: %f', acc_bin_after) self.assertGreater(acc_bin_after, 0.90) acc_med_after = mc_median.evaluate(x_test, y_test) logger.info('For median squeezer, accuracy after load parameters: %f', acc_med_after) self.assertGreater(acc_med_after, 0.90) acc_nor_after = mc_normal.evaluate(x_test, y_test) logger.info('For normal squeezer, accuracy after load parameters: %f', acc_nor_after) self.assertGreater(acc_nor_after, 0.90)
def test_squeezing_data(self): x_range = get_range(self.mc.data_container.x_train) x_train = np.copy(self.mc.data_container.x_train) squeezer = FeatureSqueezing( self.mc, clip_values=x_range, smoothing_methods=['normal', 'binary'], bit_depth=8, sigma=0.1, kernel_size=2, pretrained=False, ) # Expecting difference between input data and squeezed data mc_normal = squeezer.get_def_model_container('normal') mc_binary = squeezer.get_def_model_container('binary') self.assertFalse((x_train == mc_normal.data_container.x_train).all()) self.assertFalse((x_train == mc_binary.data_container.x_train).all()) # average perturbation l2 = np.mean(get_l2_norm(x_train, mc_normal.data_container.x_train)) logger.info('L2 norm of normal squeezer:%f', l2) self.assertLessEqual(l2, 0.2) # maximum perturbation l2 = np.max(get_l2_norm(x_train, mc_binary.data_container.x_train)) logger.info('L2 norm of binary squeezer:%f', l2) self.assertLessEqual(l2, 0.2)
def test_normalize(self): x = np.random.rand(2, 3) xmin, xmax = get_range(x) x_norm = scale_normalize(x, xmin, xmax) x_unnorm = scale_unnormalize(x_norm, xmin, xmax) np.testing.assert_almost_equal(x, x_unnorm) mean = x.mean(axis=0) x_norm = scale_normalize(x, xmin, xmax, mean) self.assertTupleEqual(x_norm.shape, x.shape) x_unnorm = scale_unnormalize(x_norm, xmin, xmax, mean) np.testing.assert_almost_equal(x, x_unnorm)
def test_detect(self): dc = self.mc.data_container x_range = get_range(dc.x_train) # Do not load pretrained parameters squeezer = FeatureSqueezing( self.mc, clip_values=x_range, smoothing_methods=['normal', 'binary'], bit_depth=8, sigma=0.1, kernel_size=2, pretrained=False, ) squeezer.load(os.path.join('save', SQUEEZER_FILE)) # testing clean set x_test = dc.x_test pred = self.mc.predict(x_test) blocked_indices, passed_x = squeezer.detect(x_test, pred, return_passed_x=True) logger.info('blocked clean samples: %d', len(blocked_indices)) self.assertLessEqual(len(blocked_indices), len(x_test) * 0.1) self.assertEqual(len(blocked_indices) + len(passed_x), len(x_test)) # the prediction parameter should not alter the result. blocked_indices_2, passed_x = squeezer.detect(x_test, return_passed_x=True) self.assertEqual(len(blocked_indices_2) + len(passed_x), len(x_test)) # NOTE: due to the randomness of the method. The blocked indices are not guaranteed to be the same! # self.assertEquals(len(blocked_indices_2), len(blocked_indices)) # test detector using FGSM attack # NOTE: the block rate is almost 0. attack = FGSMContainer(self.mc, norm=2, eps=0.2, eps_step=0.1, targeted=False, batch_size=128) adv, y_adv, x, y = attack.generate(count=1000, use_testset=True) blocked_indices, adv_passed = squeezer.detect(adv, y_adv, return_passed_x=True) logger.info('blocked adversarial: %d', len(blocked_indices)) passed_indices = np.where( np.isin(np.arange(len(adv)), blocked_indices) == False)[0] acc_adv = self.mc.evaluate(adv_passed, y[passed_indices]) logger.info('Accuracy on passed adv. examples: %f', acc_adv)
def test_fit_save(self): x_range = get_range(self.mc.data_container.x_train) squeezer = FeatureSqueezing( self.mc, clip_values=x_range, smoothing_methods=['normal', 'binary'], bit_depth=8, sigma=0.1, kernel_size=2, pretrained=False, ) x_test = np.copy(squeezer.model_container.data_container.x_test) y_test = np.copy(squeezer.model_container.data_container.y_test) mc_normal = squeezer.get_def_model_container('normal') mc_binary = squeezer.get_def_model_container('binary') # predictions before fit # without pre-trained parameter, expecting low accuracy acc_nor_before = mc_normal.evaluate(x_test, y_test) logger.info('[Before fit] Accuracy of normal squeezer: %f', acc_nor_before) self.assertLessEqual(acc_nor_before, 0.60) acc_bin_before = mc_binary.evaluate(x_test, y_test) logger.info('[Before fit] Accuracy of binary squeezer: %f', acc_bin_before) self.assertLessEqual(acc_bin_before, 0.60) squeezer.fit(max_epochs=MAX_EPOCHS, batch_size=128) # predictions after fit acc_nor_after = mc_normal.evaluate(x_test, y_test) logger.info('[After fit] Accuracy of normal squeezer: %f', acc_nor_after) self.assertGreater(acc_nor_after, acc_nor_before) acc_bin_after = mc_binary.evaluate(x_test, y_test) logger.info('[After fit] Accuracy of binary squeezer: %f', acc_bin_after) self.assertGreater(acc_bin_after, acc_bin_before) # save parameters and check the existence of the files squeezer.save(SQUEEZER_FILE, True) self.assertTrue( os.path.exists( os.path.join('save', 'test', 'IrisNN_Iris_e200_normal.pt'))) self.assertTrue( os.path.exists( os.path.join('save', 'test', 'IrisNN_Iris_e200_binary.pt')))
def test_carlini_l2_attack(self): clip_values = get_range(self.dc.x_train) # Lower the upper bound of `c_range` will reduce the norm of perturbation. attack = attacks.CarliniL2V2Container(self.mc, learning_rate=0.01, binary_search_steps=9, max_iter=1000, confidence=0.0, initial_const=0.01, c_range=(0, 1e4), batch_size=16, clip_values=clip_values) blocked_indices, adv_success_rate = self.preform_attack(attack, count=NUM_ADV) block_rate = len(blocked_indices) / NUM_ADV self.assertGreater(block_rate, adv_success_rate * 0.6) logger.info('[%s] Block rate: %f', attack.__class__.__name__, block_rate)
def test_carlini(self): clip_values = get_range(self.dc.x_train) # Lower the upper bound of `c_range` will reduce the norm of perturbation. attack = attacks.CarliniL2V2Container( self.mc, learning_rate=0.01, binary_search_steps=9, max_iter=1000, confidence=0.0, initial_const=0.01, c_range=(0, 1e4), batch_size=16, clip_values=clip_values ) adv, y_adv, x_clean, y_clean = attack.generate(count=NUM_ADV) # At least made some change from clean images self.assertFalse((adv == x_clean).all()) # test accuracy accuracy = self.mc.evaluate(adv, y_clean) logger.info('Accuracy on adv. examples: %f', accuracy) self.assertLessEqual(accuracy, 0.4) # test success rate success_rate = (y_adv != y_clean).sum() / len(y_adv) logger.info('Success rate of adv. attack: %f', success_rate) self.assertGreaterEqual(success_rate, 0.6) # sum success rate (missclassified) and accuracy (correctly classified) self.assertAlmostEqual(success_rate + accuracy, 1.0, places=4) # Check the max perturbation dif = np.max(np.abs(adv - x_clean)) logger.info('Max perturbation (L1-norm): %f', dif) self.assertLessEqual(dif, 1.0 + 1e-4) # Check bounding box self.assertLessEqual(np.max(adv), 1.0 + 1e-4) self.assertGreaterEqual(np.min(adv), 0 - 1e-4) l2 = np.max(get_l2_norm(adv, x_clean)) logger.info('L2 norm = %f', l2)
def test_fit_save(self): x_range = get_range(self.mc.data_container.x_train) squeezer = FeatureSqueezing( self.mc, clip_values=x_range, smoothing_methods=SQUEEZING_METHODS, bit_depth=8, sigma=0.1, kernel_size=3, pretrained=False, ) squeezer.fit(max_epochs=MAX_EPOCHS, batch_size=128) # save parameters and check the existence of the files squeezer.save(SQUEEZER_FILE, True) self.assertTrue( os.path.exists( os.path.join('save', 'test', 'MnistCnnV2_MNIST_e50_median.pt')))
def main(): master_seed(SEED) dataset = DATASET_LIST[DATA_NAME] dc = DataContainer(dataset, get_data_path()) dc(shuffle=True, normalize=True) print('# of trainset: {}, # of testset: {}'.format(len(dc.x_train), len(dc.x_test))) num_classes = dc.num_classes num_features = dc.dim_data[0] model = IrisNN(num_features=num_features, hidden_nodes=num_features * 4, num_classes=num_classes) mc = ModelContainerPT(model, dc) mc.load(MODEL_FILE) accuracy = mc.evaluate(dc.x_test, dc.y_test) print('Accuracy on test set: {}'.format(accuracy)) clip_values = get_range(dc.x_train, is_image=False) print('clip_values', clip_values) attack = CarliniL2V2Container(mc, targeted=False, learning_rate=0.01, binary_search_steps=9, max_iter=1000, confidence=0.0, initial_const=0.01, c_range=(0, 1e4), batch_size=16, clip_values=clip_values) adv, y_adv, x_clean, y_clean = attack.generate(count=100) l2 = np.mean(get_l2_norm(adv, x_clean)) print('L2 norm: {}'.format(l2)) not_match = y_adv != y_clean success_rate = len(not_match[not_match == True]) / len(adv) print('Success rate: {}'.format(success_rate)) accuracy = mc.evaluate(adv, y_clean) print('Accuracy on adv. examples: {}'.format(accuracy))
def test_detect(self): dc = self.mc.data_container x_range = get_range(dc.x_train) # Do not load pretrained parameters squeezer = FeatureSqueezing( self.mc, clip_values=x_range, smoothing_methods=SQUEEZING_METHODS, bit_depth=8, sigma=0.1, kernel_size=3, pretrained=False, ) squeezer.load(os.path.join('save', SQUEEZER_FILE)) # testing clean set x_test = dc.x_test pred = self.mc.predict(x_test) blocked_indices, passed_x = squeezer.detect(x_test, pred, return_passed_x=True) logger.info('blocked clean samples: %d', len(blocked_indices)) self.assertLessEqual(len(blocked_indices), len(x_test) * 0.1) self.assertEqual(len(blocked_indices) + len(passed_x), len(x_test)) # the prediction parameter should not alter the result. blocked_indices_2 = squeezer.detect(x_test, return_passed_x=False) self.assertTrue((blocked_indices_2 == blocked_indices).all()) # test detector using FGSM attack # NOTE: the block rate is almost 0. attack = FGSMContainer(self.mc, norm=np.inf, eps=0.3, eps_step=0.1, targeted=False, batch_size=128) adv, y_adv, x, y = attack.generate(count=1000, use_testset=True) blocked_indices = squeezer.detect(adv, y_adv, return_passed_x=False) logger.info('blocked adversarial: %d', len(blocked_indices)) self.assertGreater(len(blocked_indices), 500)
def test_data(self): x_train = np.copy(self.mc.data_container.x_train) y_train = np.copy(self.mc.data_container.y_train) x_range = get_range(self.mc.data_container.x_train) squeezer = FeatureSqueezing( self.mc, clip_values=x_range, smoothing_methods=SQUEEZING_METHODS, bit_depth=8, sigma=0.1, kernel_size=3, pretrained=True, ) # expecting test set and train set have not been altered. res = np.all( squeezer.model_container.data_container.x_train == x_train) self.assertTrue(res) res = np.all( squeezer.model_container.data_container.y_train == y_train) self.assertTrue(res) x_test = np.copy(squeezer.model_container.data_container.x_test) y_test = np.copy(squeezer.model_container.data_container.y_test) x_train = np.copy(squeezer.model_container.data_container.x_train) y_train = np.copy(squeezer.model_container.data_container.y_train) squeezer.fit(max_epochs=2, batch_size=128) # expecting test set and train set have not been altered. res = np.all(squeezer.model_container.data_container.x_test == x_test) self.assertTrue(res) res = np.all(squeezer.model_container.data_container.y_test == y_test) self.assertTrue(res) res = np.all( squeezer.model_container.data_container.x_train == x_train) self.assertTrue(res) res = np.all( squeezer.model_container.data_container.y_train == y_train) self.assertTrue(res)
def test_squeezing_data(self): x_range = get_range(self.mc.data_container.x_train) x_train = np.copy(self.mc.data_container.x_train) squeezer = FeatureSqueezing( self.mc, clip_values=x_range, smoothing_methods=SQUEEZING_METHODS, bit_depth=8, sigma=0.1, kernel_size=3, pretrained=False, ) # Expecting difference between input data and squeezed data mc_binary = squeezer.get_def_model_container(SQUEEZING_METHODS[0]) mc_median = squeezer.get_def_model_container(SQUEEZING_METHODS[1]) mc_normal = squeezer.get_def_model_container(SQUEEZING_METHODS[2]) self.assertFalse((x_train == mc_binary.data_container.x_train).all()) self.assertFalse((x_train == mc_median.data_container.x_train).all()) self.assertFalse((x_train == mc_normal.data_container.x_train).all()) # maximum perturbation l2 = np.max(get_l2_norm(x_train, mc_binary.data_container.x_train)) logger.info('L2 norm of binary squeezer:%f', l2) self.assertLessEqual(l2, 0.5) # average perturbation # No upper bound on median filter l2 = np.mean(get_l2_norm(x_train, mc_median.data_container.x_train)) logger.info('L2 norm of median squeezer:%f', l2) self.assertLessEqual(l2, 2.0) # average perturbation l2 = np.mean(get_l2_norm(x_train, mc_normal.data_container.x_train)) logger.info('L2 norm of normal squeezer:%f', l2) self.assertLessEqual(l2, 2.0)
def test_fit_save(self): x_range = get_range(self.mc.data_container.x_train) squeezer = FeatureSqueezing( self.mc, clip_values=x_range, smoothing_methods=SQUEEZING_METHODS, bit_depth=8, sigma=0.1, kernel_size=3, pretrained=False, ) x_test = np.copy(squeezer.model_container.data_container.x_test) y_test = np.copy(squeezer.model_container.data_container.y_test) mc_binary = squeezer.get_def_model_container(SQUEEZING_METHODS[0]) mc_median = squeezer.get_def_model_container(SQUEEZING_METHODS[1]) mc_normal = squeezer.get_def_model_container(SQUEEZING_METHODS[2]) # predictions before fit # without pre-trained parameter, expecting lower accuracy acc_bin_before = mc_binary.evaluate( squeezer.apply_binary_transform(x_test), y_test) logger.info('[Before fit] Accuracy of binary squeezer: %f', acc_bin_before) self.assertLessEqual(acc_bin_before, 0.80) acc_med_before = mc_median.evaluate( squeezer.apply_median_transform(x_test), y_test) logger.info('[Before fit] Accuracy of median squeezer: %f', acc_bin_before) self.assertLessEqual(acc_med_before, 0.80) acc_nor_before = mc_normal.evaluate( squeezer.apply_normal_transform(x_test), y_test) logger.info('[Before fit] Accuracy of normal squeezer: %f', acc_nor_before) self.assertLessEqual(acc_nor_before, 0.80) squeezer.fit(max_epochs=MAX_EPOCHS, batch_size=128) # predictions after fit acc_bin_after = mc_binary.evaluate( squeezer.apply_binary_transform(x_test), y_test) logger.info('[After fit] Accuracy of binary squeezer: %f', acc_bin_after) self.assertGreater(acc_bin_after, acc_bin_before) acc_med_after = mc_median.evaluate( squeezer.apply_median_transform(x_test), y_test) logger.info('[After fit] Accuracy of median squeezer: %f', acc_bin_after) self.assertGreater(acc_med_after, acc_med_before) acc_nor_after = mc_normal.evaluate( squeezer.apply_normal_transform(x_test), y_test) logger.info('[After fit] Accuracy of normal squeezer: %f', acc_nor_after) self.assertGreater(acc_nor_after, acc_nor_before) # save parameters and check the existence of the files squeezer.save(SQUEEZER_FILE, True) self.assertTrue( os.path.exists( os.path.join('save', 'test', 'MnistCnnV2_MNIST_e50_binary.pt'))) self.assertTrue( os.path.exists( os.path.join('save', 'test', 'MnistCnnV2_MNIST_e50_median.pt'))) self.assertTrue( os.path.exists( os.path.join('save', 'test', 'MnistCnnV2_MNIST_e50_normal.pt')))