def test_get_range(self):
x = np.array([[2, 2], [0, 0], [1, 1]], dtype=np.float32)
out = get_range(x)
e = ([0., 0.], [2., 2.])
r = [all(xx == ee) for xx, ee in zip(out, e)]
self.assertTrue(all(r))
master_seed(SEED)
x = np.random.rand(10, 1, 28, 28)
out = np.array(list(get_range(x, is_image=True)))
np.testing.assert_almost_equal(out, [0., 0.9998], decimal=4)
def test_load(self):
x_range = get_range(self.mc.data_container.x_train)
# Do not load pretrained parameters
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=False,
)
squeezer.load(os.path.join('save', SQUEEZER_FILE))
mc_binary = squeezer.get_def_model_container(SQUEEZING_METHODS[0])
mc_median = squeezer.get_def_model_container(SQUEEZING_METHODS[1])
mc_normal = squeezer.get_def_model_container(SQUEEZING_METHODS[2])
x_test = squeezer.model_container.data_container.x_test
y_test = squeezer.model_container.data_container.y_test
acc_bin_after = mc_binary.evaluate(x_test, y_test)
logger.info('For binary squeezer, accuracy after load parameters: %f',
acc_bin_after)
self.assertGreater(acc_bin_after, 0.90)
acc_med_after = mc_median.evaluate(x_test, y_test)
logger.info('For median squeezer, accuracy after load parameters: %f',
acc_med_after)
self.assertGreater(acc_med_after, 0.90)
acc_nor_after = mc_normal.evaluate(x_test, y_test)
logger.info('For normal squeezer, accuracy after load parameters: %f',
acc_nor_after)
self.assertGreater(acc_nor_after, 0.90)
def test_squeezing_data(self):
x_range = get_range(self.mc.data_container.x_train)
x_train = np.copy(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=['normal', 'binary'],
bit_depth=8,
sigma=0.1,
kernel_size=2,
pretrained=False,
)
# Expecting difference between input data and squeezed data
mc_normal = squeezer.get_def_model_container('normal')
mc_binary = squeezer.get_def_model_container('binary')
self.assertFalse((x_train == mc_normal.data_container.x_train).all())
self.assertFalse((x_train == mc_binary.data_container.x_train).all())
# average perturbation
l2 = np.mean(get_l2_norm(x_train, mc_normal.data_container.x_train))
logger.info('L2 norm of normal squeezer:%f', l2)
self.assertLessEqual(l2, 0.2)
# maximum perturbation
l2 = np.max(get_l2_norm(x_train, mc_binary.data_container.x_train))
logger.info('L2 norm of binary squeezer:%f', l2)
self.assertLessEqual(l2, 0.2)
def test_normalize(self):
x = np.random.rand(2, 3)
xmin, xmax = get_range(x)
x_norm = scale_normalize(x, xmin, xmax)
x_unnorm = scale_unnormalize(x_norm, xmin, xmax)
np.testing.assert_almost_equal(x, x_unnorm)
mean = x.mean(axis=0)
x_norm = scale_normalize(x, xmin, xmax, mean)
self.assertTupleEqual(x_norm.shape, x.shape)
x_unnorm = scale_unnormalize(x_norm, xmin, xmax, mean)
np.testing.assert_almost_equal(x, x_unnorm)
def test_detect(self):
dc = self.mc.data_container
x_range = get_range(dc.x_train)
# Do not load pretrained parameters
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=['normal', 'binary'],
bit_depth=8,
sigma=0.1,
kernel_size=2,
pretrained=False,
)
squeezer.load(os.path.join('save', SQUEEZER_FILE))
# testing clean set
x_test = dc.x_test
pred = self.mc.predict(x_test)
blocked_indices, passed_x = squeezer.detect(x_test,
pred,
return_passed_x=True)
logger.info('blocked clean samples: %d', len(blocked_indices))
self.assertLessEqual(len(blocked_indices), len(x_test) * 0.1)
self.assertEqual(len(blocked_indices) + len(passed_x), len(x_test))
# the prediction parameter should not alter the result.
blocked_indices_2, passed_x = squeezer.detect(x_test,
return_passed_x=True)
self.assertEqual(len(blocked_indices_2) + len(passed_x), len(x_test))
# NOTE: due to the randomness of the method. The blocked indices are not guaranteed to be the same!
# self.assertEquals(len(blocked_indices_2), len(blocked_indices))
# test detector using FGSM attack
# NOTE: the block rate is almost 0.
attack = FGSMContainer(self.mc,
norm=2,
eps=0.2,
eps_step=0.1,
targeted=False,
batch_size=128)
adv, y_adv, x, y = attack.generate(count=1000, use_testset=True)
blocked_indices, adv_passed = squeezer.detect(adv,
y_adv,
return_passed_x=True)
logger.info('blocked adversarial: %d', len(blocked_indices))
passed_indices = np.where(
np.isin(np.arange(len(adv)), blocked_indices) == False)[0]
acc_adv = self.mc.evaluate(adv_passed, y[passed_indices])
logger.info('Accuracy on passed adv. examples: %f', acc_adv)
def test_fit_save(self):
x_range = get_range(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=['normal', 'binary'],
bit_depth=8,
sigma=0.1,
kernel_size=2,
pretrained=False,
)
x_test = np.copy(squeezer.model_container.data_container.x_test)
y_test = np.copy(squeezer.model_container.data_container.y_test)
mc_normal = squeezer.get_def_model_container('normal')
mc_binary = squeezer.get_def_model_container('binary')
# predictions before fit
# without pre-trained parameter, expecting low accuracy
acc_nor_before = mc_normal.evaluate(x_test, y_test)
logger.info('[Before fit] Accuracy of normal squeezer: %f',
acc_nor_before)
self.assertLessEqual(acc_nor_before, 0.60)
acc_bin_before = mc_binary.evaluate(x_test, y_test)
logger.info('[Before fit] Accuracy of binary squeezer: %f',
acc_bin_before)
self.assertLessEqual(acc_bin_before, 0.60)
squeezer.fit(max_epochs=MAX_EPOCHS, batch_size=128)
# predictions after fit
acc_nor_after = mc_normal.evaluate(x_test, y_test)
logger.info('[After fit] Accuracy of normal squeezer: %f',
acc_nor_after)
self.assertGreater(acc_nor_after, acc_nor_before)
acc_bin_after = mc_binary.evaluate(x_test, y_test)
logger.info('[After fit] Accuracy of binary squeezer: %f',
acc_bin_after)
self.assertGreater(acc_bin_after, acc_bin_before)
# save parameters and check the existence of the files
squeezer.save(SQUEEZER_FILE, True)
self.assertTrue(
os.path.exists(
os.path.join('save', 'test', 'IrisNN_Iris_e200_normal.pt')))
self.assertTrue(
os.path.exists(
os.path.join('save', 'test', 'IrisNN_Iris_e200_binary.pt')))
def test_carlini_l2_attack(self):
clip_values = get_range(self.dc.x_train)
# Lower the upper bound of `c_range` will reduce the norm of perturbation.
attack = attacks.CarliniL2V2Container(self.mc,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e4),
batch_size=16,
clip_values=clip_values)
blocked_indices, adv_success_rate = self.preform_attack(attack,
count=NUM_ADV)
block_rate = len(blocked_indices) / NUM_ADV
self.assertGreater(block_rate, adv_success_rate * 0.6)
logger.info('[%s] Block rate: %f', attack.__class__.__name__,
block_rate)
def test_carlini(self):
clip_values = get_range(self.dc.x_train)
# Lower the upper bound of `c_range` will reduce the norm of perturbation.
attack = attacks.CarliniL2V2Container(
self.mc,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e4),
batch_size=16,
clip_values=clip_values
)
adv, y_adv, x_clean, y_clean = attack.generate(count=NUM_ADV)
# At least made some change from clean images
self.assertFalse((adv == x_clean).all())
# test accuracy
accuracy = self.mc.evaluate(adv, y_clean)
logger.info('Accuracy on adv. examples: %f', accuracy)
self.assertLessEqual(accuracy, 0.4)
# test success rate
success_rate = (y_adv != y_clean).sum() / len(y_adv)
logger.info('Success rate of adv. attack: %f', success_rate)
self.assertGreaterEqual(success_rate, 0.6)
# sum success rate (missclassified) and accuracy (correctly classified)
self.assertAlmostEqual(success_rate + accuracy, 1.0, places=4)
# Check the max perturbation
dif = np.max(np.abs(adv - x_clean))
logger.info('Max perturbation (L1-norm): %f', dif)
self.assertLessEqual(dif, 1.0 + 1e-4)
# Check bounding box
self.assertLessEqual(np.max(adv), 1.0 + 1e-4)
self.assertGreaterEqual(np.min(adv), 0 - 1e-4)
l2 = np.max(get_l2_norm(adv, x_clean))
logger.info('L2 norm = %f', l2)
def test_fit_save(self):
x_range = get_range(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=False,
)
squeezer.fit(max_epochs=MAX_EPOCHS, batch_size=128)
# save parameters and check the existence of the files
squeezer.save(SQUEEZER_FILE, True)
self.assertTrue(
os.path.exists(
os.path.join('save', 'test',
'MnistCnnV2_MNIST_e50_median.pt')))
def main():
master_seed(SEED)
dataset = DATASET_LIST[DATA_NAME]
dc = DataContainer(dataset, get_data_path())
dc(shuffle=True, normalize=True)
print('# of trainset: {}, # of testset: {}'.format(len(dc.x_train),
len(dc.x_test)))
num_classes = dc.num_classes
num_features = dc.dim_data[0]
model = IrisNN(num_features=num_features,
hidden_nodes=num_features * 4,
num_classes=num_classes)
mc = ModelContainerPT(model, dc)
mc.load(MODEL_FILE)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
print('Accuracy on test set: {}'.format(accuracy))
clip_values = get_range(dc.x_train, is_image=False)
print('clip_values', clip_values)
attack = CarliniL2V2Container(mc,
targeted=False,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e4),
batch_size=16,
clip_values=clip_values)
adv, y_adv, x_clean, y_clean = attack.generate(count=100)
l2 = np.mean(get_l2_norm(adv, x_clean))
print('L2 norm: {}'.format(l2))
not_match = y_adv != y_clean
success_rate = len(not_match[not_match == True]) / len(adv)
print('Success rate: {}'.format(success_rate))
accuracy = mc.evaluate(adv, y_clean)
print('Accuracy on adv. examples: {}'.format(accuracy))
def test_detect(self):
dc = self.mc.data_container
x_range = get_range(dc.x_train)
# Do not load pretrained parameters
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=False,
)
squeezer.load(os.path.join('save', SQUEEZER_FILE))
# testing clean set
x_test = dc.x_test
pred = self.mc.predict(x_test)
blocked_indices, passed_x = squeezer.detect(x_test,
pred,
return_passed_x=True)
logger.info('blocked clean samples: %d', len(blocked_indices))
self.assertLessEqual(len(blocked_indices), len(x_test) * 0.1)
self.assertEqual(len(blocked_indices) + len(passed_x), len(x_test))
# the prediction parameter should not alter the result.
blocked_indices_2 = squeezer.detect(x_test, return_passed_x=False)
self.assertTrue((blocked_indices_2 == blocked_indices).all())
# test detector using FGSM attack
# NOTE: the block rate is almost 0.
attack = FGSMContainer(self.mc,
norm=np.inf,
eps=0.3,
eps_step=0.1,
targeted=False,
batch_size=128)
adv, y_adv, x, y = attack.generate(count=1000, use_testset=True)
blocked_indices = squeezer.detect(adv, y_adv, return_passed_x=False)
logger.info('blocked adversarial: %d', len(blocked_indices))
self.assertGreater(len(blocked_indices), 500)
def test_data(self):
x_train = np.copy(self.mc.data_container.x_train)
y_train = np.copy(self.mc.data_container.y_train)
x_range = get_range(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=True,
)
# expecting test set and train set have not been altered.
res = np.all(
squeezer.model_container.data_container.x_train == x_train)
self.assertTrue(res)
res = np.all(
squeezer.model_container.data_container.y_train == y_train)
self.assertTrue(res)
x_test = np.copy(squeezer.model_container.data_container.x_test)
y_test = np.copy(squeezer.model_container.data_container.y_test)
x_train = np.copy(squeezer.model_container.data_container.x_train)
y_train = np.copy(squeezer.model_container.data_container.y_train)
squeezer.fit(max_epochs=2, batch_size=128)
# expecting test set and train set have not been altered.
res = np.all(squeezer.model_container.data_container.x_test == x_test)
self.assertTrue(res)
res = np.all(squeezer.model_container.data_container.y_test == y_test)
self.assertTrue(res)
res = np.all(
squeezer.model_container.data_container.x_train == x_train)
self.assertTrue(res)
res = np.all(
squeezer.model_container.data_container.y_train == y_train)
self.assertTrue(res)
def test_squeezing_data(self):
x_range = get_range(self.mc.data_container.x_train)
x_train = np.copy(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=False,
)
# Expecting difference between input data and squeezed data
mc_binary = squeezer.get_def_model_container(SQUEEZING_METHODS[0])
mc_median = squeezer.get_def_model_container(SQUEEZING_METHODS[1])
mc_normal = squeezer.get_def_model_container(SQUEEZING_METHODS[2])
self.assertFalse((x_train == mc_binary.data_container.x_train).all())
self.assertFalse((x_train == mc_median.data_container.x_train).all())
self.assertFalse((x_train == mc_normal.data_container.x_train).all())
# maximum perturbation
l2 = np.max(get_l2_norm(x_train, mc_binary.data_container.x_train))
logger.info('L2 norm of binary squeezer:%f', l2)
self.assertLessEqual(l2, 0.5)
# average perturbation
# No upper bound on median filter
l2 = np.mean(get_l2_norm(x_train, mc_median.data_container.x_train))
logger.info('L2 norm of median squeezer:%f', l2)
self.assertLessEqual(l2, 2.0)
# average perturbation
l2 = np.mean(get_l2_norm(x_train, mc_normal.data_container.x_train))
logger.info('L2 norm of normal squeezer:%f', l2)
self.assertLessEqual(l2, 2.0)
def test_fit_save(self):
x_range = get_range(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=False,
)
x_test = np.copy(squeezer.model_container.data_container.x_test)
y_test = np.copy(squeezer.model_container.data_container.y_test)
mc_binary = squeezer.get_def_model_container(SQUEEZING_METHODS[0])
mc_median = squeezer.get_def_model_container(SQUEEZING_METHODS[1])
mc_normal = squeezer.get_def_model_container(SQUEEZING_METHODS[2])
# predictions before fit
# without pre-trained parameter, expecting lower accuracy
acc_bin_before = mc_binary.evaluate(
squeezer.apply_binary_transform(x_test), y_test)
logger.info('[Before fit] Accuracy of binary squeezer: %f',
acc_bin_before)
self.assertLessEqual(acc_bin_before, 0.80)
acc_med_before = mc_median.evaluate(
squeezer.apply_median_transform(x_test), y_test)
logger.info('[Before fit] Accuracy of median squeezer: %f',
acc_bin_before)
self.assertLessEqual(acc_med_before, 0.80)
acc_nor_before = mc_normal.evaluate(
squeezer.apply_normal_transform(x_test), y_test)
logger.info('[Before fit] Accuracy of normal squeezer: %f',
acc_nor_before)
self.assertLessEqual(acc_nor_before, 0.80)
squeezer.fit(max_epochs=MAX_EPOCHS, batch_size=128)
# predictions after fit
acc_bin_after = mc_binary.evaluate(
squeezer.apply_binary_transform(x_test), y_test)
logger.info('[After fit] Accuracy of binary squeezer: %f',
acc_bin_after)
self.assertGreater(acc_bin_after, acc_bin_before)
acc_med_after = mc_median.evaluate(
squeezer.apply_median_transform(x_test), y_test)
logger.info('[After fit] Accuracy of median squeezer: %f',
acc_bin_after)
self.assertGreater(acc_med_after, acc_med_before)
acc_nor_after = mc_normal.evaluate(
squeezer.apply_normal_transform(x_test), y_test)
logger.info('[After fit] Accuracy of normal squeezer: %f',
acc_nor_after)
self.assertGreater(acc_nor_after, acc_nor_before)
# save parameters and check the existence of the files
squeezer.save(SQUEEZER_FILE, True)
self.assertTrue(
os.path.exists(
os.path.join('save', 'test',
'MnistCnnV2_MNIST_e50_binary.pt')))
self.assertTrue(
os.path.exists(
os.path.join('save', 'test',
'MnistCnnV2_MNIST_e50_median.pt')))
self.assertTrue(
os.path.exists(
os.path.join('save', 'test',
'MnistCnnV2_MNIST_e50_normal.pt')))