def test_squeezing_data(self):
x_range = get_range(self.mc.data_container.x_train)
x_train = np.copy(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=['normal', 'binary'],
bit_depth=8,
sigma=0.1,
kernel_size=2,
pretrained=False,
)
# Expecting difference between input data and squeezed data
mc_normal = squeezer.get_def_model_container('normal')
mc_binary = squeezer.get_def_model_container('binary')
self.assertFalse((x_train == mc_normal.data_container.x_train).all())
self.assertFalse((x_train == mc_binary.data_container.x_train).all())
# average perturbation
l2 = np.mean(get_l2_norm(x_train, mc_normal.data_container.x_train))
logger.info('L2 norm of normal squeezer:%f', l2)
self.assertLessEqual(l2, 0.2)
# maximum perturbation
l2 = np.max(get_l2_norm(x_train, mc_binary.data_container.x_train))
logger.info('L2 norm of binary squeezer:%f', l2)
self.assertLessEqual(l2, 0.2)
def test_load(self):
x_range = get_range(self.mc.data_container.x_train)
# Do not load pretrained parameters
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=['normal', 'binary'],
bit_depth=8,
sigma=0.1,
kernel_size=2,
pretrained=False,
)
squeezer.load(os.path.join('save', SQUEEZER_FILE))
mc_normal = squeezer.get_def_model_container('normal')
mc_binary = squeezer.get_def_model_container('binary')
x_test = squeezer.model_container.data_container.x_test
y_test = squeezer.model_container.data_container.y_test
acc_nor_after = mc_normal.evaluate(x_test, y_test)
logger.info('For normal squeezer, accuracy after load parameters: %f',
acc_nor_after)
self.assertGreater(acc_nor_after, 0.90)
acc_bin_after = mc_binary.evaluate(x_test, y_test)
logger.info('For binary squeezer, accuracy after load parameters: %f',
acc_bin_after)
self.assertGreater(acc_bin_after, 0.90)
def test_data(self):
x_train = np.copy(self.mc.data_container.x_train)
y_train = np.copy(self.mc.data_container.y_train)
x_range = get_range(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=True,
)
# expecting test set and train set have not been altered.
res = np.all(
squeezer.model_container.data_container.x_train == x_train)
self.assertTrue(res)
res = np.all(
squeezer.model_container.data_container.y_train == y_train)
self.assertTrue(res)
x_test = np.copy(squeezer.model_container.data_container.x_test)
y_test = np.copy(squeezer.model_container.data_container.y_test)
x_train = np.copy(squeezer.model_container.data_container.x_train)
y_train = np.copy(squeezer.model_container.data_container.y_train)
squeezer.fit(max_epochs=2, batch_size=128)
# expecting test set and train set have not been altered.
res = np.all(squeezer.model_container.data_container.x_test == x_test)
self.assertTrue(res)
res = np.all(squeezer.model_container.data_container.y_test == y_test)
self.assertTrue(res)
res = np.all(
squeezer.model_container.data_container.x_train == x_train)
self.assertTrue(res)
res = np.all(
squeezer.model_container.data_container.y_train == y_train)
self.assertTrue(res)
def test_squeezing_data(self):
x_range = get_range(self.mc.data_container.x_train)
x_train = np.copy(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=False,
)
# Expecting difference between input data and squeezed data
mc_binary = squeezer.get_def_model_container(SQUEEZING_METHODS[0])
mc_median = squeezer.get_def_model_container(SQUEEZING_METHODS[1])
mc_normal = squeezer.get_def_model_container(SQUEEZING_METHODS[2])
self.assertFalse((x_train == mc_binary.data_container.x_train).all())
self.assertFalse((x_train == mc_median.data_container.x_train).all())
self.assertFalse((x_train == mc_normal.data_container.x_train).all())
# maximum perturbation
l2 = np.max(get_l2_norm(x_train, mc_binary.data_container.x_train))
logger.info('L2 norm of binary squeezer:%f', l2)
self.assertLessEqual(l2, 0.5)
# average perturbation
# No upper bound on median filter
l2 = np.mean(get_l2_norm(x_train, mc_median.data_container.x_train))
logger.info('L2 norm of median squeezer:%f', l2)
self.assertLessEqual(l2, 2.0)
# average perturbation
l2 = np.mean(get_l2_norm(x_train, mc_normal.data_container.x_train))
logger.info('L2 norm of normal squeezer:%f', l2)
self.assertLessEqual(l2, 2.0)
def test_load(self):
x_range = get_range(self.mc.data_container.x_train)
# Do not load pretrained parameters
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=False,
)
squeezer.load(os.path.join('save', SQUEEZER_FILE))
mc_median = squeezer.get_def_model_container(SQUEEZING_METHODS[0])
x_test = squeezer.model_container.data_container.x_test
y_test = squeezer.model_container.data_container.y_test
acc_med_after = mc_median.evaluate(x_test, y_test)
logger.info('For median squeezer, accuracy after load parameters: %f',
acc_med_after)
self.assertGreater(acc_med_after, 0.90)
def test_detect(self):
dc = self.mc.data_container
x_range = get_range(dc.x_train)
# Do not load pretrained parameters
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=['normal', 'binary'],
bit_depth=8,
sigma=0.1,
kernel_size=2,
pretrained=False,
)
squeezer.load(os.path.join('save', SQUEEZER_FILE))
# testing clean set
x_test = dc.x_test
pred = self.mc.predict(x_test)
blocked_indices, passed_x = squeezer.detect(x_test,
pred,
return_passed_x=True)
logger.info('blocked clean samples: %d', len(blocked_indices))
self.assertLessEqual(len(blocked_indices), len(x_test) * 0.1)
self.assertEqual(len(blocked_indices) + len(passed_x), len(x_test))
# the prediction parameter should not alter the result.
blocked_indices_2, passed_x = squeezer.detect(x_test,
return_passed_x=True)
self.assertEqual(len(blocked_indices_2) + len(passed_x), len(x_test))
# NOTE: due to the randomness of the method. The blocked indices are not guaranteed to be the same!
# self.assertEquals(len(blocked_indices_2), len(blocked_indices))
# test detector using FGSM attack
# NOTE: the block rate is almost 0.
attack = FGSMContainer(self.mc,
norm=2,
eps=0.2,
eps_step=0.1,
targeted=False,
batch_size=128)
adv, y_adv, x, y = attack.generate(count=1000, use_testset=True)
blocked_indices, adv_passed = squeezer.detect(adv,
y_adv,
return_passed_x=True)
logger.info('blocked adversarial: %d', len(blocked_indices))
passed_indices = np.where(
np.isin(np.arange(len(adv)), blocked_indices) == False)[0]
acc_adv = self.mc.evaluate(adv_passed, y[passed_indices])
logger.info('Accuracy on passed adv. examples: %f', acc_adv)
def test_fit_save(self):
x_range = get_range(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=['normal', 'binary'],
bit_depth=8,
sigma=0.1,
kernel_size=2,
pretrained=False,
)
x_test = np.copy(squeezer.model_container.data_container.x_test)
y_test = np.copy(squeezer.model_container.data_container.y_test)
mc_normal = squeezer.get_def_model_container('normal')
mc_binary = squeezer.get_def_model_container('binary')
# predictions before fit
# without pre-trained parameter, expecting low accuracy
acc_nor_before = mc_normal.evaluate(x_test, y_test)
logger.info('[Before fit] Accuracy of normal squeezer: %f',
acc_nor_before)
self.assertLessEqual(acc_nor_before, 0.60)
acc_bin_before = mc_binary.evaluate(x_test, y_test)
logger.info('[Before fit] Accuracy of binary squeezer: %f',
acc_bin_before)
self.assertLessEqual(acc_bin_before, 0.60)
squeezer.fit(max_epochs=MAX_EPOCHS, batch_size=128)
# predictions after fit
acc_nor_after = mc_normal.evaluate(x_test, y_test)
logger.info('[After fit] Accuracy of normal squeezer: %f',
acc_nor_after)
self.assertGreater(acc_nor_after, acc_nor_before)
acc_bin_after = mc_binary.evaluate(x_test, y_test)
logger.info('[After fit] Accuracy of binary squeezer: %f',
acc_bin_after)
self.assertGreater(acc_bin_after, acc_bin_before)
# save parameters and check the existence of the files
squeezer.save(SQUEEZER_FILE, True)
self.assertTrue(
os.path.exists(
os.path.join('save', 'test', 'IrisNN_Iris_e200_normal.pt')))
self.assertTrue(
os.path.exists(
os.path.join('save', 'test', 'IrisNN_Iris_e200_binary.pt')))
def test_detect(self):
dc = self.mc.data_container
x_range = get_range(dc.x_train)
# Do not load pretrained parameters
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=False,
)
squeezer.load(os.path.join('save', SQUEEZER_FILE))
# testing clean set
x_test = dc.x_test
pred = self.mc.predict(x_test)
blocked_indices, passed_x = squeezer.detect(x_test,
pred,
return_passed_x=True)
logger.info('blocked clean samples: %d', len(blocked_indices))
self.assertLessEqual(len(blocked_indices), len(x_test) * 0.1)
self.assertEqual(len(blocked_indices) + len(passed_x), len(x_test))
# the prediction parameter should not alter the result.
blocked_indices_2 = squeezer.detect(x_test, return_passed_x=False)
self.assertTrue((blocked_indices_2 == blocked_indices).all())
# test detector using FGSM attack
# NOTE: the block rate is almost 0.
attack = FGSMContainer(self.mc,
norm=np.inf,
eps=0.3,
eps_step=0.1,
targeted=False,
batch_size=128)
adv, y_adv, x, y = attack.generate(count=1000, use_testset=True)
blocked_indices = squeezer.detect(adv, y_adv, return_passed_x=False)
logger.info('blocked adversarial: %d', len(blocked_indices))
self.assertGreater(len(blocked_indices), 500)
def test_fit_save(self):
x_range = get_range(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=False,
)
squeezer.fit(max_epochs=MAX_EPOCHS, batch_size=128)
# save parameters and check the existence of the files
squeezer.save(SQUEEZER_FILE, True)
self.assertTrue(
os.path.exists(
os.path.join('save', 'test',
'MnistCnnV2_MNIST_e50_median.pt')))
def test_fit_save(self):
x_range = get_range(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=False,
)
x_test = np.copy(squeezer.model_container.data_container.x_test)
y_test = np.copy(squeezer.model_container.data_container.y_test)
mc_binary = squeezer.get_def_model_container(SQUEEZING_METHODS[0])
mc_median = squeezer.get_def_model_container(SQUEEZING_METHODS[1])
mc_normal = squeezer.get_def_model_container(SQUEEZING_METHODS[2])
# predictions before fit
# without pre-trained parameter, expecting lower accuracy
acc_bin_before = mc_binary.evaluate(
squeezer.apply_binary_transform(x_test), y_test)
logger.info('[Before fit] Accuracy of binary squeezer: %f',
acc_bin_before)
self.assertLessEqual(acc_bin_before, 0.80)
acc_med_before = mc_median.evaluate(
squeezer.apply_median_transform(x_test), y_test)
logger.info('[Before fit] Accuracy of median squeezer: %f',
acc_bin_before)
self.assertLessEqual(acc_med_before, 0.80)
acc_nor_before = mc_normal.evaluate(
squeezer.apply_normal_transform(x_test), y_test)
logger.info('[Before fit] Accuracy of normal squeezer: %f',
acc_nor_before)
self.assertLessEqual(acc_nor_before, 0.80)
squeezer.fit(max_epochs=MAX_EPOCHS, batch_size=128)
# predictions after fit
acc_bin_after = mc_binary.evaluate(
squeezer.apply_binary_transform(x_test), y_test)
logger.info('[After fit] Accuracy of binary squeezer: %f',
acc_bin_after)
self.assertGreater(acc_bin_after, acc_bin_before)
acc_med_after = mc_median.evaluate(
squeezer.apply_median_transform(x_test), y_test)
logger.info('[After fit] Accuracy of median squeezer: %f',
acc_bin_after)
self.assertGreater(acc_med_after, acc_med_before)
acc_nor_after = mc_normal.evaluate(
squeezer.apply_normal_transform(x_test), y_test)
logger.info('[After fit] Accuracy of normal squeezer: %f',
acc_nor_after)
self.assertGreater(acc_nor_after, acc_nor_before)
# save parameters and check the existence of the files
squeezer.save(SQUEEZER_FILE, True)
self.assertTrue(
os.path.exists(
os.path.join('save', 'test',
'MnistCnnV2_MNIST_e50_binary.pt')))
self.assertTrue(
os.path.exists(
os.path.join('save', 'test',
'MnistCnnV2_MNIST_e50_median.pt')))
self.assertTrue(
os.path.exists(
os.path.join('save', 'test',
'MnistCnnV2_MNIST_e50_normal.pt')))
def experiment(index, data_container, max_epochs, adv_file, res_file):
# STEP 1: select data and model
dname = data_container.name
num_classes = data_container.num_classes
num_features = data_container.dim_data[0]
model = IrisNN(num_features=num_features,
hidden_nodes=num_features * 4,
num_classes=num_classes)
distill_model = IrisNN(num_features=num_features,
hidden_nodes=num_features * 4,
num_classes=num_classes)
# STEP 2: train models
mc = ModelContainerPT(model, data_container)
mc.fit(max_epochs=max_epochs, batch_size=BATCH_SIZE)
accuracy = mc.evaluate(data_container.x_test, data_container.y_test)
logger.info('Accuracy on test set: %f', accuracy)
adv_res = [accuracy]
# STEP 3: generate adversarial examples
# no more than 1000 samples are required
x = data_container.x_test
y = data_container.y_test
# The test set has fixed size, 1000.
if len(x) > 1000:
x = x[:1000]
y = y[:1000]
accuracy = mc.evaluate(x, y)
adv_res.append(accuracy)
advs = np.zeros((len(ATTACK_LIST), x.shape[0], x.shape[1]),
dtype=np.float32)
pred_advs = -np.ones((len(ATTACK_LIST), len(y)),
dtype=np.int32) # assign -1 as initial value
pred_clean = mc.predict(x)
advs[0] = x
pred_advs[0] = pred_clean
att_param_json = open(os.path.join(DIR_PATH, 'AttackParams.json'))
att_params = json.load(att_param_json)
for i, att_name in enumerate(ATTACK_LIST):
# Clean set is only used in evaluation phase.
if att_name == 'Clean':
continue
logger.debug('[%d]Running %s attack...', i, att_name)
kwargs = att_params[att_name]
logger.debug('%s params: %s', att_name, str(kwargs))
Attack = get_attack(att_name)
attack = Attack(mc, **kwargs)
adv, pred_adv, x_clean, pred_clean_ = attack.generate(
use_testset=False, x=x)
assert np.all(pred_clean == pred_clean_)
assert np.all(x == x_clean)
logger.info('created %d adv examples using %s from %s', len(advs[i]),
att_name, dname)
not_match = pred_adv != pred_clean
success_rate = len(not_match[not_match == True]) / len(pred_clean)
accuracy = mc.evaluate(adv, y)
advs[i] = adv
pred_advs[i] = pred_adv
logger.info('Success rate of %s: %f', att_name, success_rate)
logger.info('Accuracy on %s: %f', att_name, accuracy)
adv_res.append(accuracy)
adv_file.write(','.join([str(r) for r in adv_res]) + '\n')
# STEP 4: train defences
blocked_res = np.zeros(len(TITLE_RESULTS), dtype=np.int32)
blocked_res[0] = index
for def_name in DEFENCE_LIST:
logger.debug('Running %s...', def_name)
if def_name == 'AdvTraining':
attack = BIMContainer(mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False)
defence = AdversarialTraining(mc, [attack])
defence.fit(max_epochs=max_epochs,
batch_size=BATCH_SIZE,
ratio=ADV_TRAIN_RATIO)
block_attack(0, advs, defence, def_name, blocked_res)
elif def_name == 'Destillation':
# A single temperature is used for all sets
temp = 20
defence = DistillationContainer(mc,
distill_model,
temperature=temp,
pretrained=False)
defence.fit(max_epochs=max_epochs, batch_size=BATCH_SIZE)
block_attack(1, advs, defence, def_name, blocked_res)
elif def_name == 'Squeezing':
defence = FeatureSqueezing(
mc,
SQUEEZER_FILTER_LIST,
bit_depth=SQUEEZER_DEPTH,
sigma=SQUEEZER_SIGMA,
pretrained=True,
)
defence.fit(max_epochs=max_epochs, batch_size=BATCH_SIZE)
block_attack(2, advs, defence, def_name, blocked_res)
elif def_name == 'AD':
ad_param_file = open(AD_PARAM_FILE)
# BreastCancer uses a different set of parameters
if dname == 'BreastCancerWisconsin':
param_file = os.path.join(DIR_PATH, 'AdParamsBC.json')
ad_param_file = open(param_file)
ad_params = json.load(ad_param_file)
logger.debug('AD params: %s', str(ad_params))
defence = ApplicabilityDomainContainer(
mc, hidden_model=model.hidden_model, **ad_params)
defence.fit()
block_attack(3, advs, defence, def_name, blocked_res)
res_file.write(','.join([str(r) for r in blocked_res]) + '\n')
def experiment(index, dname, mname, max_epochs, adv_file, res_file):
# STEP 1: select data
dc = get_data_container(dname, use_shuffle=True, use_normalize=True)
Model = get_model(mname)
model = Model()
distill_model = Model()
logger.info('Selected %s model', model.__class__.__name__)
# STEP 2: train models
mc = ModelContainerPT(model, dc)
mc.fit(max_epochs=max_epochs, batch_size=BATCH_SIZE)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
adv_res = [accuracy]
# STEP 3: generate adversarial examples
# no more than 1000 samples are required
n = 1000 if len(dc.x_test) >= 1000 else len(dc.x_test)
# idx = np.random.choice(len(dc.x_test), n, replace=False)
# x = dc.x_test[idx]
# y = dc.y_test[idx]
x = dc.x_test[:n]
y = dc.y_test[:n]
accuracy = mc.evaluate(x, y)
adv_res.append(accuracy)
advs = np.zeros(
tuple([len(ATTACK_LIST)] + list(x.shape)),
dtype=np.float32)
pred_advs = -np.ones(
(len(ATTACK_LIST), n),
dtype=np.int32) # assign -1 as initial value
pred_clean = mc.predict(x)
advs[0] = x
pred_advs[0] = pred_clean
att_param_json = open(os.path.join(DIR_PATH, 'AttackParams.json'))
att_params = json.load(att_param_json)
for i, att_name in enumerate(ATTACK_LIST):
# Clean set is only used in evaluation phase.
if att_name == 'Clean':
continue
logger.debug('[%d]Running %s attack...', i, att_name)
kwargs = att_params[att_name]
logger.debug('%s params: %s', att_name, str(kwargs))
Attack = get_attack(att_name)
attack = Attack(mc, **kwargs)
adv, pred_adv, x_clean, pred_clean_ = attack.generate(
use_testset=False,
x=x)
assert np.all(pred_clean == pred_clean_)
assert np.all(x == x_clean)
logger.info('created %d adv examples using %s from %s',
len(advs[i]),
att_name,
dname)
not_match = pred_adv != pred_clean
success_rate = len(not_match[not_match == True]) / len(pred_clean)
accuracy = mc.evaluate(adv, y)
advs[i] = adv
pred_advs[i] = pred_adv
logger.info('Success rate of %s: %f', att_name, success_rate)
logger.info('Accuracy on %s: %f', att_name, accuracy)
adv_res.append(accuracy)
adv_file.write(','.join([str(r) for r in adv_res]) + '\n')
# STEP 4: train defences
blocked_res = np.zeros(len(TITLE_RESULTS), dtype=np.int32)
blocked_res[0] = index
for def_name in DEFENCE_LIST:
logger.debug('Running %s...', def_name)
if def_name == 'AdvTraining':
attack = BIMContainer(
mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False)
defence = AdversarialTraining(mc, [attack])
defence.fit(max_epochs=max_epochs,
batch_size=BATCH_SIZE,
ratio=ADV_TRAIN_RATIO)
block_attack(0, advs, defence, def_name, blocked_res)
elif def_name == 'Destillation':
defence = DistillationContainer(
mc, distill_model, temperature=DISTILL_TEMP, pretrained=False)
defence.fit(max_epochs=max_epochs, batch_size=BATCH_SIZE)
block_attack(1, advs, defence, def_name, blocked_res)
elif def_name == 'Squeezing':
defence = FeatureSqueezing(
mc,
SQUEEZER_FILTER_LIST,
bit_depth=SQUEEZER_DEPTH,
sigma=SQUEEZER_SIGMA,
kernel_size=SQUEEZER_KERNEL,
pretrained=True,
)
defence.fit(max_epochs=max_epochs, batch_size=BATCH_SIZE)
block_attack(2, advs, defence, def_name, blocked_res)
elif def_name == 'AD':
ad_param_file = open(AD_PARAM_FILE)
ad_params = json.load(ad_param_file)
logger.debug('AD params: %s', str(ad_params))
defence = ApplicabilityDomainContainer(
mc,
hidden_model=model.hidden_model,
**ad_params)
defence.fit()
block_attack(3, advs, defence, def_name, blocked_res)
res_file.write(','.join([str(r) for r in blocked_res]) + '\n')
def main():
# load dataset and initial model
model = MnistCnnV2()
dc = DataContainer(DATASET_LIST[DATASET], get_data_path())
dc(shuffle=True, normalize=True)
mc = ModelContainerPT(model, dc)
mc.load(MODEL_FILE)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
print(f'Accuracy on test set: {accuracy}')
# train or load pretrained parameters
squeezer = FeatureSqueezing(mc, ['median', 'normal', 'binary'],
bit_depth=BIT_DEPTH,
sigma=SIGMA,
kernel_size=KERNEL_SIZE,
pretrained=True)
x_test = dc.x_test
y_test = dc.y_test
mc_binary = squeezer.get_def_model_container('binary')
mc_median = squeezer.get_def_model_container('median')
mc_normal = squeezer.get_def_model_container('normal')
print('before fit')
acc_bin = mc_binary.evaluate(squeezer.apply_binary_transform(x_test),
y_test)
print(f'Accuracy of binary squeezer: {acc_bin}')
acc_med = mc_median.evaluate(squeezer.apply_median_transform(x_test),
y_test)
print(f'Accuracy of median squeezer: {acc_med}')
acc_nor = mc_normal.evaluate(squeezer.apply_normal_transform(x_test),
y_test)
print(f'Accuracy of normal squeezer: {acc_nor}')
if not squeezer.does_pretrained_exist(MODEL_FILE):
squeezer.fit(max_epochs=MAX_EPOCHS, batch_size=128)
print('after fit')
acc_bin = mc_binary.evaluate(squeezer.apply_binary_transform(x_test),
y_test)
print(f'Accuracy of binary squeezer: {acc_bin}')
acc_med = mc_median.evaluate(squeezer.apply_median_transform(x_test),
y_test)
print(f'Accuracy of median squeezer: {acc_med}')
acc_nor = mc_normal.evaluate(squeezer.apply_normal_transform(x_test),
y_test)
print(f'Accuracy of normal squeezer: {acc_nor}')
squeezer.save(MODEL_FILE, True)
print('after load')
squeezer.load(MODEL_FILE)
acc_bin = mc_binary.evaluate(squeezer.apply_binary_transform(x_test),
y_test)
print(f'Accuracy of binary squeezer: {acc_bin}')
acc_med = mc_median.evaluate(squeezer.apply_median_transform(x_test),
y_test)
print(f'Accuracy of median squeezer: {acc_med}')
acc_nor = mc_normal.evaluate(squeezer.apply_normal_transform(x_test),
y_test)
print(f'Accuracy of normal squeezer: {acc_nor}')
# load adversarial examples
adv_list = ['FGSM', 'BIM', 'DeepFool', 'Carlini', 'Saliency']
y_file = os.path.join('save',
f'{MODEL_NAME}_{DATASET}_{adv_list[0]}_y.npy')
x_file = os.path.join('save',
f'{MODEL_NAME}_{DATASET}_{adv_list[0]}_x.npy')
x = np.load(x_file, allow_pickle=False)
y = np.load(y_file, allow_pickle=False)
acc_og = mc.evaluate(x, y)
acc_squeezer = squeezer.evaluate(x, y)
print(f'Accuracy on clean set - OG: {acc_og}, Squeezer: {acc_squeezer}')
for adv_name in adv_list:
adv_file = os.path.join(
'save', build_adv_filename(MODEL_NAME, DATASET, adv_name))
adv = np.load(adv_file, allow_pickle=False)
acc_og = mc.evaluate(adv, y)
acc_squeezer = squeezer.evaluate(adv, y)
print(
f'Accuracy on {adv_name} set - OG: {acc_og}, Squeezer: {acc_squeezer}'
)
def main():
parser = ap.ArgumentParser()
parser.add_argument(
'-m',
'--model',
type=str,
required=True,
help=
'a file which contains a pretrained model. The filename should in "<model>_<dataset>_e<max epochs>[_<date>].pt" format'
)
parser.add_argument('-e',
'--epoch',
type=int,
required=True,
help='the number of max epochs for training')
parser.add_argument(
'-d',
'--depth',
type=int,
default=0,
help=
'The image color depth for input images. Apply Binary-Depth filter when receives a parameter'
)
parser.add_argument(
'--sigma',
type=float,
default=0,
help=
'The Standard Deviation of Normal distribution. Apply Gaussian Noise filter when receives a parameter'
)
parser.add_argument(
'-k',
'--kernelsize',
type=int,
default=0,
help=
'The kernel size for Median filter. Apply median filter when receives a parameter'
)
parser.add_argument('-b',
'--batchsize',
type=int,
default=128,
help='batch size')
parser.add_argument(
'-T',
'--train',
action='store_true',
default=False,
help=
'Force the model to retrain without searching existing pretrained file'
)
parser.add_argument('-s',
'--seed',
type=int,
default=4096,
help='the seed for random number generator')
parser.add_argument('-v',
'--verbose',
action='store_true',
default=False,
help='set logger level to debug')
parser.add_argument('-l',
'--savelog',
action='store_true',
default=False,
help='save logging file')
parser.add_argument('-B',
'--bim',
action='store_true',
default=False,
help='Apply BIM attack')
parser.add_argument('-C',
'--carlini',
action='store_true',
default=False,
help='Apply Carlini L2 attack')
parser.add_argument('-D',
'--deepfool',
action='store_true',
default=False,
help='Apply DeepFool attack')
parser.add_argument('-F',
'--fgsm',
action='store_true',
default=False,
help='Apply FGSM attack')
parser.add_argument('-S',
'--saliency',
action='store_true',
default=False,
help='Apply Saliency Map attack')
args = parser.parse_args()
model_file = args.model
max_epochs = args.epoch
bit_depth = args.depth
sigma = args.sigma
kernel_size = args.kernelsize
batch_size = args.batchsize
seed = args.seed
verbose = args.verbose
save_log = args.savelog
need_train = args.train
model_name, data_name = parse_model_filename(model_file)
# Which filter should apply?
filter_list = []
if bit_depth > 0:
filter_list.append('binary')
if sigma > 0:
filter_list.append('normal')
if kernel_size > 0:
filter_list.append('median')
# Which attack should apply?
attack_list = []
if args.fgsm:
attack_list.append('FGSM')
if args.bim:
attack_list.append('BIM')
if args.deepfool:
attack_list.append('DeepFool')
if args.carlini:
attack_list.append('Carlini')
if args.saliency:
attack_list.append('Saliency')
# Quit, if there is nothing to do.
if len(filter_list) == 0 or len(attack_list) == 0:
logger.warning('Neither received any filter nor any attack. Exit')
sys.exit(0)
y_file = os.path.join('save',
f'{model_name}_{data_name}_{attack_list[0]}_y.npy')
attack_files = [
os.path.join('save',
f'{model_name}_{data_name}_{attack_list[0]}_x.npy')
]
for attack_name in attack_list:
attack_files.append(
os.path.join('save',
f'{model_name}_{data_name}_{attack_name}_adv.npy'))
# the 1st file this the clean inputs
attack_list = ['clean'] + attack_list
# Do I need train the distillation network?
pretrain_files = []
for fname in filter_list:
pretrain_file = build_squeezer_filename(model_name, data_name,
max_epochs, fname)
pretrain_files.append(pretrain_file)
if not os.path.exists(os.path.join('save', pretrain_file)):
need_train = True
# set logging config. Run this before logging anything!
set_logging(LOG_NAME, data_name, verbose, save_log)
# show parameters
print(f'[{LOG_NAME}] Running feature squeezing on {model_name}...')
logger.info('Start at : %s', get_time_str())
logger.info('RECEIVED PARAMETERS:')
logger.info('model file :%s', model_file)
logger.info('model :%s', model_name)
logger.info('dataset :%s', data_name)
logger.info('max_epochs :%d', max_epochs)
logger.info('bit_depth :%d', bit_depth)
logger.info('sigma :%f', sigma)
logger.info('kernel_size :%d', kernel_size)
logger.info('batch_size :%d', batch_size)
logger.info('seed :%d', seed)
logger.info('verbose :%r', verbose)
logger.info('save_log :%r', save_log)
logger.info('need train :%r', need_train)
logger.info('filters :%s', ', '.join(filter_list))
logger.info('attacks :%s', ', '.join(attack_list))
logger.info('pretrained :%s', ', '.join(pretrain_files))
# check files
for file_name in [model_file, y_file] + attack_files:
if not os.path.exists(file_name):
logger.error('%s does not exist!', file_name)
raise FileNotFoundError('{} does not exist!'.format(file_name))
# reset seed
master_seed(seed)
# select data
dc = get_data_container(
data_name,
use_shuffle=True,
use_normalize=True,
)
# select a model
Model = get_model(model_name)
model = Model()
if data_name in ('BankNote', 'HTRU2', 'Iris', 'WheatSeed'):
num_classes = dc.num_classes
num_features = dc.dim_data[0]
model = IrisNN(num_features=num_features,
hidden_nodes=num_features * 4,
num_classes=num_classes)
classifier_mc = ModelContainerPT(model, dc)
classifier_mc.load(model_file)
accuracy = classifier_mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
# initialize Squeezer
squeezer = FeatureSqueezing(
classifier_mc,
filter_list,
bit_depth=bit_depth,
sigma=sigma,
kernel_size=kernel_size,
pretrained=True,
)
# train or load parameters for Squeezer
if need_train:
squeezer.fit(max_epochs=max_epochs, batch_size=batch_size)
squeezer.save(model_file, True)
else:
squeezer.load(model_file)
# traverse all attacks
y = np.load(y_file, allow_pickle=False)
for i in range(len(attack_list)):
adv_file = attack_files[i]
adv_name = attack_list[i]
logger.debug('Load %s...', adv_file)
adv = np.load(adv_file, allow_pickle=False)
acc_og = classifier_mc.evaluate(adv, y)
acc_squeezer = squeezer.evaluate(adv, y)
logger.info('Accuracy on %s set - OG: %f, Squeezer: %f', adv_name,
acc_og, acc_squeezer)
blocked_indices = squeezer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s', len(blocked_indices),
len(adv), adv_name)