def test_squeezing_data(self):
x_range = get_range(self.mc.data_container.x_train)
x_train = np.copy(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=['normal', 'binary'],
bit_depth=8,
sigma=0.1,
kernel_size=2,
pretrained=False,
)
# Expecting difference between input data and squeezed data
mc_normal = squeezer.get_def_model_container('normal')
mc_binary = squeezer.get_def_model_container('binary')
self.assertFalse((x_train == mc_normal.data_container.x_train).all())
self.assertFalse((x_train == mc_binary.data_container.x_train).all())
# average perturbation
l2 = np.mean(get_l2_norm(x_train, mc_normal.data_container.x_train))
logger.info('L2 norm of normal squeezer:%f', l2)
self.assertLessEqual(l2, 0.2)
# maximum perturbation
l2 = np.max(get_l2_norm(x_train, mc_binary.data_container.x_train))
logger.info('L2 norm of binary squeezer:%f', l2)
self.assertLessEqual(l2, 0.2)
def test_deepfool(self):
attack = attacks.DeepFoolContainer(
self.mc,
max_iter=100,
epsilon=1e-6,
nb_grads=10)
adv, y_adv, x_clean, y_clean = attack.generate(count=NUM_ADV)
# At least made some change from clean images
self.assertFalse((adv == x_clean).all())
# test accuracy
accuracy = self.mc.evaluate(adv, y_clean)
logger.info('Accuracy on adv. examples: %f', accuracy)
self.assertLessEqual(accuracy, 0.2)
# test success rate
success_rate = (y_adv != y_clean).sum() / len(y_adv)
logger.info('Success rate of adv. attack: %f', success_rate)
self.assertGreaterEqual(success_rate, 0.8)
# sum success rate (missclassified) and accuracy (correctly classified)
self.assertAlmostEqual(success_rate + accuracy, 1.0, places=4)
# Check the max perturbation
dif = np.max(np.abs(adv - x_clean))
logger.info('Max perturbation (L1-norm): %f', dif)
self.assertLessEqual(dif, 1.0 + 1e-4)
# Check bounding box
self.assertLessEqual(np.max(adv), 1.0 + 1e-4)
self.assertGreaterEqual(np.min(adv), 0 - 1e-4)
l2 = np.max(get_l2_norm(adv, x_clean))
logger.info('L2 norm = %f', l2)
def main():
master_seed(SEED)
dataset = DATASET_LIST[DATA_NAME]
dc = DataContainer(dataset, get_data_path())
dc()
model = MnistCnnCW()
mc = ModelContainerPT(model, dc)
mc.load(MODEL_FILE)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
print('Accuracy on test set: {}'.format(accuracy))
attack = CarliniL2V2Container(mc,
targeted=False,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e10),
batch_size=16,
clip_values=(0.0, 1.0))
adv, y_adv, x_clean, y_clean = attack.generate(count=100)
l2 = np.mean(get_l2_norm(adv, x_clean))
print('L2 norm: {}'.format(l2))
not_match = y_adv != y_clean
success_rate = len(not_match[not_match == True]) / len(adv)
print('Success rate: {}'.format(success_rate))
accuracy = mc.evaluate(adv, y_clean)
print('Accuracy on adv. examples: {}'.format(accuracy))
def test_squeezing_data(self):
x_range = get_range(self.mc.data_container.x_train)
x_train = np.copy(self.mc.data_container.x_train)
squeezer = FeatureSqueezing(
self.mc,
clip_values=x_range,
smoothing_methods=SQUEEZING_METHODS,
bit_depth=8,
sigma=0.1,
kernel_size=3,
pretrained=False,
)
# Expecting difference between input data and squeezed data
mc_binary = squeezer.get_def_model_container(SQUEEZING_METHODS[0])
mc_median = squeezer.get_def_model_container(SQUEEZING_METHODS[1])
mc_normal = squeezer.get_def_model_container(SQUEEZING_METHODS[2])
self.assertFalse((x_train == mc_binary.data_container.x_train).all())
self.assertFalse((x_train == mc_median.data_container.x_train).all())
self.assertFalse((x_train == mc_normal.data_container.x_train).all())
# maximum perturbation
l2 = np.max(get_l2_norm(x_train, mc_binary.data_container.x_train))
logger.info('L2 norm of binary squeezer:%f', l2)
self.assertLessEqual(l2, 0.5)
# average perturbation
# No upper bound on median filter
l2 = np.mean(get_l2_norm(x_train, mc_median.data_container.x_train))
logger.info('L2 norm of median squeezer:%f', l2)
self.assertLessEqual(l2, 2.0)
# average perturbation
l2 = np.mean(get_l2_norm(x_train, mc_normal.data_container.x_train))
logger.info('L2 norm of normal squeezer:%f', l2)
self.assertLessEqual(l2, 2.0)
def test_carlini(self):
clip_values = get_range(self.dc.x_train)
# Lower the upper bound of `c_range` will reduce the norm of perturbation.
attack = attacks.CarliniL2V2Container(
self.mc,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e4),
batch_size=16,
clip_values=clip_values
)
adv, y_adv, x_clean, y_clean = attack.generate(count=NUM_ADV)
# At least made some change from clean images
self.assertFalse((adv == x_clean).all())
# test accuracy
accuracy = self.mc.evaluate(adv, y_clean)
logger.info('Accuracy on adv. examples: %f', accuracy)
self.assertLessEqual(accuracy, 0.4)
# test success rate
success_rate = (y_adv != y_clean).sum() / len(y_adv)
logger.info('Success rate of adv. attack: %f', success_rate)
self.assertGreaterEqual(success_rate, 0.6)
# sum success rate (missclassified) and accuracy (correctly classified)
self.assertAlmostEqual(success_rate + accuracy, 1.0, places=4)
# Check the max perturbation
dif = np.max(np.abs(adv - x_clean))
logger.info('Max perturbation (L1-norm): %f', dif)
self.assertLessEqual(dif, 1.0 + 1e-4)
# Check bounding box
self.assertLessEqual(np.max(adv), 1.0 + 1e-4)
self.assertGreaterEqual(np.min(adv), 0 - 1e-4)
l2 = np.max(get_l2_norm(adv, x_clean))
logger.info('L2 norm = %f', l2)
def main():
master_seed(SEED)
dataset = DATASET_LIST[DATA_NAME]
dc = DataContainer(dataset, get_data_path())
dc(shuffle=True, normalize=True)
print('# of trainset: {}, # of testset: {}'.format(len(dc.x_train),
len(dc.x_test)))
num_classes = dc.num_classes
num_features = dc.dim_data[0]
model = IrisNN(num_features=num_features,
hidden_nodes=num_features * 4,
num_classes=num_classes)
mc = ModelContainerPT(model, dc)
mc.load(MODEL_FILE)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
print('Accuracy on test set: {}'.format(accuracy))
clip_values = get_range(dc.x_train, is_image=False)
print('clip_values', clip_values)
attack = CarliniL2V2Container(mc,
targeted=False,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e4),
batch_size=16,
clip_values=clip_values)
adv, y_adv, x_clean, y_clean = attack.generate(count=100)
l2 = np.mean(get_l2_norm(adv, x_clean))
print('L2 norm: {}'.format(l2))
not_match = y_adv != y_clean
success_rate = len(not_match[not_match == True]) / len(adv)
print('Success rate: {}'.format(success_rate))
accuracy = mc.evaluate(adv, y_clean)
print('Accuracy on adv. examples: {}'.format(accuracy))
def test_carlini(self):
attack = attacks.CarliniL2V2Container(self.mc,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e10),
batch_size=64,
clip_values=(0.0, 1.0))
adv, y_adv, x_clean, y_clean = attack.generate(count=NUM_ADV)
# At least made some change from clean images
self.assertFalse((adv == x_clean).all())
# test accuracy
accuracy = self.mc.evaluate(adv, y_clean)
logger.info('Accuracy on adv. examples: %f', accuracy)
self.assertLessEqual(accuracy, 0.60)
# test success rate
success_rate = (y_adv != y_clean).sum() / len(y_adv)
logger.info('Success rate of adv. attack: %f', success_rate)
self.assertGreaterEqual(success_rate, 0.40)
# Check the max perturbation
dif = np.max(np.abs(adv - x_clean))
logger.info('Max perturbation (L1-norm): %f', dif)
self.assertLessEqual(dif, 1.0 + 1e-4)
# Check bounding box
self.assertLessEqual(np.max(adv), 1.0 + 1e-4)
self.assertGreaterEqual(np.min(adv), 0 - 1e-4)
l2 = np.max(get_l2_norm(adv, x_clean))
logger.info('L2 norm = %f', l2)