def test_carlini_l2_attack(self):
attack = attacks.CarliniL2V2Container(self.mc,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e10),
batch_size=64,
clip_values=(0.0, 1.0))
blocked_indices, adv_success_rate = self.preform_attack(attack,
count=NUM_ADV)
block_rate = len(blocked_indices) / NUM_ADV
self.assertGreater(block_rate, adv_success_rate * 0.6)
logger.info('[%s] Block rate: %f', attack.__class__.__name__,
block_rate)
def test_carlini_l2_attack(self):
clip_values = get_range(self.dc.x_train)
# Lower the upper bound of `c_range` will reduce the norm of perturbation.
attack = attacks.CarliniL2V2Container(self.mc,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e4),
batch_size=16,
clip_values=clip_values)
blocked_indices, adv_success_rate = self.preform_attack(attack,
count=NUM_ADV)
block_rate = len(blocked_indices) / NUM_ADV
self.assertGreater(block_rate, adv_success_rate * 0.6)
logger.info('[%s] Block rate: %f', attack.__class__.__name__,
block_rate)
def test_carlini(self):
clip_values = get_range(self.dc.x_train)
# Lower the upper bound of `c_range` will reduce the norm of perturbation.
attack = attacks.CarliniL2V2Container(
self.mc,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e4),
batch_size=16,
clip_values=clip_values
)
adv, y_adv, x_clean, y_clean = attack.generate(count=NUM_ADV)
# At least made some change from clean images
self.assertFalse((adv == x_clean).all())
# test accuracy
accuracy = self.mc.evaluate(adv, y_clean)
logger.info('Accuracy on adv. examples: %f', accuracy)
self.assertLessEqual(accuracy, 0.4)
# test success rate
success_rate = (y_adv != y_clean).sum() / len(y_adv)
logger.info('Success rate of adv. attack: %f', success_rate)
self.assertGreaterEqual(success_rate, 0.6)
# sum success rate (missclassified) and accuracy (correctly classified)
self.assertAlmostEqual(success_rate + accuracy, 1.0, places=4)
# Check the max perturbation
dif = np.max(np.abs(adv - x_clean))
logger.info('Max perturbation (L1-norm): %f', dif)
self.assertLessEqual(dif, 1.0 + 1e-4)
# Check bounding box
self.assertLessEqual(np.max(adv), 1.0 + 1e-4)
self.assertGreaterEqual(np.min(adv), 0 - 1e-4)
l2 = np.max(get_l2_norm(adv, x_clean))
logger.info('L2 norm = %f', l2)
def test_carlini(self):
attack = attacks.CarliniL2V2Container(self.mc,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e10),
batch_size=64,
clip_values=(0.0, 1.0))
adv, y_adv, x_clean, y_clean = attack.generate(count=NUM_ADV)
# At least made some change from clean images
self.assertFalse((adv == x_clean).all())
# test accuracy
accuracy = self.mc.evaluate(adv, y_clean)
logger.info('Accuracy on adv. examples: %f', accuracy)
self.assertLessEqual(accuracy, 0.60)
# test success rate
success_rate = (y_adv != y_clean).sum() / len(y_adv)
logger.info('Success rate of adv. attack: %f', success_rate)
self.assertGreaterEqual(success_rate, 0.40)
# Check the max perturbation
dif = np.max(np.abs(adv - x_clean))
logger.info('Max perturbation (L1-norm): %f', dif)
self.assertLessEqual(dif, 1.0 + 1e-4)
# Check bounding box
self.assertLessEqual(np.max(adv), 1.0 + 1e-4)
self.assertGreaterEqual(np.min(adv), 0 - 1e-4)
l2 = np.max(get_l2_norm(adv, x_clean))
logger.info('L2 norm = %f', l2)
def main():
dc = DataContainer(DATASET_LIST[NAME], get_data_path())
dc()
model = MnistCnnV2()
mc = ModelContainerPT(model, dc)
mc.load(MODEL_FILE)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
print(f'Accuracy on test set: {accuracy}')
carlini_attack = attacks.CarliniL2V2Container(
mc,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e10),
batch_size=BATCH_SIZE,
clip_values=(0.0, 1.0),
)
# adv, y_adv, x_clean, y_clean = carlini_attack.generate(count=1000)
# carlini_attack.save_attack(
# 'MnistCnnV2_MNIST_Carlini',
# adv,
# y_adv,
# x_clean,
# y_clean,
# True,
# )
# use pre-trained adversarial examples
adv, y_adv, x_clean, y_clean = carlini_attack.load_adv_examples(
CARLINI_FILE)
accuracy = mc.evaluate(adv, y_clean)
print(f'Accuracy on adv. examples: {accuracy}')
bim_attack = attacks.BIMContainer(
mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False,
)
k2, zeta, kappa, gamma = cross_validation(dc, model, bim_attack)
# use pre-defined parameters
# k2, zeta, kappa, gamma, score = K2, RELIABILITY, KAPPA, CONFIDENCE, 0
ad = ApplicabilityDomainContainer(
mc,
hidden_model=model.hidden_model,
k2=k2,
reliability=zeta,
sample_ratio=SAMPLE_RATIO,
kappa=kappa,
confidence=gamma,
)
print(ad.params)
ad.fit()
blocked_indices, x_passed = ad.detect(adv, y_adv, return_passed_x=True)
print('After update parameters, blocked {}/{} samples from adv. examples'.
format(len(blocked_indices), len(adv)))
def main():
dc = DataContainer(DATASET_LIST[NAME], get_data_path())
dc(shuffle=True, normalize=True, size_train=0.8)
print('Sample size: {}'.format(len(dc)))
num_classes = dc.num_classes
num_features = dc.dim_data[0]
model = IrisNN(num_features=num_features,
hidden_nodes=num_features * 4,
num_classes=num_classes)
mc = ModelContainerPT(model, dc)
# mc.fit(max_epochs=MAX_EPOCHS, batch_size=BATCH_SIZE, early_stop=False)
# filename = get_pt_model_filename('IrisNN', NAME, MAX_EPOCHS)
# mc.save(filename, overwrite=True)
mc.load(MODEL_FILE)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
print(f'Accuracy on test set: {accuracy}')
carlini_attack = attacks.CarliniL2V2Container(
mc,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e4),
batch_size=BATCH_SIZE,
clip_values=(0.0, 1.0),
)
# we need more than 30 adv. examples
# x_all = dc.x_all
# y_all = dc.x_all
# indices = np.random.permutation(np.arange(len(dc)))[:100]
# x = x_all[indices]
# y = y_all[indices]
# adv, pred_adv, x_clean, pred_clean = carlini_attack.generate(use_testset=False, x=x)
# carlini_attack.save_attack(
# 'IrisNN_Iris_Carlini',
# adv, pred_adv,
# x_clean, pred_clean,
# True,
# )
# use pre-trained adversarial examples
adv, y_adv, x_clean, pred_clean = carlini_attack.load_adv_examples(
CARLINI_FILE)
accuracy = mc.evaluate(adv, pred_clean)
print(f'Accuracy on adv. examples: {accuracy}')
bim_attack = attacks.BIMContainer(
mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False,
)
k2, zeta, kappa, gamma = cross_validation(dc, model, bim_attack)
ad = ApplicabilityDomainContainer(
mc,
hidden_model=model.hidden_model,
k2=k2,
reliability=zeta,
sample_ratio=SAMPLE_RATIO,
kappa=kappa,
confidence=gamma,
)
print(ad.params)
ad.fit()
blocked_indices, x_passed = ad.detect(adv, y_adv, return_passed_x=True)
print('After update parameters, blocked {}/{} samples from adv. examples'.
format(len(blocked_indices), len(adv)))