def test_iris_unbounded(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_kr()
# Recreate a classifier without clip values
krc = KerasClassifier(model=classifier._model,
use_logits=False,
channel_index=1)
rs = RandomizedSmoothing(classifier=krc,
sample_size=100,
scale=0.01,
alpha=0.001)
attack = FastGradientMethod(rs, eps=1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv > 1).any())
self.assertTrue((x_test_adv < 0).any())
preds_smooth = np.argmax(rs.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_smooth).all())
pred = rs.predict(x_test)
pred2 = rs.predict(x_test_adv)
acc, cov = compute_accuracy(pred, y_test)
acc2, cov2 = compute_accuracy(pred2, y_test)
logger.info(
'Accuracy on Iris with smoothing on adversarial examples: %.2f%%',
(acc * 100))
logger.info(
'Coverage on Iris with smoothing on adversarial examples: %.2f%%',
(cov * 100))
logger.info('Accuracy on Iris with smoothing: %.2f%%', (acc2 * 100))
logger.info('Coverage on Iris with smoothing: %.2f%%', (cov2 * 100))
def test_iris_clipped(self):
(_, _), (x_test, y_test) = self.iris
krc, _ = get_iris_classifier_kr()
rs = RandomizedSmoothing(classifier=krc,
sample_size=100,
scale=0.01,
alpha=0.001)
# Test untargeted attack
attack = FastGradientMethod(krc, eps=.1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_base = np.argmax(rs.predict(x_test), axis=1)
preds_smooth = np.argmax(rs.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_smooth).all())
pred = rs.predict(x_test)
pred2 = rs.predict(x_test_adv)
acc, cov = compute_accuracy(pred, y_test)
acc2, cov2 = compute_accuracy(pred2, y_test)
logger.info(
'Accuracy on Iris with smoothing on adversarial examples: %.2f%%',
(acc * 100))
logger.info(
'Coverage on Iris with smoothing on adversarial examples: %.2f%%',
(cov * 100))
logger.info('Accuracy on Iris with smoothing: %.2f%%', (acc2 * 100))
logger.info('Coverage on Iris with smoothing: %.2f%%', (cov2 * 100))
# Check basic functionality of RS object
# check predict
y_test_smooth = rs.predict(x=x_test)
self.assertEqual(y_test_smooth.shape, y_test.shape)
self.assertTrue((np.sum(y_test_smooth, axis=1) <= 1).all())
# check gradients
grad_smooth1 = rs.loss_gradient(x=x_test, y=y_test)
grad_smooth2 = rs.class_gradient(x=x_test, label=None)
grad_smooth3 = rs.class_gradient(x=x_test,
label=np.argmax(y_test, axis=1))
self.assertEqual(grad_smooth1.shape, x_test_adv.shape)
self.assertEqual(grad_smooth2.shape[0], len(x_test))
self.assertEqual(grad_smooth3.shape[0], len(x_test))
# check certification
pred, radius = rs.certify(x=x_test, n=250)
self.assertEqual(len(pred), len(x_test))
self.assertEqual(len(radius), len(x_test))
self.assertTrue((radius <= 1).all())
self.assertTrue((pred < y_test.shape[1]).all())
def test_iris_clipped(self):
(_, _), (x_test, y_test) = self.iris
ptc = get_tabular_classifier_pt()
rs = PyTorchRandomizedSmoothing(
model=ptc.model,
loss=ptc._loss,
input_shape=ptc.input_shape,
nb_classes=ptc.nb_classes,
channels_first=ptc.channels_first,
clip_values=ptc.clip_values,
sample_size=100,
scale=0.01,
alpha=0.001,
)
# Test untargeted attack
attack = FastGradientMethod(ptc, eps=0.1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_smooth = np.argmax(rs.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_smooth).all())
pred = rs.predict(x_test)
pred2 = rs.predict(x_test_adv)
acc, cov = compute_accuracy(pred, y_test)
acc2, cov2 = compute_accuracy(pred2, y_test)
logger.info("Accuracy on Iris with smoothing on adversarial examples: %.2f%%", (acc * 100))
logger.info("Coverage on Iris with smoothing on adversarial examples: %.2f%%", (cov * 100))
logger.info("Accuracy on Iris with smoothing: %.2f%%", (acc2 * 100))
logger.info("Coverage on Iris with smoothing: %.2f%%", (cov2 * 100))
# Check basic functionality of RS object
# check predict
y_test_smooth = rs.predict(x=x_test)
self.assertEqual(y_test_smooth.shape, y_test.shape)
self.assertTrue((np.sum(y_test_smooth, axis=1) <= 1).all())
# check certification
pred, radius = rs.certify(x=x_test, n=250)
self.assertEqual(len(pred), len(x_test))
self.assertEqual(len(radius), len(x_test))
self.assertTrue((radius <= 1).all())
self.assertTrue((pred < y_test.shape[1]).all())
# target_models.append(tf.keras.models.load_model('./Models/ens/ens_model_0.h5')) # 集成对抗训练
target_models.append(
tf.keras.models.load_model('./Models/ens/pr_model_0.h5')) # 投影正则化
# target_models.append(tf.keras.models.load_model('./Models/ens/model_0.h5')) # 无防御
# target_models.append(tf.keras.models.load_model('./Models/ens/pgd_model_0.h5')) # PGD对抗训练
art = Wrapper(source_models[0],
nb_classes=10,
clip_values=(0, 1),
input_shape=input_shape,
loss_object=loss_fn)
for attack in attacks:
print('Performing' + attack + 'attack:')
adv_acc = []
for eps in [0, 0.2, 0.3, 0.4]: # ！！！
if eps > 0:
FGM_params['eps'] = eps
PGD_params['eps'] = eps
adversary = eval(attack + '(art, **' + attack + '_params)')
x_adv = adversary.generate(x_test[:m])
else:
x_adv = x_test[:m]
for target in target_models:
adv_acc.append(
compute_accuracy(target.predict(x_adv), y_test[:m])[0])
# print('Target Accuracy=', round(target_acc*100, 2),'%')
# row = (lambda a: 1 if a < 10 else (10 if a > 100 else a//10))(m)
# column = np.amin([10, m])
# grid_visual(x_adv[:int(row * column)].reshape(column, row, input_shape[0], input_shape[1], input_shape[2]))
print(adv_acc)
classifier_ga2 = cnn_mnist(x_train.shape[1:], min_, max_)
classifier_ga2.fit(x_new2, y_train, nb_epochs=num_epochs, batch_size=128)
# 2.4 创建平滑的分类器
classifier_rs = RandomizedSmoothing(classifier, sample_size=100, scale=0.25, alpha=0.001)
classifier_rs1 = RandomizedSmoothing(classifier_ga1, sample_size=100, scale=sigma1, alpha=0.001)
classifier_rs2 = RandomizedSmoothing(classifier_ga2, sample_size=100, scale=sigma2, alpha=0.001)
# 3. 进行检测
# 3.1 将随机平滑模型的预测结果与原始模型f进行比较
x_preds = classifier.predict(x_test)
x_preds_rs1 = classifier_rs1.predict(x_test)
x_preds_rs2 = classifier_rs2.predict(x_test)
acc, cov = compute_accuracy(x_preds, y_test)
acc_rs1, cov_rs1 = compute_accuracy(x_preds_rs1, y_test)
acc_rs2, cov_rs2 = compute_accuracy(x_preds_rs2, y_test)
print("Original test data (first 250 images):")
print("Original Classifier")
print("Accuracy: {}".format(acc))
print("Coverage: {}".format(cov))
print("Smoothed Classifier, sigma=" + str(sigma1))
print("Accuracy: {}".format(acc_rs1))
print("Coverage: {}".format(cov_rs1))
print("Smoothed Classifier, sigma=" + str(sigma2))
print("Accuracy: {}".format(acc_rs2))
print("Coverage: {}".format(cov_rs2))
"""
4 prediction(s) abstained.