def test_iris_unbounded(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_kr()
# Recreate a classifier without clip values
krc = KerasClassifier(model=classifier._model,
use_logits=False,
channel_index=1)
rs = RandomizedSmoothing(classifier=krc,
sample_size=100,
scale=0.01,
alpha=0.001)
attack = FastGradientMethod(rs, eps=1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv > 1).any())
self.assertTrue((x_test_adv < 0).any())
preds_smooth = np.argmax(rs.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_smooth).all())
pred = rs.predict(x_test)
pred2 = rs.predict(x_test_adv)
acc, cov = compute_accuracy(pred, y_test)
acc2, cov2 = compute_accuracy(pred2, y_test)
logger.info(
'Accuracy on Iris with smoothing on adversarial examples: %.2f%%',
(acc * 100))
logger.info(
'Coverage on Iris with smoothing on adversarial examples: %.2f%%',
(cov * 100))
logger.info('Accuracy on Iris with smoothing: %.2f%%', (acc2 * 100))
logger.info('Coverage on Iris with smoothing: %.2f%%', (cov2 * 100))
def test_iris_clipped(self):
(_, _), (x_test, y_test) = self.iris
krc = get_iris_classifier_kr()
rs = RandomizedSmoothing(classifier=krc,
sample_size=100,
scale=0.01,
alpha=0.001)
# Test untargeted attack
attack = FastGradientMethod(krc, eps=.1)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_smooth = np.argmax(rs.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_smooth).all())
pred = rs.predict(x_test)
pred2 = rs.predict(x_test_adv)
acc, cov = compute_accuracy(pred, y_test)
acc2, cov2 = compute_accuracy(pred2, y_test)
logger.info(
'Accuracy on Iris with smoothing on adversarial examples: %.2f%%',
(acc * 100))
logger.info(
'Coverage on Iris with smoothing on adversarial examples: %.2f%%',
(cov * 100))
logger.info('Accuracy on Iris with smoothing: %.2f%%', (acc2 * 100))
logger.info('Coverage on Iris with smoothing: %.2f%%', (cov2 * 100))
# Check basic functionality of RS object
# check predict
y_test_smooth = rs.predict(x=x_test)
self.assertEqual(y_test_smooth.shape, y_test.shape)
self.assertTrue((np.sum(y_test_smooth, axis=1) <= 1).all())
# check gradients
grad_smooth1 = rs.loss_gradient(x=x_test, y=y_test)
grad_smooth2 = rs.class_gradient(x=x_test, label=None)
grad_smooth3 = rs.class_gradient(x=x_test,
label=np.argmax(y_test, axis=1))
self.assertEqual(grad_smooth1.shape, x_test_adv.shape)
self.assertEqual(grad_smooth2.shape[0], len(x_test))
self.assertEqual(grad_smooth3.shape[0], len(x_test))
# check certification
pred, radius = rs.certify(x=x_test, n=250)
self.assertEqual(len(pred), len(x_test))
self.assertEqual(len(radius), len(x_test))
self.assertTrue((radius <= 1).all())
self.assertTrue((pred < y_test.shape[1]).all())
def test_krclassifier(self):
"""
Test with a KerasClassifier.
:return:
"""
# Build KerasClassifier
krc = get_classifier_kr()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# First FGSM attack:
fgsm = FastGradientMethod(classifier=krc, targeted=True)
params = {'y': random_targets(y_test, krc.nb_classes())}
x_test_adv = fgsm.generate(x_test, **params)
# Initialize RS object and attack with FGSM
rs = RandomizedSmoothing(classifier=krc,
sample_size=100,
scale=0.01,
alpha=0.001)
fgsm_with_rs = FastGradientMethod(classifier=rs, targeted=True)
x_test_adv_with_rs = fgsm_with_rs.generate(x_test, **params)
# Compare results
# check shapes are equal and values are within a certain range
self.assertEqual(x_test_adv.shape, x_test_adv_with_rs.shape)
self.assertTrue((np.abs(x_test_adv - x_test_adv_with_rs) < 0.75).all())
# Check basic functionality of RS object
# check predict
y_test_smooth = rs.predict(x=x_test)
y_test_base = krc.predict(x=x_test)
self.assertEqual(y_test_smooth.shape, y_test.shape)
self.assertTrue((np.sum(y_test_smooth, axis=1) <= np.ones(
(NB_TEST, ))).all())
self.assertTrue((np.argmax(y_test_smooth,
axis=1) == np.argmax(y_test_base,
axis=1)).all())
# check gradients
grad_smooth1 = rs.loss_gradient(x=x_test, y=y_test)
grad_smooth2 = rs.class_gradient(x=x_test, label=None)
grad_smooth3 = rs.class_gradient(x=x_test,
label=np.argmax(y_test, axis=1))
self.assertEqual(grad_smooth1.shape, x_test_adv.shape)
self.assertEqual(grad_smooth2.shape[0], NB_TEST)
self.assertEqual(grad_smooth3.shape[0], NB_TEST)
# check certification
pred, radius = rs.certify(x=x_test, n=250)
self.assertEqual(len(pred), NB_TEST)
self.assertEqual(len(radius), NB_TEST)
self.assertTrue((radius <= 1).all())
self.assertTrue((pred < y_test.shape[1]).all())