def test_mnist_untargeted(self):
session = tf.Session()
k.set_session(session)
comp_params = {"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']}
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train = X_train[:nb_train], Y_train[:nb_train]
X_test, Y_test = X_test[:nb_test], Y_test[:nb_test]
im_shape = X_train[0].shape
# Get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train, Y_train, epochs=1, batch_size=batch_size, verbose=0)
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
# Perform attack
df = SaliencyMapMethod(classifier, sess=session)
df.set_params(clip_min=0, clip_max=1, theta=1)
x_test_adv = df.generate(X_test)
self.assertFalse((X_test == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((Y_test == y_pred).all())
scores = classifier.evaluate(x_test_adv, Y_test)
print('\naccuracy on adversarial examples: %.2f%%' % (scores[1] * 100))
def test_mnist(self):
session = tf.Session()
k.set_session(session)
comp_params = {
"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']
}
# get MNIST
batch_size, nb_train, nb_test = 100, 1000, 11
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train = X_train[:nb_train], Y_train[:nb_train]
X_test, Y_test = X_test[:nb_test], Y_test[:nb_test]
im_shape = X_train[0].shape
# get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train,
Y_train,
epochs=1,
batch_size=batch_size,
verbose=0)
# Attack
nf = NewtonFool(classifier, sess=session)
nf.set_params(max_iter=20)
x_test_adv = nf.generate(X_test)
self.assertFalse((X_test == x_test_adv).all())
y_pred = classifier.predict(X_test)
y_pred_adv = classifier.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= y_pred_adv_max).all())
scores1 = classifier.evaluate(X_test, Y_test)
print("\nAccuracy on test set: %.2f%%" % (scores1[1] * 100))
scores2 = classifier.evaluate(x_test_adv, Y_test)
print('\nAccuracy on adversarial examples: %.2f%%' %
(scores2[1] * 100))
self.assertTrue(scores1[1] != scores2[1])
def test_label_smooth(self):
session = tf.Session()
keras.backend.set_session(session)
# get MNIST
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train, X_test, Y_test = X_train[:
NB_TRAIN], Y_train[:
NB_TRAIN], X_test[:
NB_TEST], Y_test[:
NB_TEST]
im_shape = X_train[0].shape
classifier = CNN(im_shape, act="relu", defences=["labsmooth"])
classifier.compile({
'loss': 'categorical_crossentropy',
'optimizer': 'adam',
'metrics': ['accuracy']
})
# Fit the classifier
classifier.fit(X_train, Y_train, epochs=1, batch_size=BATCH_SIZE)
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy: %.2f%%" % (scores[1] * 100))
def test_mnist(self):
session = tf.Session()
k.set_session(session)
comp_params = {
"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']
}
# get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train = X_train[:nb_train], Y_train[:nb_train]
X_test, Y_test = X_test[:nb_test], Y_test[:nb_test]
im_shape = X_train[0].shape
# get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train,
Y_train,
epochs=1,
batch_size=batch_size,
verbose=0)
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
df = CarliniL2Method(classifier,
sess=session,
targeted=False,
max_iterations=100,
binary_search_steps=2,
learning_rate=1e-2,
initial_const=1)
params = {
'y_val':
random_targets(Y_test,
classifier.model.get_output_shape_at(-1)[-1])
}
x_test_adv = df.generate(X_test, **params)
self.assertFalse((X_test == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((Y_test == y_pred).all())
scores = classifier.evaluate(x_test_adv, Y_test)
print('\naccuracy on adversarial examples: %.2f%%' % (scores[1] * 100))
def test_cnn_brelu(self):
session = tf.Session()
keras.backend.set_session(session)
# get MNIST
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train, X_test, Y_test = X_train[:
NB_TRAIN], Y_train[:
NB_TRAIN], X_test[:
NB_TEST], Y_test[:
NB_TEST]
im_shape = X_train[0].shape
classifier = CNN(im_shape,
act="brelu",
act_params={
"alpha": 1,
"max_value": 2
})
classifier.compile({
'loss': 'categorical_crossentropy',
'optimizer': 'adam',
'metrics': ['accuracy']
})
# Fit the classifier
classifier.fit(X_train, Y_train, epochs=1, batch_size=BATCH_SIZE)
act_config = classifier.model.layers[1].get_config()
self.assertEqual(act_config["alpha"], 1)
self.assertEqual(act_config["max_value"], 2)
def test_cnn_batchnorm(self):
session = tf.Session()
keras.backend.set_session(session)
# get MNIST
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train, X_test, Y_test = X_train[:
NB_TRAIN], Y_train[:
NB_TRAIN], X_test[:
NB_TEST], Y_test[:
NB_TEST]
im_shape = X_train[0].shape
classifier = CNN(im_shape, act="relu", bnorm=True)
classifier.compile({
'loss': 'categorical_crossentropy',
'optimizer': 'adam',
'metrics': ['accuracy']
})
# Fit the classifier
classifier.fit(X_train, Y_train, epochs=1, batch_size=BATCH_SIZE)
bnorm_layer = classifier.model.layers[2]
self.assertIsInstance(bnorm_layer,
keras.layers.normalization.BatchNormalization)
def test_mnist_targeted(self):
session = tf.Session()
k.set_session(session)
comp_params = {"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']}
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
im_shape = x_train[0].shape
# Get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(x_train, y_train, epochs=1, batch_size=batch_size, verbose=0)
scores = classifier.evaluate(x_test, y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
# Generate random target classes
import numpy as np
nb_classes = np.unique(np.argmax(y_test, axis=1)).shape[0]
targets = np.random.randint(nb_classes, size=nb_test)
while (targets == np.argmax(y_test, axis=1)).any():
targets = np.random.randint(nb_classes, size=nb_test)
# Perform attack
df = SaliencyMapMethod(classifier, sess=session, clip_min=0, clip_max=1, theta=1)
x_test_adv = df.generate(x_test, y_val=targets)
self.assertFalse((x_test == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((y_test == y_pred).all())
scores = classifier.evaluate(x_test_adv, y_test)
print('\naccuracy on adversarial examples: %.2f%%' % (scores[1] * 100))
def test_save_load_cnn(self):
NB_TRAIN = 100
NB_TEST = 10
comp_params = {
'loss': 'categorical_crossentropy',
'optimizer': 'adam',
'metrics': ['accuracy']
}
session = tf.Session()
keras.backend.set_session(session)
# get MNIST
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train, X_test, Y_test = X_train[:
NB_TRAIN], Y_train[:
NB_TRAIN], X_test[:
NB_TEST], Y_test[:
NB_TEST]
im_shape = X_train[0].shape
classifier = CNN(im_shape, act="brelu")
classifier.compile(comp_params)
# Fit the classifier
classifier.fit(X_train, Y_train, epochs=1, batch_size=BATCH_SIZE)
path = "./tests/save/cnn/"
# Test saving
save_classifier(classifier, path)
self.assertTrue(os.path.isfile(path + "model.json"))
self.assertTrue(os.path.getsize(path + "model.json") > 0)
self.assertTrue(os.path.isfile(path + "weights.h5"))
self.assertTrue(os.path.getsize(path + "weights.h5") > 0)
# Test loading
loaded_classifier = load_classifier(path)
scores = classifier.evaluate(X_test, Y_test)
scores_loaded = loaded_classifier.evaluate(X_test, Y_test)
self.assertAlmostEqual(scores, scores_loaded)
def test_emp_robustness_mnist(self):
session = tf.Session()
K.set_session(session)
comp_params = {
"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']
}
# get MNIST
(X_train, Y_train), (_, _), _, _ = load_mnist()
X_train, Y_train = X_train[:NB_TRAIN], Y_train[:NB_TRAIN]
im_shape = X_train[0].shape
# Get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train, Y_train, epochs=1, batch_size=BATCH_SIZE)
# Compute minimal perturbations
params = {"eps_step": 1.1, "clip_min": 0., "clip_max": 1.}
emp_robust = empirical_robustness(X_train, classifier, session, "fgsm",
params)
self.assertEqual(emp_robust, 0.)
params = {
"eps_step": 1.,
"eps_max": 1.,
"clip_min": None,
"clip_max": None
}
emp_robust = empirical_robustness(X_train, classifier, session, "fgsm",
params)
self.assertAlmostEqual(emp_robust, 1., 3)
params = {
"eps_step": 0.1,
"eps_max": 0.2,
"clip_min": None,
"clip_max": None
}
emp_robust = empirical_robustness(X_train, classifier, session, "fgsm",
params)
self.assertLessEqual(emp_robust, 0.2)
def test_mnist(self):
session = tf.Session()
k.set_session(session)
comp_params = {"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']}
# get MNIST
batch_size, nb_train, nb_test = 100, 1000, 100
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train = X_train[:nb_train], Y_train[:nb_train]
X_test, Y_test = X_test[:nb_test], Y_test[:nb_test]
im_shape = X_train[0].shape
# get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train, Y_train, epochs=1, batch_size=batch_size, verbose=0)
scores = classifier.evaluate(X_train, Y_train)
print("\naccuracy on training set: %.2f%%" % (scores[1] * 100))
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
attack_params = {"verbose": 0,
"clip_min": 0.,
"clip_max": 1.,
"eps": 1.}
attack = FastGradientMethod(classifier, session)
X_train_adv = attack.generate(X_train, **attack_params)
X_test_adv = attack.generate(X_test, **attack_params)
self.assertFalse((X_train == X_train_adv).all())
self.assertFalse((X_test == X_test_adv).all())
train_y_pred = get_labels_np_array(classifier.predict(X_train_adv))
test_y_pred = get_labels_np_array(classifier.predict(X_test_adv))
self.assertFalse((Y_train == train_y_pred).all())
self.assertFalse((Y_test == test_y_pred).all())
scores = classifier.evaluate(X_train_adv, Y_train)
print('\naccuracy on adversarial train examples: %.2f%%' % (scores[1] * 100))
scores = classifier.evaluate(X_test_adv, Y_test)
print('\naccuracy on adversarial test examples: %.2f%%' % (scores[1] * 100))
# test minimal perturbations
attack_params = {"verbose": 0,
"clip_min": 0.,
"clip_max": 1.,
"minimal": True,
"eps_step": .1,
"eps_max": 1.}
X_train_adv_min = attack.generate(X_train, **attack_params)
X_test_adv_min = attack.generate(X_test, **attack_params)
self.assertFalse((X_train_adv_min == X_train_adv).all())
self.assertFalse((X_test_adv_min == X_test_adv).all())
self.assertFalse((X_train == X_train_adv_min).all())
self.assertFalse((X_test == X_test_adv_min).all())
train_y_pred = get_labels_np_array(classifier.predict(X_train_adv_min))
test_y_pred = get_labels_np_array(classifier.predict(X_test_adv_min))
self.assertFalse((Y_train == train_y_pred).all())
self.assertFalse((Y_test == test_y_pred).all())
scores = classifier.evaluate(X_train_adv_min, Y_train)
print('\naccuracy on adversarial train examples with minimal perturbation: %.2f%%' % (scores[1] * 100))
scores = classifier.evaluate(X_test_adv_min, Y_test)
print('\naccuracy on adversarial test examples with minimal perturbation: %.2f%%' % (scores[1] * 100))
# Get dataset
(X_train, Y_train), (X_test, Y_test), _, _ = load_dataset(args.dataset)
if os.path.isfile(args.dataset):
X_train = np.load(args.dataset)
Y_train = Y_train if "train.npy" in args.dataset else Y_test
im_shape = X_train[0].shape
session = tf.Session()
k.set_session(session)
if args.classifier == "cnn":
classifier = CNN(im_shape,
act=args.act,
bnorm=False,
defences=args.defences,
dataset=args.dataset)
elif args.classifier == "resnet":
classifier = ResNet(im_shape,
act=args.act,
bnorm=False,
defences=args.defences)
elif args.classifier == "mlp":
classifier = MLP(im_shape,
act=args.act,
bnorm=False,
defences=args.defences,
dataset=args.dataset)
# Fit the classifier
# Get session
session = tf.Session()
k.set_session(session)
# Read MNIST dataset
(x_train, y_train), (x_test, y_test), min_, max_ = load_dataset('mnist')
im_shape = x_train[0].shape
# Construct and train a Resnet convolutional neural network
comp_params = {'loss': 'categorical_crossentropy',
'optimizer': 'adam',
'metrics': ['accuracy']}
source = ResNet(im_shape, act='relu')
source.compile(comp_params)
source.fit(x_train, y_train, validation_split=.1, epochs=5, batch_size=128)
# Craft adversarial samples with DeepFool
epsilon = .1 # Maximum perturbation
adv_crafter = DeepFool(source, sess=session)
x_train_adv = adv_crafter.generate(x_val=x_train, eps=epsilon, clip_min=min_, clip_max=max_)
x_test_adv = adv_crafter.generate(x_val=x_test, eps=epsilon, clip_min=min_, clip_max=max_)
# Construct and train a convolutional neural network
target = CNN(im_shape, act='relu', dataset='mnist')
target.compile(comp_params)
target.fit(x_train, y_train, validation_split=.1, epochs=5, batch_size=128)
# Evaluate the CNN on the adversarial samples
scores = target.evaluate(x_test, y_test)
print("\nLoss on adversarial samples: %.2f%%\nAccuracy on adversarial samples: %.2f%%" % (scores[0], scores[1] * 100))
from src.classifiers.cnn import CNN
from src.utils import load_dataset
# Get session
session = tf.Session()
k.set_session(session)
# Read MNIST dataset
(x_train, y_train), (x_test, y_test), min_, max_ = load_dataset('mnist')
im_shape = x_train[0].shape
# Construct a convolutional neural network
comp_params = {'loss': 'categorical_crossentropy',
'optimizer': 'adam',
'metrics': ['accuracy']}
classifier = CNN(im_shape, act='relu', dataset='mnist')
classifier.compile(comp_params)
classifier.fit(x_train, y_train, validation_split=.1, epochs=5, batch_size=128)
# Evaluate the classifier on the test set
scores = classifier.evaluate(x_test, y_test)
print("\nTest loss: %.2f%%\nTest accuracy: %.2f%%" % (scores[0], scores[1] * 100))
# Craft adversarial samples with FGSM
epsilon = .1 # Maximum perturbation
adv_crafter = FastGradientMethod(classifier, sess=session)
x_test_adv = adv_crafter.generate(x_val=x_test, eps=epsilon, clip_min=min_, clip_max=max_)
# Evaluate the classifier on the adversarial examples
scores = classifier.evaluate(x_test_adv, y_test)
print("\nTest loss: %.2f%%\nTest accuracy: %.2f%%" % (scores[0], scores[1] * 100))
from src.classifiers.cnn import CNN
from src.utils import load_dataset
# Read CIFAR10 dataset
(x_train, y_train), (x_test, y_test), _, _ = load_dataset('cifar10')
im_shape = x_train[0].shape
# Construct a convolutional neural network with feature squeezing activated
# For CIFAR10, squeezing the features to 3 bits works well
comp_params = {
'loss': 'categorical_crossentropy',
'optimizer': 'adam',
'metrics': ['accuracy']
}
classifier = CNN(im_shape,
act='relu',
dataset='cifar10',
defences='featsqueeze3')
classifier.compile(comp_params)
classifier.fit(x_train,
y_train,
validation_split=.1,
epochs=10,
batch_size=128)
# Evaluate the classifier on the test set
scores = classifier.evaluate(x_test, y_test)
print("\nTest loss: %.2f%%\nTest accuracy: %.2f%%" %
(scores[0], scores[1] * 100))
def test_mnist(self):
session = tf.Session()
k.set_session(session)
comp_params = {"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']}
# get MNIST
batch_size, nb_train, nb_test = 10, 10, 10
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train = X_train[:nb_train], Y_train[:nb_train]
X_test, Y_test = X_test[:nb_test], Y_test[:nb_test]
im_shape = X_train[0].shape
# get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train, Y_train, epochs=1, batch_size=batch_size, verbose=0)
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
attack_params = {"verbose": 2,
"clip_min": 0.,
"clip_max": 1,
"attacker": "deepfool"}
attack = UniversalPerturbation(classifier, session)
x_train_adv = attack.generate(X_train, **attack_params)
self.assertTrue((attack.fooling_rate >= 0.2) or not attack.converged)
x_test_adv = X_test + attack.v
self.assertFalse((X_test == x_test_adv).all())
train_y_pred = get_labels_np_array(classifier.predict(x_train_adv))
test_y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((Y_test == test_y_pred).all())
self.assertFalse((Y_train == train_y_pred).all())
scores = classifier.evaluate(x_train_adv, Y_train)
print('\naccuracy on adversarial train examples: %.2f%%' % (scores[1] * 100))
scores = classifier.evaluate(x_test_adv, Y_test)
print('\naccuracy on adversarial test examples: %.2f%%' % (scores[1] * 100))