def test_mnist_untargeted(self):
session = tf.Session()
k.set_session(session)
comp_params = {"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']}
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train = X_train[:nb_train], Y_train[:nb_train]
X_test, Y_test = X_test[:nb_test], Y_test[:nb_test]
im_shape = X_train[0].shape
# Get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train, Y_train, epochs=1, batch_size=batch_size, verbose=0)
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
# Perform attack
df = SaliencyMapMethod(classifier, sess=session)
df.set_params(clip_min=0, clip_max=1, theta=1)
x_test_adv = df.generate(X_test)
self.assertFalse((X_test == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((Y_test == y_pred).all())
scores = classifier.evaluate(x_test_adv, Y_test)
print('\naccuracy on adversarial examples: %.2f%%' % (scores[1] * 100))
def test_label_smooth(self):
session = tf.Session()
keras.backend.set_session(session)
# get MNIST
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train, X_test, Y_test = X_train[:
NB_TRAIN], Y_train[:
NB_TRAIN], X_test[:
NB_TEST], Y_test[:
NB_TEST]
im_shape = X_train[0].shape
classifier = CNN(im_shape, act="relu", defences=["labsmooth"])
classifier.compile({
'loss': 'categorical_crossentropy',
'optimizer': 'adam',
'metrics': ['accuracy']
})
# Fit the classifier
classifier.fit(X_train, Y_train, epochs=1, batch_size=BATCH_SIZE)
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy: %.2f%%" % (scores[1] * 100))
def test_cnn_brelu(self):
session = tf.Session()
keras.backend.set_session(session)
# get MNIST
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train, X_test, Y_test = X_train[:
NB_TRAIN], Y_train[:
NB_TRAIN], X_test[:
NB_TEST], Y_test[:
NB_TEST]
im_shape = X_train[0].shape
classifier = CNN(im_shape,
act="brelu",
act_params={
"alpha": 1,
"max_value": 2
})
classifier.compile({
'loss': 'categorical_crossentropy',
'optimizer': 'adam',
'metrics': ['accuracy']
})
# Fit the classifier
classifier.fit(X_train, Y_train, epochs=1, batch_size=BATCH_SIZE)
act_config = classifier.model.layers[1].get_config()
self.assertEqual(act_config["alpha"], 1)
self.assertEqual(act_config["max_value"], 2)
def test_mnist(self):
session = tf.Session()
keras.backend.set_session(session)
comp_params = {
"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']
}
# get MNIST
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train, X_test, Y_test = X_train[:
NB_TRAIN], Y_train[:
NB_TRAIN], X_test[:
NB_TEST], Y_test[:
NB_TEST]
im_shape = X_train[0].shape
# Fit the classifier
classifier = ResNet(input_shape=im_shape)
classifier.compile(comp_params)
classifier.fit(X_train, Y_train, epochs=2, batch_size=BATCH_SIZE)
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy: %.2f%%" % (scores[1] * 100))
def test_cnn_batchnorm(self):
session = tf.Session()
keras.backend.set_session(session)
# get MNIST
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train, X_test, Y_test = X_train[:
NB_TRAIN], Y_train[:
NB_TRAIN], X_test[:
NB_TEST], Y_test[:
NB_TEST]
im_shape = X_train[0].shape
classifier = CNN(im_shape, act="relu", bnorm=True)
classifier.compile({
'loss': 'categorical_crossentropy',
'optimizer': 'adam',
'metrics': ['accuracy']
})
# Fit the classifier
classifier.fit(X_train, Y_train, epochs=1, batch_size=BATCH_SIZE)
bnorm_layer = classifier.model.layers[2]
self.assertIsInstance(bnorm_layer,
keras.layers.normalization.BatchNormalization)
def test_with_preprocessing(self):
session = tf.Session()
k.set_session(session)
comp_params = {"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']}
# get MNIST
batch_size, nb_train, nb_test = 100, 1000, 100
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train = X_train[:nb_train], Y_train[:nb_train]
X_test, Y_test = X_test[:nb_test], Y_test[:nb_test]
im_shape = X_train[0].shape
# get classifier
classifier = CNN(im_shape, act="relu", defences=["featsqueeze1"])
classifier.compile(comp_params)
classifier.fit(X_train, Y_train, epochs=1, batch_size=batch_size, verbose=0)
scores = classifier.evaluate(X_train, Y_train)
print("\naccuracy on training set: %.2f%%" % (scores[1] * 100))
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
attack_params = {"verbose": 0,
"clip_min": 0.,
"clip_max": 1.,
"eps": 1.}
attack = FastGradientMethod(classifier, session)
X_train_adv = attack.generate(X_train, **attack_params)
X_test_adv = attack.generate(X_test, **attack_params)
self.assertFalse((X_train == X_train_adv).all())
self.assertFalse((X_test == X_test_adv).all())
train_y_pred = get_labels_np_array(classifier.predict(X_train_adv))
test_y_pred = get_labels_np_array(classifier.predict(X_test_adv))
self.assertFalse((Y_train == train_y_pred).all())
self.assertFalse((Y_test == test_y_pred).all())
scores = classifier.evaluate(X_train_adv, Y_train)
print('\naccuracy on adversarial train examples: %.2f%%' % (scores[1] * 100))
scores = classifier.evaluate(X_test_adv, Y_test)
print('\naccuracy on adversarial test examples: %.2f%%' % (scores[1] * 100))
def test_mnist(self):
session = tf.Session()
k.set_session(session)
comp_params = {
"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']
}
# get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train = X_train[:nb_train], Y_train[:nb_train]
X_test, Y_test = X_test[:nb_test], Y_test[:nb_test]
im_shape = X_train[0].shape
# get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train,
Y_train,
epochs=1,
batch_size=batch_size,
verbose=0)
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
df = CarliniL2Method(classifier,
sess=session,
targeted=False,
max_iterations=100,
binary_search_steps=2,
learning_rate=1e-2,
initial_const=1)
params = {
'y_val':
random_targets(Y_test,
classifier.model.get_output_shape_at(-1)[-1])
}
x_test_adv = df.generate(X_test, **params)
self.assertFalse((X_test == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((Y_test == y_pred).all())
scores = classifier.evaluate(x_test_adv, Y_test)
print('\naccuracy on adversarial examples: %.2f%%' % (scores[1] * 100))
def test_emp_robustness_mnist(self):
session = tf.Session()
K.set_session(session)
comp_params = {
"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']
}
# get MNIST
(X_train, Y_train), (_, _), _, _ = load_mnist()
X_train, Y_train = X_train[:NB_TRAIN], Y_train[:NB_TRAIN]
im_shape = X_train[0].shape
# Get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train, Y_train, epochs=1, batch_size=BATCH_SIZE)
# Compute minimal perturbations
params = {"eps_step": 1.1, "clip_min": 0., "clip_max": 1.}
emp_robust = empirical_robustness(X_train, classifier, session, "fgsm",
params)
self.assertEqual(emp_robust, 0.)
params = {
"eps_step": 1.,
"eps_max": 1.,
"clip_min": None,
"clip_max": None
}
emp_robust = empirical_robustness(X_train, classifier, session, "fgsm",
params)
self.assertAlmostEqual(emp_robust, 1., 3)
params = {
"eps_step": 0.1,
"eps_max": 0.2,
"clip_min": None,
"clip_max": None
}
emp_robust = empirical_robustness(X_train, classifier, session, "fgsm",
params)
self.assertLessEqual(emp_robust, 0.2)
def test_mnist(self):
session = tf.Session()
k.set_session(session)
comp_params = {
"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']
}
# get MNIST
batch_size, nb_train, nb_test = 100, 1000, 11
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train = X_train[:nb_train], Y_train[:nb_train]
X_test, Y_test = X_test[:nb_test], Y_test[:nb_test]
im_shape = X_train[0].shape
# get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train,
Y_train,
epochs=1,
batch_size=batch_size,
verbose=0)
# Attack
nf = NewtonFool(classifier, sess=session)
nf.set_params(max_iter=20)
x_test_adv = nf.generate(X_test)
self.assertFalse((X_test == x_test_adv).all())
y_pred = classifier.predict(X_test)
y_pred_adv = classifier.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= y_pred_adv_max).all())
scores1 = classifier.evaluate(X_test, Y_test)
print("\nAccuracy on test set: %.2f%%" % (scores1[1] * 100))
scores2 = classifier.evaluate(x_test_adv, Y_test)
print('\nAccuracy on adversarial examples: %.2f%%' %
(scores2[1] * 100))
self.assertTrue(scores1[1] != scores2[1])
def test_mnist(self):
session = tf.Session()
k.set_session(session)
comp_params = {"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']}
# get MNIST
batch_size, nb_train, nb_test = 10, 10, 10
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train = X_train[:nb_train], Y_train[:nb_train]
X_test, Y_test = X_test[:nb_test], Y_test[:nb_test]
im_shape = X_train[0].shape
# get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(X_train, Y_train, epochs=1, batch_size=batch_size, verbose=0)
scores = classifier.evaluate(X_test, Y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
attack_params = {"verbose": 2,
"clip_min": 0.,
"clip_max": 1,
"attacker": "deepfool"}
attack = UniversalPerturbation(classifier, session)
x_train_adv = attack.generate(X_train, **attack_params)
self.assertTrue((attack.fooling_rate >= 0.2) or not attack.converged)
x_test_adv = X_test + attack.v
self.assertFalse((X_test == x_test_adv).all())
train_y_pred = get_labels_np_array(classifier.predict(x_train_adv))
test_y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((Y_test == test_y_pred).all())
self.assertFalse((Y_train == train_y_pred).all())
scores = classifier.evaluate(x_train_adv, Y_train)
print('\naccuracy on adversarial train examples: %.2f%%' % (scores[1] * 100))
scores = classifier.evaluate(x_test_adv, Y_test)
print('\naccuracy on adversarial test examples: %.2f%%' % (scores[1] * 100))
def test_save_load_mlp(self):
NB_TRAIN = 100
NB_TEST = 10
comp_params = {
'loss': 'categorical_crossentropy',
'optimizer': 'adam',
'metrics': ['accuracy']
}
session = tf.Session()
keras.backend.set_session(session)
# get MNIST
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train, X_test, Y_test = X_train[:
NB_TRAIN], Y_train[:
NB_TRAIN], X_test[:
NB_TEST], Y_test[:
NB_TEST]
im_shape = X_train[0].shape
classifier = MLP(im_shape, act="brelu")
classifier.compile(comp_params)
# Fit the classifier
classifier.fit(X_train, Y_train, epochs=1, batch_size=BATCH_SIZE)
path = "./tests/save/mlp/"
# Test saving
save_classifier(classifier, path)
self.assertTrue(os.path.isfile(path + "model.json"))
self.assertTrue(os.path.getsize(path + "model.json") > 0)
self.assertTrue(os.path.isfile(path + "weights.h5"))
self.assertTrue(os.path.getsize(path + "weights.h5") > 0)
# Test loading
loaded_classifier = load_classifier(path)
scores = classifier.evaluate(X_test, Y_test)
scores_loaded = loaded_classifier.evaluate(X_test, Y_test)
self.assertAlmostEqual(scores, scores_loaded)
def test_save_load_model(self):
NB_TRAIN = 100
NB_TEST = 10
comp_params = {
"loss": "categorical_crossentropy",
"optimizer": "adam",
"metrics": ["accuracy"]
}
session = tf.Session()
keras.backend.set_session(session)
# get MNIST
(X_train, Y_train), (X_test, Y_test), _, _ = load_mnist()
X_train, Y_train, X_test, Y_test = X_train[:
NB_TRAIN], Y_train[:
NB_TRAIN], X_test[:
NB_TEST], Y_test[:
NB_TEST]
im_shape = X_train[0].shape
# Fit the classifier
classifier = ResNet(input_shape=im_shape)
classifier.compile(comp_params)
classifier.fit(X_train, Y_train, epochs=1, batch_size=BATCH_SIZE)
# Test saving
path = "./tests/save/resnet/"
save_classifier(classifier, path)
self.assertTrue(os.path.isfile(path + "model.json"))
self.assertTrue(os.path.getsize(path + "model.json") > 0)
self.assertTrue(os.path.isfile(path + "weights.h5"))
self.assertTrue(os.path.getsize(path + "weights.h5") > 0)
# Test loading
loaded_classifier = load_classifier(path)
scores = classifier.evaluate(X_test, Y_test)
scores_loaded = loaded_classifier.evaluate(X_test, Y_test)
self.assertAlmostEqual(scores, scores_loaded)
def test_mnist_targeted(self):
session = tf.Session()
k.set_session(session)
comp_params = {"loss": 'categorical_crossentropy',
"optimizer": 'adam',
"metrics": ['accuracy']}
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
im_shape = x_train[0].shape
# Get classifier
classifier = CNN(im_shape, act="relu")
classifier.compile(comp_params)
classifier.fit(x_train, y_train, epochs=1, batch_size=batch_size, verbose=0)
scores = classifier.evaluate(x_test, y_test)
print("\naccuracy on test set: %.2f%%" % (scores[1] * 100))
# Generate random target classes
import numpy as np
nb_classes = np.unique(np.argmax(y_test, axis=1)).shape[0]
targets = np.random.randint(nb_classes, size=nb_test)
while (targets == np.argmax(y_test, axis=1)).any():
targets = np.random.randint(nb_classes, size=nb_test)
# Perform attack
df = SaliencyMapMethod(classifier, sess=session, clip_min=0, clip_max=1, theta=1)
x_test_adv = df.generate(x_test, y_val=targets)
self.assertFalse((x_test == x_test_adv).all())
y_pred = get_labels_np_array(classifier.predict(x_test_adv))
self.assertFalse((y_test == y_pred).all())
scores = classifier.evaluate(x_test_adv, y_test)
print('\naccuracy on adversarial examples: %.2f%%' % (scores[1] * 100))
# coding: utf-8
# In[29]:
import lasagne
from lasagne import layers
from src.utils import load_mnist, init_constant, init_diagnal, shared_dataset
#load_mnist, init_constant, init_diagnal, shared_dataset
import numpy as np
import theano
import theano.tensor as T
import time
from MGULayer import MGULayer
# In[39]:
train_set, valid_set, test_set = load_mnist('../data/mnist.pkl.gz')
print valid_set[0].shape
print test_set[0].shape
train_x, train_y = train_set
valid_x, valid_y = valid_set
test_x, test_y = test_set
# In[43]:
seq_len = 784
feature_num = 1
hidden_unit = 100
train_x = train_x.reshape((-1, seq_len, feature_num))
valid_x = valid_x.reshape((-1, seq_len, feature_num))
test_x = test_x.reshape((-1, seq_len, feature_num))
