def mnist_tutorial_jsma(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
viz_enabled=True,
nb_epochs=6,
batch_size=128,
nb_classes=10,
source_samples=100,
learning_rate=0.001):
"""
MNIST tutorial for the Jacobian-based saliency map approach (JSMA)
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# MNIST-specific dimensions
img_rows = 28
img_cols = 28
channels = 1
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session and set as Keras backend session
sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get MNIST test data
X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
y = tf.placeholder(tf.float32, shape=(None, 10))
# Define TF model graph
model = make_basic_cnn()
preds = model.get_probs(x)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
s = []
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
sess.run(tf.global_variables_initializer())
rng = np.random.RandomState([2017, 8, 30])
model_train(sess,
x,
y,
preds,
X_train,
Y_train,
args=train_params,
rng=rng)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, X_test, Y_test, args=eval_params)
assert X_test.shape[0] == test_end - test_start, X_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
'''
for i in range(0,len(X_test),1):
pred = sess.run(preds, {x:X_test[i:i+1]})
# print(pred)
# print(Y_test[i:i+1])
s.append(np.sort(pred)[0,-1]-np.sort(pred)[0,-2])
#Draw a histogram
def draw_hist(myList,Title,Xlabel,Ylabel):
plt.hist(myList,np.arange(0,1,0.01),normed=True,stacked=True,facecolor='blue')
plt.xlabel(Xlabel)
plt.ylabel(Ylabel)
plt.title(Title)
plt.show()
draw_hist(myList=s,Title='legitimate',Xlabel='difference between max and second largest',
Ylabel='Probability')
'''
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes - 1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples), dtype='f')
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols, channels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, back='tf', sess=sess)
jsma_params = {
'theta': 1.,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
figure = None
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in xrange(0, source_samples):
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, source_samples))
sample = X_test[sample_ind:(sample_ind + 1)]
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(Y_test[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
sample, (img_rows, img_cols, channels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
path = os.path.dirname(__file__)
file = path + "/data/jsma.npy"
np.save(file, adv_x)
preds_adv = model.get_probs(adv_x)
pred = sess.run(preds_adv, {x: sample})
print(pred)
print(Y_test[sample_ind])
#difference array s
s.append(np.sort(pred)[0, -1] - np.sort(pred)[0, -2])
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# Computer number of modified features
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = X_test[sample_ind].reshape(-1)
nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
# Add our adversarial example to our grid data
grid_viz_data[target, current_class, :, :, :] = np.reshape(
adv_x, (img_rows, img_cols, channels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
#Draw a histogram
def draw_hist(myList, Title, Xlabel, Ylabel):
plt.hist(myList,
np.arange(0, 1, 0.01),
normed=True,
stacked=True,
facecolor='red')
plt.xlabel(Xlabel)
plt.ylabel(Ylabel)
plt.title(Title)
plt.show()
draw_hist(myList=s,
Title='adversarial',
Xlabel='difference between max and second largest',
Ylabel='Probability')
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
# Close TF session
sess.close()
return report
def effective_train_jsma(train_start=0,
train_end=50,
test_start=0,
test_end=500,
viz_enabled=False,
nb_epochs=6,
batch_size=128,
nb_classes=10,
source_samples=10,
learning_rate=0.001):
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Get MNIST test data
X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
# Create TF session and set as Keras backend session
sess = tf.Session()
print("Created TensorFlow session.")
# sess.run(tf.global_variables_initializer())
rng = np.random.RandomState([2017, 8, 30])
# Define input TF placeholder
x1 = tf.placeholder(tf.float32, shape=(None, 28, 28, 1)) # for clean data
x2 = tf.placeholder(tf.float32, shape=(None, 28, 28, 1)) # for adv data
y = tf.placeholder(tf.float32, shape=(None, 10)) # for adv clean targets
# Initialize the model
model = make_basic_cnn()
preds = model(x1)
preds_adv = model(x2)
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, back='tf', sess=sess)
jsma_params = {
'theta': 1.,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
# Define loss
loss = (model_loss(y, preds) + model_loss(y, preds_adv)) / 2
train_step = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_step = train_step.minimize(loss)
def evaluate_2(adv_examples_last_batch, adv_clean_labels_last_batch):
# Accuracy of adversarially trained model on legitimate test inputs
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess,
x1,
y,
preds,
X_test,
Y_test,
args=eval_params)
print('Test accuracy on legitimate examples: %0.4f' % accuracy)
report.adv_train_clean_eval = accuracy
# Accuracy of the adversarially trained model on adversarial examples
accuracy = model_eval(sess,
x2,
y,
preds_adv,
adv_examples_last_batch,
adv_clean_labels_last_batch,
args=eval_params)
print('Test accuracy on last batch of adversarial examples: %0.4f' %
accuracy)
report.adv_train_adv_eval = accuracy
with sess.as_default():
tf.global_variables_initializer().run()
for epoch in xrange(nb_epochs):
# Compute number of batches
nb_batches = int(math.ceil(float(len(X_train)) / batch_size))
assert nb_batches * batch_size >= len(X_train)
# Indices to shuffle training set
index_shuf = list(range(len(X_train)))
rng.shuffle(index_shuf)
prev = time.time()
for batch in range(nb_batches):
print('--------------------------------------')
# create an array for storing adv examples
print('batch: %i/%i' % (batch + 1, nb_batches))
adv_examples = np.empty([1, 28, 28, 1])
# for target labels
#adv_targets = np.empty([1,10])
# corresponding clean/correct label
adv_clean_labels = np.empty([1, 10])
# correspongding clean data
adv_clean_examples = np.empty([1, 28, 28, 1])
for sample_ind in xrange(0, batch_size):
print('Attacking input %i/%i' %
(sample_ind + 1, batch_size))
# Compute batch start and end indices
start, end = batch_indices(batch, len(X_train), batch_size)
X_this_batch = X_train[index_shuf[start:end]]
Y_this_batch = Y_train[index_shuf[start:end]]
# Perform one training step
# feed_dict = {x: X_train[index_shuf[start:end]],y: Y_train[index_shuf[start:end]]}
sample = X_this_batch[sample_ind:(
sample_ind + 1)] # generate from training data
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(Y_this_batch[sample_ind])
) # generate from training data
target_classes = other_classes(nb_classes, current_class)
print('Current class is ', current_class)
# For the grid visualization, keep original images along the diagonal
# grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
# sample, (img_rows, img_cols, channels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' %
target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes),
dtype=np.float32)
#create fake target
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(
sample, **jsma_params
) # get numpy array (1, 28, 28, 1), not Tensor
# Check if success was achieved
# res = int(model_argmax(sess, x, preds, adv_x) == target)
# if succeeds
# if res == 1:
# append new adv_x to adv_examples array
# append sample here, so that the number of times sample is appended mmatches number of adv_ex.
adv_examples = np.append(adv_examples, adv_x, axis=0)
#adv_targets = np.append(adv_targets, one_hot_target, axis=0)
adv_clean_labels = np.append(
adv_clean_labels,
np.expand_dims(Y_this_batch[sample_ind], axis=0),
axis=0) # generate from training data
adv_clean_examples = np.append(adv_clean_examples,
sample,
axis=0)
# what we have for this batch, batch_size * 9 data
adv_examples = adv_examples[1:, :, :, :]
#adv_targets = adv_targets[1:,:]
adv_clean_labels = adv_clean_labels[1:, :]
adv_clean_examples = adv_clean_examples[1:, :, :, :]
feed_dict = {
x1: adv_clean_examples,
x2: adv_examples,
y: adv_clean_labels
}
train_step.run(feed_dict=feed_dict)
cur = time.time()
_logger.info("Epoch " + str(epoch) + " took " + str(cur - prev) +
" seconds")
evaluate_2(adv_examples, adv_clean_labels)
print('Training finished.')
# report on clean test data
preds_test = model(x1)
eval_par = {'batch_size': 10}
acc_clean = model_eval(sess,
x1,
y,
preds_test,
X_test,
Y_test,
args=eval_par)
print('Test accuracy on legitimate examples: %0.4f\n' % acc_clean)
# reload fgsm successfully attacking adv test data
with np.load("adversarial_fgsm.npz") as data:
adv_X_test, adv_clean_Y_test, adv_clean_X_test = data[
'adv_examples'], data['adv_clean_labels'], data[
'adv_clean_examples']
print('FGSM adversarial data are successfully reloaded.')
preds_adv_test = model(x1)
# Evaluate the accuracy of the MNIST model on adversarial examples
# eval_par = {'batch_size': 10}
acc = model_eval(sess,
x1,
y,
preds_adv_test,
adv_X_test,
adv_clean_Y_test,
args=eval_par)
print(
'Test accuracy on pre-generated adversarial examples of fgsm: %0.4f\n'
% acc)
# reload fgsm successfully attacking adv test data
with np.load("adversarial_mnist_test_from_1500.npz") as data:
adv_X_test, adv_clean_Y_test, adv_clean_X_test = data[
'adv_examples'], data['adv_clean_labels'], data[
'adv_clean_examples']
print('JSMA adversarial data are successfully reloaded.')
# Evaluate the accuracy of the MNIST model on adversarial examples
acc2 = model_eval(sess,
x1,
y,
preds_adv_test,
adv_X_test,
adv_clean_Y_test,
args=eval_par)
print(
'Test accuracy on pre-generated adversarial examples of jsma: %0.4f\n'
% acc2)
# Close TF session
sess.close()
def main(argv=None):
"""
MNIST cleverhans tutorial for the Jacobian-based saliency map approach (JSMA)
:return:
"""
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
###########################################################################
# Define the dataset and model
###########################################################################
# Image dimensions ordering should follow the Theano convention
if keras.backend.image_dim_ordering() != 'th':
keras.backend.set_image_dim_ordering('th')
print(
"INFO: '~/.keras/keras.json' sets 'image_dim_ordering' to 'tf', temporarily setting to 'th'"
)
# Create TF session and set as Keras backend session
sess = tf.Session()
keras.backend.set_session(sess)
print("Created TensorFlow session and set Keras backend.")
# Get MNIST test data
X_train, Y_train, X_test, Y_test = data_mnist()
print("Loaded MNIST test data.")
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, 1, 28, 28))
y = tf.placeholder(tf.float32, shape=(None, 10))
# Define TF model graph
model = model_mnist()
predictions = model(x)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model if it does not exist in the train_dir folder
saver = tf.train.Saver()
save_path = os.path.join(FLAGS.train_dir, FLAGS.filename)
if os.path.isfile(save_path):
saver.restore(sess, os.path.join(FLAGS.train_dir, FLAGS.filename))
else:
tf_model_train(sess, x, y, predictions, X_train, Y_train)
saver.save(sess, save_path)
# Evaluate the accuracy of the MNIST model on legitimate test examples
accuracy = tf_model_eval(sess, x, y, predictions, X_test, Y_test)
assert X_test.shape[0] == 10000, X_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(FLAGS.source_samples) + ' * ' +
str(FLAGS.nb_classes) + ' adversarial examples')
# This array indicates whether an adversarial example was found for each
# test set sample and target class
results = np.zeros((FLAGS.nb_classes, FLAGS.source_samples), dtype='i')
# This array contains the fraction of perturbed features for each test set
# sample and target class
perturbations = np.zeros((FLAGS.nb_classes, FLAGS.source_samples),
dtype='f')
# Define the TF graph for the model's Jacobian
grads = jacobian_graph(predictions, x)
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in xrange(FLAGS.source_samples):
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
target_classes = other_classes(FLAGS.nb_classes,
int(np.argmax(Y_test[sample_ind])))
# Loop over all target classes
for target in target_classes:
print('--------------------------------------')
print('Creating adversarial example for target class ' +
str(target))
# This call runs the Jacobian-based saliency map approach
_, result, percentage_perterb = jsma(
sess,
x,
predictions,
grads,
X_test[sample_ind:(sample_ind + 1)],
target,
theta=1,
gamma=0.1,
increase=True,
back='tf',
clip_min=0,
clip_max=1)
# Update the arrays for later analysis
results[target, sample_ind] = result
perturbations[target, sample_ind] = percentage_perterb
# Compute the number of adversarial examples that were successfuly found
success_rate = float(np.sum(results)) / (
(FLAGS.nb_classes - 1) * FLAGS.source_samples)
print('Avg. rate of successful misclassifcations {0}'.format(success_rate))
# Compute the average distortion introduced by the algorithm
percentage_perturbed = np.mean(perturbations)
print('Avg. rate of perterbed features {0}'.format(percentage_perturbed))
# Close TF session
sess.close()
def test_other_classes_return_val(self):
res = utils.other_classes(5, 2)
res_expected = [0, 1, 3, 4]
self.assertTrue(res == res_expected)
def mnist_tutorial_jsma(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
viz_enabled=True,
nb_epochs=6,
batch_size=128,
nb_classes=10,
source_samples=10,
learning_rate=0.001):
"""
MNIST tutorial for the Jacobian-based saliency map approach (JSMA)
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# MNIST-specific dimensions
img_rows = 28
img_cols = 28
channels = 1
# Set TF random seed to improve reproducibility
tf.set_random_seed(4254264)
set_log_level(logging.DEBUG)
# Get MNIST test data
# X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,
# train_end=train_end,
# test_start=test_start,
# test_end=test_end)
# Get notMNIST data
# with np.load("notmnist.npz") as data:
# X_train, Y_train, X_test, Y_test = data['examples_train'], data['labels_train'], data['examples_test'], data['labels_test']
# Get MNISTnotMNIST data
with np.load("mnist.npz") as data:
X_train, Y_train, X_test, Y_test = data['X_train'], data[
'Y_train'], data['X_test'], data['Y_test']
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
y = tf.placeholder(tf.float32, shape=(None, 10))
# Create TF session and set as Keras backend session
sess = tf.Session()
print("Created TensorFlow session.")
# Define TF model graph
model_path = "./"
model_name = "clean_trained_mnist_model"
model = make_basic_cnn(nb_classes=nb_classes)
if tf_model_load(sess, file_path=os.path.join(model_path, model_name)):
print(model_name, " reloaded.")
preds = model.get_probs(x)
# print('shape is', preds.get_shape())
# clean_train = True
# if clean_train:
# train_params = {
# 'nb_epochs': nb_epochs,
# 'batch_size': batch_size,
# 'learning_rate': learning_rate
# }
# model_path = "./"
# model_name = "clean_trained__model_notmnist"
# rng = np.random.RandomState([1989, 12, 13])
# model = make_basic_cnn()
# preds = model.get_probs(x)
#
# def evaluate():
# # Evaluate the accuracy of the MNIST model on legitimate test
# # examples
# eval_params = {'batch_size': batch_size}
# acc = model_eval(
# sess, x, y, preds, X_test, Y_test, args=eval_params)
# report.clean_train_clean_eval = acc
# assert X_test.shape[0] == test_end - test_start, X_test.shape
# print('Test accuracy on legitimate examples: %0.4f' % acc)
# model_train(sess, x, y, preds, X_train, Y_train, evaluate=evaluate,args=train_params, rng=rng)
#
# save_path = os.path.join(model_path, model_name)
# saver = tf.train.Saver()
# saver.save(sess, save_path)
# _logger.info("Completed model training and saved at: " + str(save_path))
# print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model
# train_params = {
# 'nb_epochs': nb_epochs,
# 'batch_size': batch_size,
# 'learning_rate': learning_rate,
# 'train_dir': model_path,
# 'filename': model_name
# }
# sess.run(tf.global_variables_initializer())
# rng = np.random.RandomState([2017, 8, 30])
# model_train(sess, x, y, preds, X_train, Y_train, save=True, args=train_params,
# rng=rng)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, X_test, Y_test, args=eval_params)
assert X_test.shape[0] == test_end - test_start, X_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
# report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes - 1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i')
# misclassify
results2 = np.zeros((nb_classes, source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples), dtype='f')
# Initialize our array for grid visualization
# grid_shape = (nb_classes, nb_classes, img_rows, img_cols, channels)
# grid_viz_data = np.zeros(grid_shape, dtype='f')
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, back='tf', sess=sess)
jsma_params = {
'theta': 1,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
figure = None
rng = np.random.RandomState([1358, 23, 234])
index_shuf = list(range(len(X_test)))
rng.shuffle(index_shuf)
X_test = X_test[index_shuf]
Y_test = Y_test[index_shuf]
# create a dictionary to keep track of occurence of each letter
# create a 2D array to kee track of successful attacks
occurence = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0}
# 10:0, 11:0, 12:0, 13:0, 14:0, 15:0, 16:0, 17:0, 18:0, 19:0}
rate_table = np.zeros((nb_classes, nb_classes), dtype='f')
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in xrange(0, source_samples):
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, source_samples))
sample = X_test[sample_ind:(sample_ind + 1)]
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(Y_test[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# add one to current class occurence
occurence[current_class] += 1
# For the grid visualization, keep original images along the diagonal
# grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
# sample, (img_rows, img_cols, channels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# misclassify
res2 = int(model_argmax(sess, x, preds, adv_x) != current_class)
# if success, add one to successful rate table
if res == 1:
rate_table[current_class, target] += 1.
# Computer number of modified features
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = X_test[sample_ind].reshape(-1)
nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
# Display the original and adversarial images side-by-side
# if viz_enabled:
# figure = pair_visual(
# np.reshape(sample, (img_rows, img_cols)),
# np.reshape(adv_x, (img_rows, img_cols)), figure)
# Add our adversarial example to our grid data
# grid_viz_data[target, current_class, :, :, :] = np.reshape(
# adv_x, (img_rows, img_cols, channels))
# Update the arrays for later analysis
results[target, sample_ind] = res
results2[target, sample_ind] = res2
perturbations[target, sample_ind] = percent_perturb
print('--------------------------------------')
# Close TF session
sess.close()
# Compute success rate of each letter attacking each target
for cur in range(nb_classes):
if occurence[cur] != 0:
rate_table[cur, :] /= float(occurence[cur])
print("The table of rate of successful attacking is shown below")
print(rate_table)
print("the number of occurrence of each class is ", occurence)
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
# misclassify
succ_rate2 = float(np.sum(results2)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(succ_rate))
print(
'Avg. rate of misclassified adv. examples {0:.4f}'.format(succ_rate2))
# report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
# Finally, block & display a grid of all the adversarial examples
# if viz_enabled:
# import matplotlib.pyplot as plt
# plt.close(figure)
# _ = grid_visual(grid_viz_data)
return report
def generate_attacks(save_path,
file_path,
dataset,
x_set,
y_set,
attack,
gamma,
first_index,
last_index,
batch_size=1):
"""
Applies the voting saliency map attack against the specified model in targeted mode.
Parameters
----------
save_path: str
The path of the folder in which the crafted adversarial samples will be saved.
file_path: str
The path to the joblib file of the model to attack.
x_set: numpy.ndarray
The dataset input array.
y_set: numpy.ndarray
The dataset output array.
attack: str
The type of used attack (either "jsma", "wjsma" or "tjsma").
gamma: float
Maximum percentage of perturbed features.
first_index:
The index of the first image attacked.
last_index: int
The index of the last image attacked.
batch_size: int
The size of the image batches.
"""
if not os.path.exists(save_path):
os.makedirs(save_path)
sess = tf.Session()
img_rows, img_cols, channels = x_set.shape[1:4]
nb_classes = y_set.shape[1]
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, channels))
with sess.as_default():
print(file_path)
if dataset == "mnist":
model = MNISTModel(file_path) # load(file_path)
else:
model = CIFARModel(file_path)
jsma = SaliencyMapMethod(model, sess=sess)
jsma_params = {
'theta': 1,
'gamma': gamma,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None,
'attack': attack
}
preds = model(x)
y_set = np.argmax(y_set, axis=1).astype(int)
indices = range(first_index, last_index)
batch_indices = [
indices[t * batch_size:batch_size * (t + 1)] for t in range(
len(indices) // batch_size + (len(indices) % batch_size != 0))
]
sample_count = last_index - first_index
sample_crafted = 0
ori_points = np.zeros(10)
pixel_points = np.zeros(10)
per_pixels = np.zeros(10)
start_time = time.time()
fake_pixels = np.zeros(10)
for batch in batch_indices:
samples = []
sample_classes = []
current_class_batch = []
target_classes_batch = []
for sample_index in batch:
sample = x_set[sample_index]
current_class = y_set[sample_index]
target_classes = other_classes(nb_classes, current_class)
current_class_batch.append(current_class)
target_classes_batch += target_classes
samples.append(
np.repeat(sample.reshape((1, ) + sample.shape), 9, axis=0))
y_target = np.zeros((len(target_classes), nb_classes))
y_target[np.arange(len(target_classes)), target_classes] = 1
sample_classes.append(y_target)
samples = np.concatenate(samples)
sample_classes = np.concatenate(sample_classes)
jsma_params['y_target'] = sample_classes
adversarial_batch = jsma.generate_np(samples, **jsma_params)
for index, sample_index in zip(range(len(batch)), batch):
results = pd.DataFrame()
adversarial_samples = adversarial_batch[index * (nb_classes -
1):(index + 1) *
(nb_classes - 1)]
current_class = current_class_batch[index]
target_classes = target_classes_batch[index * (nb_classes -
1):(index + 1) *
(nb_classes - 1)]
ori_points[current_class] += 1
for target, adv_sample in zip(target_classes, adversarial_samples):
adv_sample = adv_sample.reshape(
(1, img_rows, img_cols, channels)).astype(np.float32)
feed_dict = {x: adv_sample}
probabilities = sess.run(preds, feed_dict)
if adv_sample.shape[0] == 1:
res = np.argmax(probabilities)
else:
res = np.argmax(probabilities, axis=1)
res = int(res == target)
if res == 0:
fake_pixels[target] += 1
adv_x_reshape = adv_sample.reshape(-1)
test_in_reshape = x_set[sample_index].reshape(-1)
#nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
perturbations = np.sum(np.abs(adv_x_reshape - test_in_reshape))
nb_changed = np.where(
np.abs(adv_x_reshape - test_in_reshape) > 1 /
255.)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x_reshape.shape[0]
if res:
pixel_points[target] += nb_changed
per_pixels[target] += perturbations
#results['number_' + str(sample_index) + '_' + str(current_class) + '_to_' + str(target)] = np.concatenate([adv_x_reshape.reshape(-1), np.array([nb_changed, percent_perturb, res])])
sample = samples[index * (nb_classes - 1)]
#results['original_image_' + str(sample_index)] = np.concatenate([sample.reshape(-1), np.zeros((3,))])
print("ori_points", ori_points)
print("fake_pixels", fake_pixels)
print("pixel_points", pixel_points)
print("per_pixels", per_pixels)
#results.to_csv(save_path + '/' + attack + '_image_' + str(sample_index) + '.csv', index=False)
sample_crafted += len(batch)
print("Done: ", sample_crafted, "/", sample_count)
print(time.time() - start_time)
def test_other_classes_neg_class_ind(self):
with self.assertRaises(Exception) as context:
utils.other_classes(10, -1)
self.assertTrue(context.exception)
def mnist_tutorial_jsma(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
viz_enabled=VIZ_ENABLED,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
source_samples=SOURCE_SAMPLES,
learning_rate=LEARNING_RATE):
"""
MNIST tutorial for the Jacobian-based saliency map approach (JSMA)
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session and set as Keras backend session
#replace
num_threads = None
if num_threads:
config_args = dict(intra_op_parallelism_threads=1)
else:
config_args = {}
sess = tf.Session(config=tf.ConfigProto(**config_args))
#with sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get MNIST test data
mnist = MNIST(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
x_train, y_train = mnist.get_set('train')
x_test, y_test = mnist.get_set('test')
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
nb_filters = 64
# Define TF model graph
model = make_basic_picklable_cnn()
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
dataset = tf.data.Dataset.from_tensor_slices(
(tf.reshape(x_train, [60000, 28, 28]), y_train))
dataset = dataset.batch(32)
val_dataset = tf.data.Dataset.from_tensor_slices(
(tf.reshape(x_test, [10000, 28, 28]), y_test))
val_dataset = val_dataset.batch(32)
sess.run(tf.global_variables_initializer())
rng = np.random.RandomState([2017, 8, 30])
if TRAIN_NEW == 1:
with sess.as_default():
train(sess, loss, x_train, y_train, args=train_params, rng=rng)
save("test.joblib", model)
else:
with sess.as_default():
model = load("test.joblib") #changed
assert len(model.get_params()) > 0
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
assert x_test.shape[0] == test_end - test_start, x_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes - 1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples), dtype='f')
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols, nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, sess=sess)
jsma_params = {
'theta': 1.,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
figure = None
# Loop over the samples we want to perturb into adversarial examples
seed(SEED)
for sample_ind in xrange(0, source_samples):
img = randint(0, 10000)
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, source_samples))
sample = x_test[img:(img +
1)] #sample = x_test[sample_ind:(sample_ind + 1)]
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(
y_test[img])) #current_class = int(np.argmax(y_test[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
sample, (img_rows, img_cols, nchannels))
tn = 0
totc = 0
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# Compute number of modified features
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = x_test[sample_ind].reshape(-1)
nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
diff = np.array(adv_x - sample)
#print(np.sum(diff))
diff = np.reshape(diff, (28, 28))
diff = diff * 255
cv2.imwrite("test.png", diff)
diff = cv2.imread("test.png")
diff = cv2.cvtColor(diff, cv2.COLOR_BGR2GRAY)
nieghbors = 0
tc = 0
for i in range(0, 28, 1):
for j in range(0, 28, 1):
if diff[i, j] > 0:
tc = tc + 1
totc = totc + 1
if i > 0 and i < 27 and j > 0 and j < 27: #main grid not edges or corners
if diff[i - 1, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j + 1] > 0:
nieghbors = nieghbors + 1
else:
#corners
if i == 0 and j == 0:
if diff[i, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j] > 0:
nieghbors = nieghbors + 1
if i == 27 and j == 0:
if diff[i, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j] > 0:
nieghbors = nieghbors + 1
if i == 0 and j == 27:
if diff[i, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j] > 0:
nieghbors = nieghbors + 1
if i == 27 and j == 27:
if diff[i, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j] > 0:
nieghbors = nieghbors + 1
#edges
if i == 0 and j > 0 and j < 27: #left side
if diff[i, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j + 1] > 0:
nieghbors = nieghbors + 1
if i == 27 and j > 0 and j < 27: #right side
if diff[i, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j + 1] > 0:
nieghbors = nieghbors + 1
if j == 0 and i > 0 and i < 27: #top side
if diff[i - 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j + 1] > 0:
nieghbors = nieghbors + 1
if j == 27 and i > 0 and i < 27: #bot side
if diff[i - 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j - 1] > 0:
nieghbors = nieghbors + 1
# print(tc)
# print(nieghbors)
tn = tn + nieghbors
# if tc > 0:
# print(nieghbors/tc)
# Display the original and adversarial images side-by-side
if viz_enabled:
figure = pair_visual(
np.reshape(sample, (img_rows, img_cols, nchannels)),
np.reshape(adv_x, (img_rows, img_cols, nchannels)), figure)
# Add our adversarial example to our grid data
grid_viz_data[target, current_class, :, :, :] = np.reshape(
adv_x, (img_rows, img_cols, nchannels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
#print(perturbations[target, sample_ind])
print('--------------------------------------')
print("average neighbors per modified pixel ", tn / totc)
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.8f}'.format(succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
s = perturbations.shape
myPert = np.empty(0)
myResults = np.empty(0)
for i in range(s[0]):
for j in range(s[1]):
if perturbations[i][j] > 0:
myPert = np.append(myPert, perturbations[i][j])
myResults = np.append(myResults, results[i][j])
min_perturbed = np.min(myPert)
max_perturbed = np.max(myPert)
s2 = myResults.shape
final = np.empty(0)
for i in range(s2[0]):
if myResults[i] > 0:
final = np.append(final, myPert[i])
print('Avg. rate of perturbed features {0:.8f}'.format(percent_perturbed))
print('MIN of perturbed features {0:.8f}'.format(min_perturbed))
print('MAX of perturbed features {0:.8f}'.format(max_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
min_perturb_succ = np.min(final)
max_perturb_succ = np.max(final)
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.8f}'.format(percent_perturb_succ))
print('Min of perturbed features for successful '
'adversarial examples {0:.8f}'.format(min_perturb_succ))
print('Max of perturbed features for successful '
'adversarial examples {0:.8f}'.format(max_perturb_succ))
#Close TF session
sess.close()
# Finally, block & display a grid of all the adversarial examples
if viz_enabled:
import matplotlib.pyplot as plt
plt.close(figure)
_ = grid_visual(grid_viz_data)
return report
def main(argv=None):
"""
CIFAR10 CleverHans tutorial
:return:
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# CIFAR10-specific dimensions
img_rows = 32
img_cols = 32
channels = 3
nb_classes = 10
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
sess = tf.Session()
set_log_level(logging.DEBUG)
# Get CIFAR10 test data
X_train, Y_train, X_test, Y_test = data_cifar10()
# Label smoothing
assert Y_train.shape[1] == 10.
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, channels))
y = tf.placeholder(tf.float32, shape=(None, 10))
model_path = FLAGS.model_path
nb_samples = FLAGS.nb_samples
from cnn_models import make_basic_cnn
model = make_basic_cnn('fp_',
input_shape=(None, img_rows, img_cols, channels),
nb_filters=FLAGS.nb_filters)
preds = model(x)
print("Defined TensorFlow model graph with %d parameters" % model.n_params)
rng = np.random.RandomState([2017, 8, 30])
def evaluate(eval_params):
# Evaluate the model on legitimate test examples
acc = model_eval(sess, x, y, preds, X_test, Y_test, args=eval_params)
return acc
model_load(sess, model_path)
print('Restored model from %s' % model_path)
eval_params = {'batch_size': FLAGS.batch_size}
accuracy = evaluate(eval_params)
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(nb_samples) + ' * ' + str(nb_classes - 1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, nb_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, nb_samples), dtype='f')
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols, channels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
from cleverhans.attacks import SaliencyMapMethod
jsma = SaliencyMapMethod(model, sess=sess)
jsma_params = {
'gamma': FLAGS.gamma,
'theta': 1.,
'symbolic_impl': True,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
figure = None
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in range(0, nb_samples):
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, nb_samples))
sample = X_test[sample_ind:(sample_ind + 1)]
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(Y_test[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
sample, (img_rows, img_cols, channels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# Computer number of modified features
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = X_test[sample_ind].reshape(-1)
nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
# Display the original and adversarial images side-by-side
if FLAGS.viz_enabled:
figure = pair_visual(
np.reshape(sample, (img_rows, img_cols, channels)),
np.reshape(adv_x, (img_rows, img_cols, channels)), figure)
# Add our adversarial example to our grid data
grid_viz_data[target, current_class, :, :, :] = np.reshape(
adv_x, (img_rows, img_cols, channels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * nb_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
# Close TF session
sess.close()
# Finally, block & display a grid of all the adversarial examples
if FLAGS.viz_enabled:
import matplotlib.pyplot as plt
plt.close(figure)
_ = grid_visual(grid_viz_data)
def mnist_tutorial_jsma(train_start=0, train_end=60000, test_start=0,
test_end=10000, viz_enabled=True, nb_epochs=6,
batch_size=128, nb_classes=10, source_samples=10,
learning_rate=0.001):
"""
MNIST tutorial for the Jacobian-based saliency map approach (JSMA)
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# MNIST-specific dimensions
img_rows = 28
img_cols = 28
channels = 1
# Disable Keras learning phase since we will be serving through tensorflow
keras.layers.core.K.set_learning_phase(0)
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Image dimensions ordering should follow the TensorFlow convention
if keras.backend.image_dim_ordering() != 'tf':
keras.backend.set_image_dim_ordering('tf')
print("INFO: '~/.keras/keras.json' sets 'image_dim_ordering' "
"to 'th', temporarily setting to 'tf'")
# Create TF session and set as Keras backend session
sess = tf.Session()
keras.backend.set_session(sess)
print("Created TensorFlow session and set Keras backend.")
set_log_level(logging.DEBUG)
# Get MNIST test data
X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
y = tf.placeholder(tf.float32, shape=(None, 10))
# Define TF model graph
model = cnn_model()
preds = model(x)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
sess.run(tf.global_variables_initializer())
model_train(sess, x, y, preds, X_train, Y_train, args=train_params)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, X_test, Y_test, args=eval_params)
assert X_test.shape[0] == test_end - test_start, X_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes-1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples), dtype='f')
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols, channels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Instantiate a SaliencyMapMethod attack object
wrap = KerasModelWrapper(model)
jsma = SaliencyMapMethod(wrap, back='tf', sess=sess)
jsma_params = {'theta': 1., 'gamma': 0.1,
'clip_min': 0., 'clip_max': 1.,
'y_target': None}
figure = None
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in xrange(0, source_samples):
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, source_samples))
sample = X_test[sample_ind:(sample_ind+1)]
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(Y_test[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
sample, (img_rows, img_cols, channels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# Computer number of modified features
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = X_test[sample_ind].reshape(-1)
nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
# Display the original and adversarial images side-by-side
if viz_enabled:
figure = pair_visual(
np.reshape(sample, (img_rows, img_cols)),
np.reshape(adv_x, (img_rows, img_cols)), figure)
# Add our adversarial example to our grid data
grid_viz_data[target, current_class, :, :, :] = np.reshape(
adv_x, (img_rows, img_cols, channels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
# Close TF session
sess.close()
# Finally, block & display a grid of all the adversarial examples
if viz_enabled:
import matplotlib.pyplot as plt
plt.close(figure)
_ = grid_visual(grid_viz_data)
return report
def mnist_tutorial_jsma(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
viz_enabled=False,
nb_epochs=6,
batch_size=128,
nb_classes=10,
source_samples=10,
learning_rate=0.001):
"""
MNIST tutorial for the Jacobian-based saliency map approach (JSMA)
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# MNIST-specific dimensions
img_rows = 28
img_cols = 28
channels = 1
# Set TF random seed to improve reproducibility
tf.set_random_seed(7076)
# Create TF session and set as Keras backend session
sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get MNIST test data
X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
y = tf.placeholder(tf.float32, shape=(None, 10))
# Define TF model graph
model = make_basic_cnn()
preds = model(x)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
sess.run(tf.global_variables_initializer())
rng = np.random.RandomState([2017, 8, 30])
model_train(sess,
x,
y,
preds,
X_train,
Y_train,
args=train_params,
rng=rng)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, X_test, Y_test, args=eval_params)
assert X_test.shape[0] == test_end - test_start, X_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes - 1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples), dtype='f')
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols, channels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, back='tf', sess=sess)
jsma_params = {
'theta': 1.,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
figure = None
# create an array for storing adv examples
adv_examples = np.empty([1, 28, 28, 1])
# for target labels
adv_targets = np.empty([1, 10])
# corresponding clean/correct label
adv_clean_labels = np.empty([1, 10])
# correspongding clean data
adv_clean_examples = np.empty([1, 28, 28, 1])
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in xrange(0, source_samples):
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, source_samples))
sample = X_train[sample_ind:(sample_ind +
1)] # generate from training data
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(
Y_train[sample_ind])) # generate from training data
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
# grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
# sample, (img_rows, img_cols, channels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
#create fake target
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
# print('adv_x\'shape is ', np.shape(adv_x)) # (1,28,28,1)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# if succeeds
if res == 1:
# append new adv_x to adv_examples array
# append sample here, so that the number of times sample is appended mmatches number of adv_ex.
adv_examples = np.append(adv_examples, adv_x, axis=0)
adv_targets = np.append(adv_targets, one_hot_target, axis=0)
adv_clean_labels = np.append(
adv_clean_labels,
np.expand_dims(Y_train[sample_ind], axis=0),
axis=0) # generate from training data
adv_clean_examples = np.append(adv_clean_examples,
sample,
axis=0)
# Compute the number of modified features
# adv_x.reshape(-1) means reshape into (1, n), in this case, n=28x28
# it makes comparison simplier
# adv_x_reshape = adv_x.reshape(-1)
# test_in_reshape = X_test[sample_ind].reshape(-1)
# nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
# percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
adv_x_reshape = adv_x.reshape(-1)
train_in_reshape = X_train[sample_ind].reshape(-1)
nb_changed = np.where(
adv_x_reshape != train_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
# Display the original and adversarial images side-by-side
viz_enabled = False
if viz_enabled:
figure = pair_visual(np.reshape(sample, (img_rows, img_cols)),
np.reshape(adv_x, (img_rows, img_cols)),
figure)
# Add our adversarial example to our grid data
# grid_viz_data[target, current_class, :, :, :] = np.reshape(
# adv_x, (img_rows, img_cols, channels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
print('--------------------------------------')
adv_examples = adv_examples[1:, :, :, :]
adv_targets = adv_targets[1:, :]
adv_clean_labels = adv_clean_labels[1:, :]
adv_clean_examples = adv_clean_examples[1:, :, :, :]
np.savez('adversarial',
adv_examples=adv_examples,
adv_targets=adv_targets,
adv_clean_labels=adv_clean_labels,
adv_clean_examples=adv_clean_examples)
print(np.shape(adv_targets)[0], "adversarial examples have been saved.")
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
# Close TF session
sess.close()
# Finally, block & display a grid of all the adversarial examples
if viz_enabled:
import matplotlib.pyplot as plt
plt.close(figure)
_ = grid_visual(grid_viz_data)
return report
def mnist_tutorial(train_start=0, train_end=60000, test_start=0,
test_end=10000, nb_epochs=6, batch_size=128,
learning_rate=0.001,
clean_train=True,
testing=False,
backprop_through_attack=False,
nb_filters=64):
"""
MNIST cleverhans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param clean_train: perform normal training on clean examples only
before performing adversarial training.
:param testing: if true, complete an AccuracyReport for unit tests
to verify that performance is adequate
:param backprop_through_attack: If True, backprop through adversarial
example construction process during
adversarial training.
:param clean_train: if true, train on clean examples
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
sess = tf.Session()
# Get MNIST test data
X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
# Use label smoothing
assert Y_train.shape[1] == 10
label_smooth = .1
Y_train = Y_train.clip(label_smooth / 9., 1. - label_smooth)
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
y = tf.placeholder(tf.float32, shape=(None, 10))
model_path = "models/mnist"
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
fgsm_params = {'eps': 0.3,
'clip_min': 0.,
'clip_max': 1.}
rng = np.random.RandomState([2017, 8, 30])
if clean_train:
model = make_basic_cnn(nb_filters=nb_filters)
preds = model.get_probs(x)
def evaluate():
# Evaluate the accuracy of the MNIST model on legitimate test
# examples
eval_params = {'batch_size': batch_size}
acc = model_eval(
sess, x, y, preds, X_test, Y_test, args=eval_params)
report.clean_train_clean_eval = acc
assert X_test.shape[0] == test_end - test_start, X_test.shape
print('Test accuracy on legitimate examples: %0.4f' % acc)
model_train(sess, x, y, preds, X_train, Y_train, evaluate=evaluate,
args=train_params, rng=rng)
# Calculate training error
if testing:
eval_params = {'batch_size': batch_size}
acc = model_eval(
sess, x, y, preds, X_train, Y_train, args=eval_params)
report.train_clean_train_clean_eval = acc
# Initialize the Fast Gradient Sign Method (FGSM) attack object and
# graph
fgsm = FastGradientMethod(model, sess=sess)
adv_x = fgsm.generate(x, **fgsm_params)
print(adv_x)
preds_adv = model.get_probs(adv_x)
# Evaluate the accuracy of the MNIST model on adversarial examples
eval_par = {'batch_size': batch_size}
# Define accuracy symbolically
if LooseVersion(tf.__version__) >= LooseVersion('1.0.0'):
correct_preds = tf.not_equal(tf.argmax(y, axis=-1),
tf.argmax(preds_adv, axis=-1))
else:
correct_preds = tf.not_equal(tf.argmax(y, axis=tf.rank(y) - 1),
tf.argmax(preds_adv,
axis=tf.rank(preds_adv) - 1))
# print("the shape of correct_preds is ", correct_preds.get_shape())
# correct_preds is a boolean Tensor with shape (size,)
success_adv_x = tf.boolean_mask(adv_x, correct_preds)
success_clean_x = tf.boolean_mask(x, correct_preds)
success_clean_y = tf.boolean_mask(y, correct_preds)
fgsm_adv_x, fgsm_clean_x, fgsm_clean_y = sess.run([success_adv_x, success_clean_x, success_clean_y], feed_dict={x:X_test,y:Y_test})
np.savez('adversarial_fgsm',adv_examples=fgsm_adv_x, adv_clean_labels=fgsm_clean_y, adv_clean_examples=fgsm_clean_x)
print("the shape of adversarial examples we save is ", np.shape(fgsm_adv_x))
print("the shape of clean targets we save is ", np.shape(fgsm_clean_y))
acc = model_eval(sess, x, y, preds_adv, X_test, Y_test, args=eval_par)
print('Test accuracy on adversarial examples fgsm: %0.4f\n' % acc)
report.clean_train_adv_eval = acc
adv_x_test_for_save = sess.run(adv_x, {x: X_test})
np.savez("adv_test_fgsm_data.npz", adv_examples=adv_x_test_for_save, adv_clean_labels=Y_test, adv_clean_examples=X_test)
# Calculate training error
if testing:
eval_par = {'batch_size': batch_size}
acc = model_eval(sess, x, y, preds_adv, X_train,
Y_train, args=eval_par)
report.train_clean_train_adv_eval = acc
print("Repeating the process, using adversarial training")
# Redefine TF model graph
model_2 = make_basic_cnn(nb_filters=nb_filters)
preds_2 = model_2(x)
fgsm2 = FastGradientMethod(model_2, sess=sess)
adv_x_2 = fgsm2.generate(x, **fgsm_params)
if not backprop_through_attack:
# For the fgsm attack used in this tutorial, the attack has zero
# gradient so enabling this flag does not change the gradient.
# For some other attacks, enabling this flag increases the cost of
# training, but gives the defender the ability to anticipate how
# the atacker will change their strategy in response to updates to
# the defender's parameters.
adv_x_2 = tf.stop_gradient(adv_x_2)
preds_2_adv = model_2(adv_x_2)
def evaluate_2():
# Accuracy of adversarially trained model on legitimate test inputs
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds_2, X_test, Y_test,
args=eval_params)
print('Test accuracy on legitimate examples: %0.4f' % accuracy)
report.adv_train_clean_eval = accuracy
# Accuracy of the adversarially trained model on adversarial examples
accuracy = model_eval(sess, x, y, preds_2_adv, X_test,
Y_test, args=eval_params)
print('Test accuracy on adversarial examples: %0.4f' % accuracy)
report.adv_train_adv_eval = accuracy
# Perform and evaluate adversarial training
print("pred_adv", preds_2_adv.get_shape())
model_train(sess, x, y, preds_2, X_train, Y_train,
predictions_adv=preds_2_adv, evaluate=evaluate_2,
args=train_params, rng=rng)
# Calculate training errors
if testing:
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds_2, X_train, Y_train,
args=eval_params)
report.train_adv_train_clean_eval = accuracy
accuracy = model_eval(sess, x, y, preds_2_adv, X_train,
Y_train, args=eval_params)
report.train_adv_train_adv_eval = accuracy
# Initialize the Fast Gradient Sign Method (FGSM) attack object and
# graph
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
source_samples = 10000
nb_classes = 10
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes-1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples), dtype='f')
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model_2, back='tf', sess=sess)
jsma_params = {'theta': 1., 'gamma': 0.1,
'clip_min': 0., 'clip_max': 1.,
'y_target': None}
figure = None
# create an array for storing adv examples
adv_examples = np.empty([1,28,28,1])
# for target labels
adv_targets = np.empty([1,10])
# corresponding clean/correct label
adv_clean_labels = np.empty([1,10])
# correspongding clean data
adv_clean_examples = np.empty([1,28,28,1])
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in xrange(0, source_samples):
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, source_samples))
sample = X_test[sample_ind:(sample_ind+1)] # generate from testing data
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(Y_test[sample_ind])) # generate from testing data
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
# grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
# sample, (img_rows, img_cols, channels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
#create fake target
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
# print('adv_x\'shape is ', np.shape(adv_x)) # (1,28,28,1)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# if succeeds
if res == 1:
# append new adv_x to adv_examples array
# append sample here, so that the number of times sample is appended mmatches number of adv_ex.
adv_examples = np.append(adv_examples, adv_x, axis=0)
adv_targets = np.append(adv_targets, one_hot_target, axis=0)
adv_clean_labels = np.append(adv_clean_labels, np.expand_dims(Y_test[sample_ind],axis=0), axis=0) # generate from testing data
adv_clean_examples = np.append(adv_clean_examples, sample, axis=0)
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = X_test[sample_ind].reshape(-1)
nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
print('--------------------------------------')
adv_examples = adv_examples[1:,:,:,:]
adv_targets = adv_targets[1:,:]
adv_clean_labels = adv_clean_labels[1:,:]
adv_clean_examples = adv_clean_examples[1:,:,:,:]
np.savez('adversarial_jsma_actual_full',adv_examples=adv_examples, adv_targets=adv_targets, adv_clean_labels=adv_clean_labels,adv_clean_examples=adv_clean_examples)
print(np.shape(adv_targets)[0], "adversarial examples have been saved.")
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(succ_rate))
report.clean_test_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
return report
jsma = SaliencyMapMethod(wrap, sess=sess)
jsma_params = {'theta': 1.,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None}
#Genrating adversaries for jsma
adv=X_test
for index in range((len(X_test))):
sample = X_test[index: index + 1]
current = int(np.argmax(Y_test[index]))
target_classes = other_classes(10, current)
target=random.choice(target_classes)
one_hot_target = np.zeros((1, 10), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
adv[index]=adv_x
#predicting the classes for images generated from JSMA attack with the loaded network
predicted_classes = keras_model.predict_classes(adv)
#Checking accuracy and classification report of adversaries
new=np.nonzero(Y_test)
results=pd.DataFrame()
def gen_adv(sess,
dataset,
dataset_name,
attack_method,
attack_params,
attack_name,
testing=False,
adv_range=range(0, 20),
output_dir='./adv_output',
show_prediction=False):
# Object used to keep track of (and return) key accuracies
print("========= Start attack with method {} on {} =========".format(
attack_name, dataset_name))
report = AccuracyReport()
model = CNNModel(dataset)
# Initialize the Fast Gradient Sign Method (FGSM) attack object
wrap = KerasModelWrapper(model.model)
attack = attack_method(wrap, sess=sess)
# if fgsm_params is None:
# fgsm_params = {'eps': 0.3, 'clip_min': 0., 'clip_max': 1., 'y_target': None}
adv_acc_metric = get_adversarial_acc_metric(model.model, attack,
attack_params)
model.compile(loss='categorical_crossentropy',
metrics=['accuracy', adv_acc_metric])
# Train an MNIST model
model.fit()
# Evaluate the accuracy on legitimate and adversarial test examples
_, acc, adv_acc = model.evaluate()
report.clean_train_clean_eval = acc
report.clean_train_adv_eval = adv_acc
print('Test accuracy on legitimate examples: %0.4f' % acc)
print('Test accuracy on adversarial examples: %0.4f\n' % adv_acc)
for sample_ind in adv_range:
sample = model.x_test[sample_ind:(sample_ind + 1)]
current_class = int(np.argmax(model.y_test[sample_ind]))
target_classes = other_classes(model.nb_classes, current_class)
if not osp.isdir(osp.join(output_dir, dataset_name, attack_name)):
os.makedirs(osp.join(output_dir, dataset_name, attack_name), )
fn = osp.join(output_dir, dataset_name, attack_name,
str(sample_ind) + "_input.tiff")
imageio.imwrite(fn, np.reshape(sample,
(model.img_rows, model.img_cols)))
if show_prediction:
print("Prediction for the input is: \n", model.predict_one(sample))
for target in target_classes:
one_hot_target = np.zeros((1, model.nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
attack_params['y_target'] = one_hot_target
adv_x = attack.generate_np(sample, **attack_params)
fn = osp.join(output_dir, dataset_name, attack_name,
str(sample_ind) + "_adv{}.tiff".format(target))
imageio.imwrite(
fn, np.reshape(adv_x, (model.img_rows, model.img_cols)))
if show_prediction:
print("Prediction for the target {} is: \n".format(target),
model.predict_one(adv_x))
# Calculate training error
if testing:
_, train_acc, train_adv_acc = model.evaluate()
report.train_clean_train_clean_eval = train_acc
report.train_clean_train_adv_eval = train_adv_acc
print("========= Finish attack with method {} on {} =========".format(
attack_name, dataset_name))
return report
def generate_images():
print('==> Preparing data..')
if not hasattr(backend, "tf"):
raise RuntimeError("This tutorial requires keras to be configured"
" to use the TensorFlow backend.")
# Image dimensions ordering should follow the Theano convention
if keras.backend.image_dim_ordering() != 'tf':
keras.backend.set_image_dim_ordering('tf')
print(
"INFO: '~/.keras/keras.json' sets 'image_dim_ordering' to "
"'th', temporarily setting to 'tf'")
# Create TF session and set as Keras backend session
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.5
sess = tf.Session(config=config)
keras.backend.set_session(sess)
print "==> Beginning Session"
# Get CIFAR10 test data
X_train, Y_train, X_test, Y_test = data_cifar10()
assert Y_train.shape[1] == 10.
label_smooth = .1
Y_train = Y_train.clip(label_smooth / 9., 1. - label_smooth)
x = tf.placeholder(tf.float32, shape=(None, 32, 32, 3))
y = tf.placeholder(tf.float32, shape=(None, 10))
# Load model
print "==> loading vgg model"
args = load_args()
if args.model == 'vgg6': model = vggbn(top=True, pool=args.pool)
if args.model == 'vgg15': model = vgg15(top=True, pool=args.pool)
if args.model == 'generic': model = generic(top=True, pool=args.pool)
if args.model == 'resnet18': model = resnet.build_resnet_18(args.pool)
predictions = model(x)
model.load_weights(args.load)
eval_params = {'batch_size': FLAGS.batch_size}
accuracy = model_eval(sess,
x,
y,
predictions,
X_test,
Y_test,
args=eval_params)
print '==> Accuracy : {}'.format(accuracy)
def evaluate():
# Evaluate the accuracy of the CIFAR10 model on legitimate test examples
eval_params = {'batch_size': FLAGS.batch_size}
accuracy = model_eval(sess,
x,
y,
predictions,
X_test,
Y_test,
args=eval_params)
assert X_test.shape[0] == 10000, X_test.shape
print('Test accuracy on legitimate test examples: ' + str(accuracy))
# Train an CIFAR10 model
train_params = {
'nb_epochs': FLAGS.nb_epochs,
'batch_size': FLAGS.batch_size,
'learning_rate': FLAGS.learning_rate
}
im_base = '/im_'
model_name = args.model + '_p' + str(args.pool)
if args.attack == 'fgsm' or args.attack == 'FGSM':
result_dir = os.getcwd() + '/images/fgsm/'
print "==> creating fgsm adversarial wrapper"
adv_x = fgsm(x, predictions, eps=0.3)
print "==> sending to batch evaluator to finalize adversarial images"
eval_params = {'batch_size': FLAGS.batch_size}
X_train_adv, = batch_eval(sess, [x], [adv_x], [X_train],
args=eval_params)
i = 0
if not os.path.exists(result_dir + model_name):
os.makedirs(result_dir + model_name)
print "==> saving images to {}".format(result_dir + model_name)
for ad in X_train_adv:
scipy.misc.imsave(
result_dir + model_name + im_base + str(i) + '.png', ad)
i += 1
sess.close()
""" JSMA """
if args.attack == 'jsma' or args.attack == 'JSMA':
result_dir = os.getcwd() + '/images/jsma/trial_single_adv'
print('Crafting ' + str(FLAGS.source_samples) + ' * ' +
str(FLAGS.nb_classes - 1) + ' adversarial examples')
results = np.zeros((FLAGS.nb_classes, FLAGS.source_samples), dtype='i')
# This array contains the fraction of perturbed features for each test set
perturbations = np.zeros((FLAGS.nb_classes, FLAGS.source_samples),
dtype='f')
# Define the TF graph for the model's Jacobian
grads = jacobian_graph(predictions, x, FLAGS.nb_classes)
# Initialize our array for grid visualization
grid_shape = (FLAGS.nb_classes, FLAGS.nb_classes, FLAGS.img_rows,
FLAGS.img_cols, FLAGS.nb_channels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
i_saved = 0
n_image = 0
# Loop over the samples we want to perturb into adversarial examples
print "==> saving images to {}".format(result_dir + model_name)
for sample_ind in xrange(7166, FLAGS.source_samples):
# We want to find an adversarial example for each possible target class
current_class = int(np.argmax(Y_train[sample_ind]))
target_classes = other_classes(FLAGS.nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
X_train[sample_ind:(sample_ind + 1)],
(FLAGS.img_rows, FLAGS.img_cols, FLAGS.nb_channels))
# Loop over all target classes
adversarials = []
for idx, target in enumerate(target_classes):
print "image {}".format(sample_ind)
# here we hold all successful adversarials for this iteration
# since we dont want 500k images, we will uniformly sample an image to save after each target
print('--------------------------------------')
print('Creating adv. example for target class ' + str(target))
# This call runs the Jacobian-based saliency map approach
adv_x, res, percent_perturb = jsma(
sess,
x,
predictions,
grads,
X_train[sample_ind:(sample_ind + 1)],
target,
theta=1,
gamma=0.1,
increase=True,
back='tf',
clip_min=0,
clip_max=1)
# Display the original and adversarial images side-by-side
adversarial = np.reshape(
adv_x, (FLAGS.img_rows, FLAGS.img_cols, FLAGS.nb_channels))
original = np.reshape(
X_train[sample_ind:(sample_ind + 1)],
(FLAGS.img_rows, FLAGS.img_cols, FLAGS.nb_channels))
if FLAGS.viz_enabled:
if 'figure' not in vars():
figure = pair_visual(original, adversarial)
else:
figure = pair_visual(original, adversarial, figure)
if not os.path.exists(result_dir + model_name):
os.makedirs(result_dir + model_name)
if res == 1:
adversarials.append(adversarial)
if idx == FLAGS.nb_classes - 2:
try:
if len(adversarials) == 1:
idx_uniform = 0
else:
idx_uniform = np.random.randint(
0,
len(adversarials) - 1)
print idx_uniform
scipy.misc.imsave(
result_dir + model_name + im_base +
str(sample_ind) + '.png',
adversarials[idx_uniform])
i_saved += 1
print "==> images saved: {}".format(i_saved)
except:
print "No adversarials generated"
# Add our adversarial example to our grid data
grid_viz_data[target, current_class, :, :, :] = np.reshape(
adv_x, (FLAGS.img_rows, FLAGS.img_cols, FLAGS.nb_channels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
n_image += 1
# Compute the number of adversarial examples that were successfuly found
nb_targets_tried = ((FLAGS.nb_classes - 1) * FLAGS.source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print(
'Avg. rate of successful adv. examples {0:.2f}'.format(succ_rate))
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.2f}'.format(
percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print(
'Avg. rate of perturbed features for successful '
'adversarial examples {0:.2f}'.format(percent_perturb_succ))
# Close TF session
sess.close()
# Finally, block & display a grid of all the adversarial examples
if FLAGS.viz_enabled:
_ = grid_visual(grid_viz_data)
def main(argv=None):
"""
MNIST tutorial for the Jacobian-based saliency map approach (JSMA)
:return:
"""
# Disable Keras learning phase since we will be serving through tensorflow
keras.layers.core.K.set_learning_phase(0)
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Image dimensions ordering should follow the Theano convention
if keras.backend.image_dim_ordering() != 'tf':
keras.backend.set_image_dim_ordering('tf')
print("INFO: '~/.keras/keras.json' sets 'image_dim_ordering' "
"to 'th', temporarily setting to 'tf'")
# Create TF session and set as Keras backend session
sess = tf.Session()
keras.backend.set_session(sess)
print("Created TensorFlow session and set Keras backend.")
# Get MNIST test data
X_train, Y_train, X_test, Y_test = data_mnist()
print("Loaded MNIST test data.")
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
y = tf.placeholder(tf.float32, shape=(None, 10))
# Define TF model graph
model = cnn_model()
preds = model(x)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model if it does not exist in the train_dir folder
saver = tf.train.Saver()
save_path = os.path.join(FLAGS.train_dir, FLAGS.filename)
if os.path.isfile(save_path):
saver.restore(sess, os.path.join(FLAGS.train_dir, FLAGS.filename))
else:
train_params = {
'nb_epochs': FLAGS.nb_epochs,
'batch_size': FLAGS.batch_size,
'learning_rate': FLAGS.learning_rate
}
model_train(sess, x, y, preds, X_train, Y_train,
args=train_params)
saver.save(sess, save_path)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': FLAGS.batch_size}
accuracy = model_eval(sess, x, y, preds, X_test, Y_test,
args=eval_params)
assert X_test.shape[0] == 10000, X_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(FLAGS.source_samples) + ' * ' +
str(FLAGS.nb_classes-1) + ' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((FLAGS.nb_classes, FLAGS.source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((FLAGS.nb_classes, FLAGS.source_samples),
dtype='f')
# Initialize our array for grid visualization
grid_shape = (FLAGS.nb_classes,
FLAGS.nb_classes,
FLAGS.img_rows,
FLAGS.img_cols,
FLAGS.nb_channels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Define the SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, back='tf', sess=sess)
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in xrange(0, FLAGS.source_samples):
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, FLAGS.source_samples))
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(Y_test[sample_ind]))
target_classes = other_classes(FLAGS.nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
X_test[sample_ind:(sample_ind+1)],
(FLAGS.img_rows, FLAGS.img_cols, FLAGS.nb_channels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, FLAGS.nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params = {'theta': 1., 'gamma': 0.1,
'nb_classes': FLAGS.nb_classes, 'clip_min': 0.,
'clip_max': 1., 'targets': y,
'y_val': one_hot_target}
adv_x = jsma.generate_np(X_test[sample_ind:(sample_ind+1)],
**jsma_params)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# Computer number of modified features
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = X_test[sample_ind].reshape(-1)
nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
# Display the original and adversarial images side-by-side
if FLAGS.viz_enabled:
if 'figure' not in vars():
figure = pair_visual(
np.reshape(X_test[sample_ind:(sample_ind+1)],
(FLAGS.img_rows, FLAGS.img_cols)),
np.reshape(adv_x,
(FLAGS.img_rows, FLAGS.img_cols)))
else:
figure = pair_visual(
np.reshape(X_test[sample_ind:(sample_ind+1)],
(FLAGS.img_rows, FLAGS.img_cols)),
np.reshape(adv_x, (FLAGS.img_rows,
FLAGS.img_cols)), figure)
# Add our adversarial example to our grid data
grid_viz_data[target, current_class, :, :, :] = np.reshape(
adv_x, (FLAGS.img_rows, FLAGS.img_cols, FLAGS.nb_channels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((FLAGS.nb_classes - 1) * FLAGS.source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(succ_rate))
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
# Close TF session
sess.close()
# Finally, block & display a grid of all the adversarial examples
if FLAGS.viz_enabled:
_ = grid_visual(grid_viz_data)
def do_jsma():
print('Crafting ' + str(source_samples) + ' * ' +
str(nb_classes - 1) + ' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples), dtype='f')
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols, channels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, back='tf', sess=sess)
jsma_params = {
'theta': 1.,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
figure = None
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in xrange(0, source_samples):
print('--------------------------------------')
print('Attacking input %i/%i' %
(sample_ind + 1, source_samples))
sample = X_test[sample_ind:(sample_ind + 1)]
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(Y_test[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
grid_viz_data[current_class,
current_class, :, :, :] = np.reshape(
sample, (img_rows, img_cols, channels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' %
target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes),
dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# Computer number of modified features
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = X_test[sample_ind].reshape(-1)
nb_changed = np.where(
adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(
-1).shape[0]
# Display the original and adversarial images side-by-side
if FLAGS.viz_enabled:
figure = pair_visual(
np.reshape(sample, (img_rows, img_cols)),
np.reshape(adv_x, (img_rows, img_cols)), figure)
# Add our adversarial example to our grid data
grid_viz_data[target, current_class, :, :, :] = np.reshape(
adv_x, (img_rows, img_cols, channels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(
succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(
percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
if FLAGS.viz_enabled:
import matplotlib.pyplot as plt
plt.close(figure)
_ = grid_visual(grid_viz_data)
return report
'symbolic_impl': True,
'clip_min': 0.,
'clip_max': 255.,
'y_target': None
}
figure = None
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in range(0, nb_samples):
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, nb_samples))
sample = X_test[sample_ind:(sample_ind + 1)]
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(Y_test[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the
# diagonal
grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
sample, (img_rows, img_cols, channels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
def mnist_tutorial_jsma(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
viz_enabled=VIZ_ENABLED,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
source_samples=SOURCE_SAMPLES,
learning_rate=LEARNING_RATE):
"""
MNIST tutorial for the Jacobian-based saliency map approach (JSMA)
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session and set as Keras backend session
sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get MNIST test data
mnist = MNIST(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
x_train, y_train = mnist.get_set('train')
x_test, y_test = mnist.get_set('test')
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
nb_filters = 64
# Define TF model graph
model = ModelBasicCNN('model1', nb_classes, nb_filters)
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
sess.run(tf.global_variables_initializer())
rng = np.random.RandomState([2017, 8, 30])
train(sess, loss, x_train, y_train, args=train_params, rng=rng)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
assert x_test.shape[0] == test_end - test_start, x_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes - 1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples), dtype='f')
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols, nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, sess=sess)
jsma_params = {
'theta': 1.,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
figure = None
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in xrange(0, source_samples):
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, source_samples))
sample = x_test[sample_ind:(sample_ind + 1)]
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(y_test[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
sample, (img_rows, img_cols, nchannels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# Computer number of modified features
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = x_test[sample_ind].reshape(-1)
nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
# Display the original and adversarial images side-by-side
if viz_enabled:
figure = pair_visual(
np.reshape(sample, (img_rows, img_cols, nchannels)),
np.reshape(adv_x, (img_rows, img_cols, nchannels)), figure)
# Add our adversarial example to our grid data
grid_viz_data[target, current_class, :, :, :] = np.reshape(
adv_x, (img_rows, img_cols, nchannels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
# Close TF session
sess.close()
# Finally, block & display a grid of all the adversarial examples
if viz_enabled:
import matplotlib.pyplot as plt
plt.close(figure)
_ = grid_visual(grid_viz_data)
return report
def cifar10_tutorial_jsma(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
viz_enabled=VIZ_ENABLED,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
source_samples=SOURCE_SAMPLES,
learning_rate=LEARNING_RATE,
model_path=MODEL_PATH,
noise_output=NOISE_OUTPUT):
"""
CIFAR10 tutorial for the Jacobian-based saliency map approach (JSMA)
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session and set as Keras backend session
sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get CIFAR10 test data
cifar10 = CIFAR10(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
x_train, y_train = cifar10.get_set('train')
x_test, y_test = cifar10.get_set('test')
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
nb_filters = 64
# Define TF model graph
model = ModelAllConvolutional('model1',
nb_classes,
nb_filters,
input_shape=[32, 32, 3])
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an CIFAR10 model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate,
'filename': os.path.split(model_path)[-1]
}
sess.run(tf.global_variables_initializer())
rng = np.random.RandomState([2017, 8, 30])
train(sess, loss, x_train, y_train, args=train_params, rng=rng)
# Evaluate the accuracy of the CIFAR10 model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
assert x_test.shape[0] == test_end - test_start, x_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes - 1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples), dtype='f')
# Initialize our array for grid visualization
grid_shape = (nb_classes, 1, img_rows, img_cols, nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, sess=sess)
jsma_params = {
'theta': 1.,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
# Loop over the samples we want to perturb into adversarial examples
adv_all = np.zeros((nb_classes, img_rows, img_cols, nchannels), dtype='f')
sample_all = np.zeros((nb_classes, img_rows, img_cols, nchannels),
dtype='f')
for sample_ind in xrange(0, source_samples):
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, source_samples))
sample = x_test[sample_ind:(sample_ind + 1)]
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(y_test[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
# grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
# sample, (img_rows, img_cols, nchannels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
adv_all[current_class] = adv_x
sample_all[current_class] = sample
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# Computer number of modified features
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = x_test[sample_ind].reshape(-1)
nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
# Display the original and adversarial images side-by-side
# if viz_enabled:
# figure = pair_visual(
# np.reshape(sample, (img_rows, img_cols, nchannels)),
# np.reshape(adv_x, (img_rows, img_cols, nchannels)), figure)
# # Add our adversarial example to our grid data
# grid_viz_data[target, current_class, :, :, :] = np.reshape(
# adv_x, (img_rows, img_cols, nchannels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
# Compute the average distortion introduced by the algorithm
l2_norm = np.mean(np.sum((adv_all - sample_all)**2, axis=(1, 2, 3))**.5)
print('Avg. L_2 norm of perturbations {0:.4f}'.format(l2_norm))
for i in range(nb_classes):
if noise_output:
image = adv_all[i] - sample_all[i]
else:
image = adv_all[i]
grid_viz_data[i, 0] = image
# Close TF session
sess.close()
def save_visual(data, path):
"""
Modified version of cleverhans.plot.pyplot
"""
import matplotlib.pyplot as plt
figure = plt.figure()
# figure.canvas.set_window_title('Cleverhans: Grid Visualization')
# Add the images to the plot
num_cols = data.shape[0]
num_rows = data.shape[1]
num_channels = data.shape[4]
for y in range(num_rows):
for x in range(num_cols):
figure.add_subplot(num_rows, num_cols,
(x + 1) + (y * num_cols))
plt.axis('off')
if num_channels == 1:
plt.imshow(data[x, y, :, :, 0], cmap='gray')
else:
plt.imshow(data[x, y, :, :, :])
# Draw the plot and return
plt.savefig(path)
# Finally, block & display a grid of all the adversarial examples
if viz_enabled:
if noise_output:
image_name = "output/jsma_cifar10_noise.png"
else:
image_name = "output/jsma_cifar10.png"
_ = save_visual(grid_viz_data, image_name)
return report
def test_other_classes_invalid_class_ind(self):
with self.assertRaises(Exception) as context:
utils.other_classes(5, 8)
self.assertTrue(context.exception)
def main(argv=None):
"""
MNIST tutorial for the Jacobian-based saliency map approach (JSMA)
:return:
"""
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
###########################################################################
# Define the dataset and model
###########################################################################
# Image dimensions ordering should follow the Theano convention
if keras.backend.image_dim_ordering() != 'tf':
keras.backend.set_image_dim_ordering('tf')
print("INFO: '~/.keras/keras.json' sets 'image_dim_ordering' "
"to 'th', temporarily setting to 'tf'")
# Create TF session and set as Keras backend session
sess = tf.Session()
keras.backend.set_session(sess)
print("Created TensorFlow session and set Keras backend.")
# Get MNIST test data
X_train, Y_train, X_test, Y_test = data_mnist()
print("Loaded MNIST test data.")
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
y = tf.placeholder(tf.float32, shape=(None, 10))
# Define TF model graph
model = cnn_model()
predictions = model(x)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model if it does not exist in the train_dir folder
saver = tf.train.Saver()
save_path = os.path.join(FLAGS.train_dir, FLAGS.filename)
if os.path.isfile(save_path):
saver.restore(sess, os.path.join(FLAGS.train_dir, FLAGS.filename))
else:
train_params = {
'nb_epochs': FLAGS.nb_epochs,
'batch_size': FLAGS.batch_size,
'learning_rate': FLAGS.learning_rate
}
model_train(sess,
x,
y,
predictions,
X_train,
Y_train,
args=train_params)
saver.save(sess, save_path)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': FLAGS.batch_size}
accuracy = model_eval(sess,
x,
y,
predictions,
X_test,
Y_test,
args=eval_params)
assert X_test.shape[0] == 10000, X_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(FLAGS.source_samples) + ' * ' +
str(FLAGS.nb_classes - 1) + ' adversarial examples')
# This array indicates whether an adversarial example was found for each
# test set sample and target class
results = np.zeros((FLAGS.nb_classes, FLAGS.source_samples), dtype='i')
# This array contains the fraction of perturbed features for each test set
# sample and target class
perturbations = np.zeros((FLAGS.nb_classes, FLAGS.source_samples),
dtype='f')
# Define the TF graph for the model's Jacobian
grads = jacobian_graph(predictions, x, FLAGS.nb_classes)
# Initialize our array for grid visualization
grid_shape = (FLAGS.nb_classes, FLAGS.nb_classes, FLAGS.img_rows,
FLAGS.img_cols, FLAGS.nb_channels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in xrange(0, FLAGS.source_samples):
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(Y_test[sample_ind]))
target_classes = other_classes(FLAGS.nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
X_test[sample_ind:(sample_ind + 1)],
(FLAGS.img_rows, FLAGS.img_cols, FLAGS.nb_channels))
# Loop over all target classes
for target in target_classes:
print('--------------------------------------')
print('Creating adv. example for target class ' + str(target))
# This call runs the Jacobian-based saliency map approach
adv_x, res, percent_perturb = jsma(sess,
x,
predictions,
grads,
X_test[sample_ind:(sample_ind +
1)],
target,
theta=1,
gamma=0.1,
increase=True,
back='tf',
clip_min=0,
clip_max=1)
# Display the original and adversarial images side-by-side
if FLAGS.viz_enabled:
if 'figure' not in vars():
figure = pair_visual(
np.reshape(X_test[sample_ind:(sample_ind + 1)],
(FLAGS.img_rows, FLAGS.img_cols)),
np.reshape(adv_x, (FLAGS.img_rows, FLAGS.img_cols)))
else:
figure = pair_visual(
np.reshape(X_test[sample_ind:(sample_ind + 1)],
(FLAGS.img_rows, FLAGS.img_cols)),
np.reshape(adv_x, (FLAGS.img_rows, FLAGS.img_cols)),
figure)
# Add our adversarial example to our grid data
grid_viz_data[target, current_class, :, :, :] = np.reshape(
adv_x, (FLAGS.img_rows, FLAGS.img_cols, FLAGS.nb_channels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
# Compute the number of adversarial examples that were successfuly found
nb_targets_tried = ((FLAGS.nb_classes - 1) * FLAGS.source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.2f}'.format(succ_rate))
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.2f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.2f}'.format(percent_perturb_succ))
# Close TF session
sess.close()
# Finally, block & display a grid of all the adversarial examples
if FLAGS.viz_enabled:
_ = grid_visual(grid_viz_data)
def mnist_tutorial_jsma(train_start=0, train_end=60000, test_start=0,
test_end=10000, viz_enabled=True, nb_epochs=6,
batch_size=128, source_samples=10,
learning_rate=0.001):
"""
MNIST tutorial for the Jacobian-based saliency map approach (JSMA)
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session and set as Keras backend session
sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get MNIST test data
x_train, y_train, x_test, y_test = data_mnist(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols,
nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
nb_filters = 64
# Define TF model graph
model = ModelBasicCNN('model1', nb_classes, nb_filters)
preds = model.get_logits(x)
loss = LossCrossEntropy(model, smoothing=0.1)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
sess.run(tf.global_variables_initializer())
rng = np.random.RandomState([2017, 8, 30])
train(sess, loss, x, y, x_train, y_train, args=train_params,
rng=rng)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
assert x_test.shape[0] == test_end - test_start, x_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes-1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples), dtype='f')
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols, nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, back='tf', sess=sess)
jsma_params = {'theta': 1., 'gamma': 0.1,
'clip_min': 0., 'clip_max': 1.,
'y_target': None}
figure = None
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in xrange(0, source_samples):
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, source_samples))
sample = x_test[sample_ind:(sample_ind+1)]
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(y_test[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
sample, (img_rows, img_cols, nchannels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# Computer number of modified features
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = x_test[sample_ind].reshape(-1)
nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
# Display the original and adversarial images side-by-side
if viz_enabled:
figure = pair_visual(
np.reshape(sample, (img_rows, img_cols, nchannels)),
np.reshape(adv_x, (img_rows, img_cols, nchannels)), figure)
# Add our adversarial example to our grid data
grid_viz_data[target, current_class, :, :, :] = np.reshape(
adv_x, (img_rows, img_cols, nchannels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
# Close TF session
sess.close()
# Finally, block & display a grid of all the adversarial examples
if viz_enabled:
import matplotlib.pyplot as plt
plt.close(figure)
_ = grid_visual(grid_viz_data)
return report
def minist_fgsm_saliency(
train_start=0,
train_end=10,
test_start=0,
test_end=5,
nb_epochs=2,
batch_size=128,
learning_rate=0.001,
clean_train=True,
testing=False,
backprop_through_attack=False,
nb_filters=64,
nb_classes=10,
source_samples=10,
):
"""
MNIST cleverhans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param clean_train: perform normal training on clean examples only
before performing adversarial training.
:param testing: if true, complete an AccuracyReport for unit tests
to verify that performance is adequate
:param backprop_through_attack: If True, backprop through adversarial
example construction process during
adversarial training.
:param clean_train: if true, train on clean examples
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
sess = tf.Session()
# Get MNIST test data
X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
# Use label smoothing
assert Y_train.shape[1] == 10
label_smooth = .1
# this way, all the 9 zeroes -> 0.1/9 because
# the one-bit becomes 0.9
Y_train = Y_train.clip(label_smooth / 9., 1. - label_smooth)
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
y = tf.placeholder(tf.float32, shape=(None, 10))
# placeholder for y_target --> for saliency tensor
y_target = tf.placeholder(tf.float32, shape=(None, 10))
model_path = "models/mnist"
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
fgsm_params = {'eps': 0.3, 'clip_min': 0., 'clip_max': 1.}
rng = np.random.RandomState([2017, 8, 30])
###########################################################################
# Training the CNN model using TensorFlow: model --> base model
###########################################################################
model = make_basic_cnn(nb_filters=nb_filters)
preds = model.get_probs(x)
if clean_train:
# omg -> creates a cnn model
# model = make_basic_cnn(nb_filters=nb_filters)
# preds = model.get_probs(x)
def evaluate():
# Evaluate the accuracy of the MNIST model on legitimate test
# examples
eval_params = {'batch_size': batch_size}
acc = model_eval(sess,
x,
y,
preds,
X_test,
Y_test,
args=eval_params)
report.clean_train_clean_eval = acc
assert X_test.shape[0] == test_end - test_start, X_test.shape
print('Test accuracy on legitimate examples: %0.4f' % acc)
###########################################################################
# MODEL Train!!!!!!!!!!!!
###########################################################################
# training the basic model, using train_params
model_train(sess,
x,
y,
preds,
X_train,
Y_train,
evaluate=evaluate,
args=train_params,
rng=rng)
# Calculate training error
if testing:
eval_params = {'batch_size': batch_size}
acc = model_eval(sess,
x,
y,
preds,
X_train,
Y_train,
args=eval_params)
report.train_clean_train_clean_eval = acc
###########################################################################
# Generate FGSM Adversarial based on model, and
# Compute Base Model Accuracy
###########################################################################
# Initialize the Fast Gradient Sign Method (FGSM) attack object and
# graph
fgsm = FastGradientMethod(model, sess=sess)
# todo: follow the paper and run Cleverhans Output?
fgsm_params_y = {'eps': 0.3, 'y': y, 'clip_min': 0., 'clip_max': 1.}
#adv_x = fgsm.generate(x, **fgsm_params)
adv_x = fgsm.generate(x, **fgsm_params_y)
preds_adv = model.get_probs(adv_x)
# Evaluate the accuracy of the MNIST model on adversarial examples
eval_par = {'batch_size': batch_size}
acc = model_eval(sess, x, y, preds_adv, X_test, Y_test, args=eval_par)
print('Test accuracy on FGSM adversarial examples: %0.4f\n' % acc)
report.clean_train_adv_eval = acc
# Calculate training error
if testing:
eval_par = {'batch_size': batch_size}
acc = model_eval(sess,
x,
y,
preds_adv,
X_train,
Y_train,
args=eval_par)
report.train_clean_train_adv_eval = acc
###########################################################################
# Generate Saliency Map Adversarial Example and
# Compute base model accuracy (only 10)
###########################################################################
print("Saliency Map Attack On The Base Model")
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes - 1) +
' adversarial examples')
# Instantiate a SaliencyMapMethod attack object --> modify y_target for each test_data again
jsma = SaliencyMapMethod(model, back='tf', sess=sess)
jsma_params = {
'theta': 1.,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
# Keep track of success (adversarial example classified in target)
# Need this info to compute the success rate
results = np.zeros((nb_classes, source_samples), dtype='i')
# each sample will get 9 adversarial samples
# adv_x_set: place_holder for all the x variations
# correct_y_set: correct_y_output used for training
adv_x_set = None
adv_y_target = None
# we need multi x_train_saliency / y_train_saliency
#
x_train_saliency = None
y_train_saliency = None
for sample_ind in xrange(0, source_samples):
print('--------------------------------------')
print('Saliency Attacking input %i/%i' %
(sample_ind + 1, source_samples))
sample = X_train[sample_ind:(sample_ind + 1)]
y_sample = Y_train[sample_ind:(sample_ind + 1)]
current_class = int(np.argmax(Y_train[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# Create x_train_saliency, corresponding to y_train_saliency
if x_train_saliency is not None:
x_train_saliency = np.concatenate(
(x_train_saliency, sample), axis=0)
y_train_saliency = np.concatenate(
(y_train_saliency, y_sample), axis=0)
else:
x_train_saliency = sample
y_train_saliency = y_sample
print("sample shape: ", x_train_saliency.shape)
print("y_sample shape: ", y_train_saliency.shape)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x_np = jsma.generate_np(sample, **jsma_params)
# Add to adv_x_set, correct_y_set
if adv_x_set is not None:
adv_y_target = np.concatenate(
(adv_y_target, one_hot_target), axis=0)
adv_x_set = np.concatenate((adv_x_np, adv_x_set), axis=0)
else:
adv_y_target = one_hot_target
adv_x_set = adv_x_np
print("adv_y_target shape(one-hot-encoding): ",
adv_y_target.shape)
print("adv_x_set(np) shape: ", adv_x_np.shape)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x_np) == target)
# Update the arrays for later analysis
results[target, sample_ind] = res
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful Saliency adv. examples {0:.4f}'.format(
succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# here we have successfully stacked up x_adversarial_set, y_correct_set
# these can be used to provide training to our model now
print("\n\n\n*****************************")
print("Checking x_adv_set shape: ", adv_x_set.shape)
print("Checking correct_y_set shape: ", adv_y_target.shape)
print("x_training_saliency shape:", x_train_saliency.shape)
print("y_training_saliency shape:", y_train_saliency.shape)
# now construct model 3, define output -> input relationship tensor
model_3 = make_basic_cnn(nb_filters=nb_filters)
# define the x, the placeholder input - > preds_3 output
preds_3 = model_3(x)
# jsma3 = SaliencyMapMethod(model_3, sess=sess)
#
# jsma_params = {'theta': 1., 'gamma': 0.1,
# 'clip_min': 0., 'clip_max': 1.,
# 'y_target': y_target}
#
# # create adv_saliency set tensor, using x_train data and jsma_params containing adv_y_target
# adv_jsma = jsma3.generate(x, jsma_params)
# # create adv preds tensor
# preds_jsma_adv = model_3(adv_jsma)
# define saliency training model accuracy
def evaluate_saliency():
# Accuracy of adversarially trained model on legitimate test inputs
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess,
x,
y,
preds_3,
x_train_saliency,
y_train_saliency,
args=eval_params)
print('Test accuracy on legitimate examples: %0.4f' % accuracy)
report.adv_train_clean_eval = accuracy
###########################################################################
# MODEL Train for Saliency Map
###########################################################################
# Perform and evaluate adversarial training with FSGM MODEL!!!
# Train the model with samples of normal and adversarial examples!
model_train(sess,
x,
y,
model_3,
x_train_saliency,
y_train_saliency,
evaluate=evaluate_saliency(),
args=train_params,
rng=rng)
#todo: use jsma to create adversarial testing??? or training???
# Redefine TF model FGSM!!!
model_2 = make_basic_cnn(nb_filters=nb_filters)
preds_2 = model_2(x)
fgsm2 = FastGradientMethod(model_2, sess=sess)
# parameter for FGSM
fgsm_params_y = {'eps': 0.3, 'y': y, 'clip_min': 0., 'clip_max': 1.}
adv_x_2 = fgsm2.generate(x, **fgsm_params_y)
if not backprop_through_attack:
# For the fgsm attack used in this tutorial, the attack has zero
# gradient so enabling this flag does not change the gradient.
# For some other attacks, enabling this flag increases the cost of
# training, but gives the defender the ability to anticipate how
# the atacker will change their strategy in response to updates to
# the defender's parameters.
adv_x_2 = tf.stop_gradient(adv_x_2)
preds_2_adv = model_2(adv_x_2)
def evaluate_2():
# Accuracy of adversarially trained model on legitimate test inputs
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess,
x,
y,
preds_2,
X_test,
Y_test,
args=eval_params)
print('Test accuracy on legitimate examples: %0.4f' % accuracy)
report.adv_train_clean_eval = accuracy
# Accuracy of the adversarially trained model on adversarial examples
accuracy = model_eval(sess,
x,
y,
preds_2_adv,
X_test,
Y_test,
args=eval_params)
print('Test accuracy on adversarial examples: %0.4f' % accuracy)
report.adv_train_adv_eval = accuracy
###########################################################################
# MODEL Train for FGSM
###########################################################################
# Perform and evaluate adversarial training with FSGM MODEL!!!
model_train(sess,
x,
y,
preds_2,
X_train,
Y_train,
predictions_adv=preds_2_adv,
evaluate=evaluate_2,
args=train_params,
rng=rng)
# Calculate training errors
if testing:
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess,
x,
y,
preds_2,
X_train,
Y_train,
args=eval_params)
report.train_adv_train_clean_eval = accuracy
accuracy = model_eval(sess,
x,
y,
preds_2_adv,
X_train,
Y_train,
args=eval_params)
report.train_adv_train_adv_eval = accuracy
return report