def prep_bbox(sess,
x,
y,
x_train,
y_train,
x_test,
y_test,
nb_epochs,
batch_size,
learning_rate,
rng,
nb_classes=10,
img_rows=28,
img_cols=28,
nchannels=1):
"""
Define and train a model that simulates the "remote"
black-box oracle described in the original paper.
:param sess: the TF session
:param x: the input placeholder for MNIST
:param y: the ouput placeholder for MNIST
:param x_train: the training data for the oracle
:param y_train: the training labels for the oracle
:param x_test: the testing data for the oracle
:param y_test: the testing labels for the oracle
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param rng: numpy.random.RandomState
:return:
"""
# Define TF model graph (for the black-box model)
nb_filters = 64
model = make_basic_picklable_cnn(nb_filters=nb_filters,
nb_classes=nb_classes)
loss = CrossEntropy(model, smoothing=0.1)
predictions = model.get_logits(x)
print("Defined TensorFlow model graph.")
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
train(sess, loss, x_train, y_train, args=train_params, rng=rng)
# Print out the accuracy on legitimate data
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess,
x,
y,
predictions,
x_test,
y_test,
args=eval_params)
print('Test accuracy of black-box on legitimate test '
'examples: ' + str(accuracy))
return model, predictions, accuracy
def mnist_tutorial(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
learning_rate=LEARNING_RATE,
clean_train=CLEAN_TRAIN,
testing=False,
backprop_through_attack=BACKPROP_THROUGH_ATTACK,
nb_filters=NB_FILTERS,
num_threads=None,
label_smoothing=0.1):
"""
MNIST cleverhans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param clean_train: perform normal training on clean examples only
before performing adversarial training.
:param testing: if true, complete an AccuracyReport for unit tests
to verify that performance is adequate
:param backprop_through_attack: If True, backprop through adversarial
example construction process during
adversarial training.
:param label_smoothing: float, amount of label smoothing for cross entropy
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
if num_threads:
config_args = dict(intra_op_parallelism_threads=1)
else:
config_args = {}
sess = tf.Session(config=tf.ConfigProto(**config_args))
# Get MNIST test data
mnist = MNIST(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
x_train, y_train = mnist.get_set('train')
x_test, y_test = mnist.get_set('test')
# Use Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
eval_params = {'batch_size': batch_size}
fgsm_params = {'eps': 0.3, 'clip_min': 0., 'clip_max': 1.}
rng = np.random.RandomState([2017, 8, 30])
def do_eval(preds, x_set, y_set, report_key, is_adv=None):
acc = model_eval(sess, x, y, preds, x_set, y_set, args=eval_params)
setattr(report, report_key, acc)
if is_adv is None:
report_text = None
elif is_adv:
report_text = 'adversarial'
else:
report_text = 'legitimate'
if report_text:
print('Test accuracy on %s examples: %0.4f' % (report_text, acc))
if clean_train:
model = make_basic_picklable_cnn()
# Tag the model so that when it is saved to disk, future scripts will
# be able to tell what data it was trained on
model.dataset_factory = mnist.get_factory()
preds = model.get_logits(x)
assert len(model.get_params()) > 0
loss = CrossEntropy(model, smoothing=label_smoothing)
def evaluate():
do_eval(preds, x_test, y_test, 'clean_train_clean_eval', False)
train(sess,
loss,
x_train,
y_train,
evaluate=evaluate,
args=train_params,
rng=rng,
var_list=model.get_params())
with sess.as_default():
save("clean_model.joblib", model)
print("Now that the model has been saved, you can evaluate it in a"
" separate process using `evaluate_pickled_model.py`. "
"You should get exactly the same result for both clean and "
"adversarial accuracy as you get within this program.")
# Calculate training error
if testing:
do_eval(preds, x_train, y_train, 'train_clean_train_clean_eval')
# Initialize the Fast Gradient Sign Method (FGSM) attack object and
# graph
fgsm = FastGradientMethod(model, sess=sess)
adv_x = fgsm.generate(x, **fgsm_params)
preds_adv = model.get_logits(adv_x)
# Evaluate the accuracy of the MNIST model on adversarial examples
do_eval(preds_adv, x_test, y_test, 'clean_train_adv_eval', True)
# Calculate training error
if testing:
do_eval(preds_adv, x_train, y_train, 'train_clean_train_adv_eval')
print('Repeating the process, using adversarial training')
# Create a new model and train it to be robust to FastGradientMethod
model2 = make_basic_picklable_cnn()
# Tag the model so that when it is saved to disk, future scripts will
# be able to tell what data it was trained on
model2.dataset_factory = mnist.get_factory()
fgsm2 = FastGradientMethod(model2, sess=sess)
def attack(x):
return fgsm2.generate(x, **fgsm_params)
loss2 = CrossEntropy(model2, smoothing=label_smoothing, attack=attack)
preds2 = model2.get_logits(x)
adv_x2 = attack(x)
if not backprop_through_attack:
# For the fgsm attack used in this tutorial, the attack has zero
# gradient so enabling this flag does not change the gradient.
# For some other attacks, enabling this flag increases the cost of
# training, but gives the defender the ability to anticipate how
# the atacker will change their strategy in response to updates to
# the defender's parameters.
adv_x2 = tf.stop_gradient(adv_x2)
preds2_adv = model2.get_logits(adv_x2)
def evaluate2():
# Accuracy of adversarially trained model on legitimate test inputs
do_eval(preds2, x_test, y_test, 'adv_train_clean_eval', False)
# Accuracy of the adversarially trained model on adversarial examples
do_eval(preds2_adv, x_test, y_test, 'adv_train_adv_eval', True)
# Perform and evaluate adversarial training
train(sess,
loss2,
x_train,
y_train,
evaluate=evaluate2,
args=train_params,
rng=rng,
var_list=model2.get_params())
with sess.as_default():
save("adv_model.joblib", model2)
print(
"Now that the model has been saved, you can evaluate it in a "
"separate process using "
"`python evaluate_pickled_model.py adv_model.joblib`. "
"You should get exactly the same result for both clean and "
"adversarial accuracy as you get within this program."
" You can also move beyond the tutorials directory and run the "
" real `compute_accuracy.py` script (make sure cleverhans/scripts "
"is in your PATH) to see that this FGSM-trained "
"model is actually not very robust---it's just a model that trains "
" quickly so the tutorial does not take a long time")
# Calculate training errors
if testing:
do_eval(preds2, x_train, y_train, 'train_adv_train_clean_eval')
do_eval(preds2_adv, x_train, y_train, 'train_adv_train_adv_eval')
return report
def mnist_tutorial(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
learning_rate=LEARNING_RATE,
clean_train=CLEAN_TRAIN,
testing=False,
preprocess='',
backprop_through_attack=BACKPROP_THROUGH_ATTACK,
nb_filters=NB_FILTERS,
num_threads=None,
label_smoothing=0.1):
"""
MNIST cleverhans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param clean_train: perform normal training on clean examples only
before performing adversarial training.
:param testing: if true, complete an AccuracyReport for unit tests
to verify that performance is adequate
:param backprop_through_attack: If True, backprop through adversarial
example construction process during
adversarial training.
:param label_smoothing: float, amount of label smoothing for cross entropy
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
if num_threads:
config_args = dict(intra_op_parallelism_threads=1)
else:
config_args = {}
sess = tf.Session(config=tf.ConfigProto(**config_args))
### CHANGE DATASET ###
# Get MNIST data
# mnist = MNIST_67(train_start=train_start, train_end=train_end,
# test_start=test_start, test_end=test_end)
# x_train, y_train = mnist.get_set('train')
# x_test, y_test = mnist.get_set('test')
x_train, y_train, x_test, y_test = get_MNIST_67_preprocess(
preprocess=preprocess)
with open('../pickle/{}_y_train.pickle'.format(FILENAME), 'wb') as handle:
pickle.dump(y_train, handle)
with open('../pickle/{}_y_test.pickle'.format(FILENAME), 'wb') as handle:
pickle.dump(y_test, handle)
# Use Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
eval_params = {'batch_size': batch_size}
fgsm_params = {'eps': 0.3, 'clip_min': 0., 'clip_max': 1.}
rng = np.random.RandomState([2017, 8, 30])
### ADD PARAMETERS ###
def do_eval(preds, x_set, y_set, report_key, is_adv=None):
acc = model_eval(sess,
x,
y,
preds,
x_set,
y_set,
save_logit=True,
filename=FLAGS.filename + "_" + report_key,
args=eval_params)
setattr(report, report_key, acc)
if is_adv is None:
report_text = None
elif is_adv:
report_text = 'adversarial'
else:
report_text = 'legitimate'
if report_text:
print('Test accuracy on %s examples: %0.4f' % (report_text, acc))
if clean_train:
### picklable ###
#model = ModelBasicCNN('model1', nb_classes, nb_filters)
model = make_basic_picklable_cnn(nb_filters=nb_filters,
nb_classes=nb_classes)
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=label_smoothing)
def evaluate():
do_eval(preds, x_test, y_test, 'clean_train_clean_eval', False)
train(sess,
loss,
x_train,
y_train,
evaluate=evaluate,
args=train_params,
rng=rng,
var_list=model.get_params())
#Now, save the graph
with sess.as_default():
save("../models/CNN_{}.joblib".format(preprocess), model)
# Calculate training error
if testing:
do_eval(preds, x_train, y_train, 'train_clean_train_clean_eval')
exit()
# Initialize the Fast Gradient Sign Method (FGSM) attack object and
# graph
fgsm = FastGradientMethod(model, sess=sess)
adv_x = fgsm.generate(x, **fgsm_params)
preds_adv = model.get_logits(adv_x)
# Evaluate the accuracy of the MNIST model on adversarial examples
do_eval(preds_adv, x_test, y_test, 'clean_train_adv_eval', True)
# Calculate training error
if testing:
do_eval(preds_adv, x_train, y_train, 'train_clean_train_adv_eval')
print('Repeating the process, using adversarial training')
# Create a new model and train it to be robust to FastGradientMethod
### picklable ###
#model2 = ModelBasicCNN('model2', nb_classes, nb_filters)
model2 = make_basic_picklable_cnn(nb_filters=nb_filters,
nb_classes=nb_classes)
fgsm2 = FastGradientMethod(model2, sess=sess)
def attack(x):
return fgsm2.generate(x, **fgsm_params)
loss2 = CrossEntropy(model2, smoothing=label_smoothing, attack=attack)
preds2 = model2.get_logits(x)
adv_x2 = attack(x)
if not backprop_through_attack:
# For the fgsm attack used in this tutorial, the attack has zero
# gradient so enabling this flag does not change the gradient.
# For some other attacks, enabling this flag increases the cost of
# training, but gives the defender the ability to anticipate how
# the atacker will change their strategy in response to updates to
# the defender's parameters.
adv_x2 = tf.stop_gradient(adv_x2)
preds2_adv = model2.get_logits(adv_x2)
def evaluate2():
# Accuracy of adversarially trained model on legitimate test inputs
do_eval(preds2, x_test, y_test, 'adv_train_clean_eval', False)
# Accuracy of the adversarially trained model on adversarial examples
do_eval(preds2_adv, x_test, y_test, 'adv_train_adv_eval', True)
# Perform and evaluate adversarial training
train(sess,
loss2,
x_train,
y_train,
evaluate=evaluate2,
args=train_params,
rng=rng,
var_list=model2.get_params())
#Now, save the graph
with sess.as_default():
save("../models/{}_{}.joblib".format(FILENAME, preprocess), model2)
# Calculate training errors
if testing:
do_eval(preds2, x_train, y_train, 'train_adv_train_clean_eval')
do_eval(preds2_adv, x_train, y_train, 'train_adv_train_adv_eval')
return report
def mnist_tutorial_jsma(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
viz_enabled=VIZ_ENABLED,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
source_samples=SOURCE_SAMPLES,
learning_rate=LEARNING_RATE):
"""
MNIST tutorial for the Jacobian-based saliency map approach (JSMA)
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session and set as Keras backend session
#replace
num_threads = None
if num_threads:
config_args = dict(intra_op_parallelism_threads=1)
else:
config_args = {}
sess = tf.Session(config=tf.ConfigProto(**config_args))
#with sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get MNIST test data
mnist = MNIST(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
x_train, y_train = mnist.get_set('train')
x_test, y_test = mnist.get_set('test')
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
nb_filters = 64
# Define TF model graph
model = make_basic_picklable_cnn()
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
dataset = tf.data.Dataset.from_tensor_slices(
(tf.reshape(x_train, [60000, 28, 28]), y_train))
dataset = dataset.batch(32)
val_dataset = tf.data.Dataset.from_tensor_slices(
(tf.reshape(x_test, [10000, 28, 28]), y_test))
val_dataset = val_dataset.batch(32)
sess.run(tf.global_variables_initializer())
rng = np.random.RandomState([2017, 8, 30])
if TRAIN_NEW == 1:
with sess.as_default():
train(sess, loss, x_train, y_train, args=train_params, rng=rng)
save("test.joblib", model)
else:
with sess.as_default():
model = load("test.joblib") #changed
assert len(model.get_params()) > 0
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
assert x_test.shape[0] == test_end - test_start, x_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes - 1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples), dtype='f')
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols, nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, sess=sess)
jsma_params = {
'theta': 1.,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
figure = None
# Loop over the samples we want to perturb into adversarial examples
seed(SEED)
for sample_ind in xrange(0, source_samples):
img = randint(0, 10000)
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, source_samples))
sample = x_test[img:(img +
1)] #sample = x_test[sample_ind:(sample_ind + 1)]
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(
y_test[img])) #current_class = int(np.argmax(y_test[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
sample, (img_rows, img_cols, nchannels))
tn = 0
totc = 0
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# Compute number of modified features
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = x_test[sample_ind].reshape(-1)
nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
diff = np.array(adv_x - sample)
#print(np.sum(diff))
diff = np.reshape(diff, (28, 28))
diff = diff * 255
cv2.imwrite("test.png", diff)
diff = cv2.imread("test.png")
diff = cv2.cvtColor(diff, cv2.COLOR_BGR2GRAY)
nieghbors = 0
tc = 0
for i in range(0, 28, 1):
for j in range(0, 28, 1):
if diff[i, j] > 0:
tc = tc + 1
totc = totc + 1
if i > 0 and i < 27 and j > 0 and j < 27: #main grid not edges or corners
if diff[i - 1, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j + 1] > 0:
nieghbors = nieghbors + 1
else:
#corners
if i == 0 and j == 0:
if diff[i, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j] > 0:
nieghbors = nieghbors + 1
if i == 27 and j == 0:
if diff[i, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j] > 0:
nieghbors = nieghbors + 1
if i == 0 and j == 27:
if diff[i, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j] > 0:
nieghbors = nieghbors + 1
if i == 27 and j == 27:
if diff[i, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j] > 0:
nieghbors = nieghbors + 1
#edges
if i == 0 and j > 0 and j < 27: #left side
if diff[i, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j + 1] > 0:
nieghbors = nieghbors + 1
if i == 27 and j > 0 and j < 27: #right side
if diff[i, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j + 1] > 0:
nieghbors = nieghbors + 1
if j == 0 and i > 0 and i < 27: #top side
if diff[i - 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i, j + 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j + 1] > 0:
nieghbors = nieghbors + 1
if j == 27 and i > 0 and i < 27: #bot side
if diff[i - 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j] > 0:
nieghbors = nieghbors + 1
if diff[i - 1, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i, j - 1] > 0:
nieghbors = nieghbors + 1
if diff[i + 1, j - 1] > 0:
nieghbors = nieghbors + 1
# print(tc)
# print(nieghbors)
tn = tn + nieghbors
# if tc > 0:
# print(nieghbors/tc)
# Display the original and adversarial images side-by-side
if viz_enabled:
figure = pair_visual(
np.reshape(sample, (img_rows, img_cols, nchannels)),
np.reshape(adv_x, (img_rows, img_cols, nchannels)), figure)
# Add our adversarial example to our grid data
grid_viz_data[target, current_class, :, :, :] = np.reshape(
adv_x, (img_rows, img_cols, nchannels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
#print(perturbations[target, sample_ind])
print('--------------------------------------')
print("average neighbors per modified pixel ", tn / totc)
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.8f}'.format(succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
s = perturbations.shape
myPert = np.empty(0)
myResults = np.empty(0)
for i in range(s[0]):
for j in range(s[1]):
if perturbations[i][j] > 0:
myPert = np.append(myPert, perturbations[i][j])
myResults = np.append(myResults, results[i][j])
min_perturbed = np.min(myPert)
max_perturbed = np.max(myPert)
s2 = myResults.shape
final = np.empty(0)
for i in range(s2[0]):
if myResults[i] > 0:
final = np.append(final, myPert[i])
print('Avg. rate of perturbed features {0:.8f}'.format(percent_perturbed))
print('MIN of perturbed features {0:.8f}'.format(min_perturbed))
print('MAX of perturbed features {0:.8f}'.format(max_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
min_perturb_succ = np.min(final)
max_perturb_succ = np.max(final)
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.8f}'.format(percent_perturb_succ))
print('Min of perturbed features for successful '
'adversarial examples {0:.8f}'.format(min_perturb_succ))
print('Max of perturbed features for successful '
'adversarial examples {0:.8f}'.format(max_perturb_succ))
#Close TF session
sess.close()
# Finally, block & display a grid of all the adversarial examples
if viz_enabled:
import matplotlib.pyplot as plt
plt.close(figure)
_ = grid_visual(grid_viz_data)
return report
def mnist_tutorial_jsma(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
viz_enabled=False,
nb_epochs=6,
batch_size=128,
nb_classes=10,
source_samples=10,
learning_rate=0.001):
"""
MNIST tutorial for the Jacobian-based saliency map approach (JSMA)
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# MNIST-specific dimensions
img_rows = 28
img_cols = 28
channels = 1
# Set TF random seed to improve reproducibility
tf.set_random_seed(7076)
# Create TF session and set as Keras backend session
sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get MNIST test data
X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
y = tf.placeholder(tf.float32, shape=(None, 10))
# Define TF model graph
model = make_basic_picklable_cnn()
preds = model(x)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
sess.run(tf.global_variables_initializer())
rng = np.random.RandomState([2017, 8, 30])
model_train(sess,
x,
y,
preds,
X_train,
Y_train,
args=train_params,
rng=rng)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, X_test, Y_test, args=eval_params)
assert X_test.shape[0] == test_end - test_start, X_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes - 1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i') # i = interger
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples),
dtype='f') #f = floating
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols, channels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, back='tf', sess=sess)
jsma_params = {
'theta': 1.,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
figure = None #None
# create an array for storing adv examples
adv_examples = np.empty([1, 28, 28, 1])
# for target labels
adv_targets = np.empty([1, 10])
# corresponding clean/correct label
adv_clean_labels = np.empty([1, 10])
# correspongding clean data
adv_clean_examples = np.empty([1, 28, 28, 1])
# Loop over the samples we want to perturb into adversarial examples
for sample_ind in xrange(0, source_samples): #source_samples set 10
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, source_samples))
sample = X_train[sample_ind:(sample_ind +
1)] # generate from training data
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(
Y_train[sample_ind])) # generate from training data
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
sample, (img_rows, img_cols, channels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
#create fake target
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
# print('adv_x\'shape is ', np.shape(adv_x)) # (1,28,28,1)
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# if succeeds
if res == 1:
# append new adv_x to adv_examples array
# append sample here, so that the number of times sample is appended mmatches number of adv_ex.
adv_examples = np.append(adv_examples, adv_x, axis=0)
adv_targets = np.append(adv_targets, one_hot_target, axis=0)
adv_clean_labels = np.append(
adv_clean_labels,
np.expand_dims(Y_train[sample_ind], axis=0),
axis=0) # generate from training data
adv_clean_examples = np.append(adv_clean_examples,
sample,
axis=0)
# Compute the number of modified features
# adv_x.reshape(-1) means reshape into (1, n), in this case, n=28x28
# it makes comparison simplier
# adv_x_reshape = adv_x.reshape(-1)
# test_in_reshape = X_test[sample_ind].reshape(-1)
# nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
# percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
adv_x_reshape = adv_x.reshape(-1)
train_in_reshape = X_train[sample_ind].reshape(-1)
nb_changed = np.where(
adv_x_reshape != train_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
# Display the original and adversarial images side-by-side
viz_enabled = True #False
if viz_enabled:
figure = pair_visual(np.reshape(sample, (img_rows, img_cols)),
np.reshape(adv_x, (img_rows, img_cols)),
figure)
# Add our adversarial example to our grid data
grid_viz_data[target, current_class, :, :, :] = np.reshape(
adv_x, (img_rows, img_cols, channels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
print('--------------------------------------')
adv_examples = adv_examples[1:, :, :, :]
adv_targets = adv_targets[1:, :]
adv_clean_labels = adv_clean_labels[1:, :]
adv_clean_examples = adv_clean_examples[1:, :, :, :]
np.savez('adversarial',
adv_examples=adv_examples,
adv_targets=adv_targets,
adv_clean_labels=adv_clean_labels,
adv_clean_examples=adv_clean_examples)
print(np.shape(adv_targets)[0], "adversarial examples have been saved.")
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
# Close TF session
sess.close()
# Finally, block & display a grid of all the adversarial examples
if viz_enabled:
import matplotlib.pyplot as plt
plt.close(figure)
_ = grid_visual(grid_viz_data)
return report
