def test_get_logits(self):
import tensorflow as tf
model = KerasModelWrapper(self.model)
x = tf.placeholder(tf.float32, shape=(None, 100))
preds = model.get_probs(x)
logits = model.get_logits(x)
x_val = np.random.rand(2, 100)
tf.global_variables_initializer().run(session=self.sess)
p_val, logits = self.sess.run([preds, logits], feed_dict={x: x_val})
p_gt = np.exp(logits) / np.sum(np.exp(logits), axis=1, keepdims=True)
self.assertTrue(np.allclose(p_val, p_gt, atol=1e-6))
def test_get_probs(self):
import tensorflow as tf
model = KerasModelWrapper(self.model)
x = tf.placeholder(tf.float32, shape=(None, 100))
preds = model.get_probs(x)
x_val = np.random.rand(2, 100)
tf.global_variables_initializer().run(session=self.sess)
p_val = self.sess.run(preds, feed_dict={x: x_val})
self.assertTrue(np.allclose(np.sum(p_val, axis=1), 1, atol=1e-6))
self.assertTrue(np.all(p_val >= 0))
self.assertTrue(np.all(p_val <= 1))
def test_get_probs(self):
import tensorflow as tf
model = KerasModelWrapper(self.model)
x = tf.placeholder(tf.float32, shape=(None, 100))
preds = model.get_probs(x)
x_val = np.random.rand(2, 100)
tf.global_variables_initializer().run(session=self.sess)
p_val = self.sess.run(preds, feed_dict={x: x_val})
self.assertTrue(np.allclose(np.sum(p_val, axis=1), 1, atol=1e-6))
self.assertTrue(np.all(p_val>=0))
self.assertTrue(np.all(p_val<=1))
def test_get_logits(self):
import tensorflow as tf
model = KerasModelWrapper(self.model)
x = tf.placeholder(tf.float32, shape=(None, 100))
preds = model.get_probs(x)
logits = model.get_logits(x)
x_val = np.random.rand(2, 100)
tf.global_variables_initializer().run(session=self.sess)
p_val, logits = self.sess.run([preds, logits], feed_dict={x: x_val})
p_gt = np.exp(logits)/np.sum(np.exp(logits), axis=1, keepdims=True)
self.assertTrue(np.allclose(p_val, p_gt, atol=1e-6))
class VGG16(Model):
__metaclass__ = ABCMeta
def __init__(self):
Model.__init__(self)
from keras.applications.vgg16 import VGG16, preprocess_input, decode_predictions
self.keras_model = VGG16(weights='imagenet')
self.model = KerasModelWrapper(self.keras_model)
self.preprocess_input = preprocess_input
self.decode_predictions = decode_predictions
def get_logits(self, x):
return self.model.get_logits(self.preprocess_input(x))
def get_probs(self, x):
return self.model.get_probs(self.preprocess_input(x))
def get_layer(self, x, layer):
output = self.model.fprop(self.preprocess_input(x))
try:
requested = output[layer]
except KeyError:
raise NoSuchLayerError()
return requested
def get_layer_names(self):
"""
:return: Names of all the layers kept by Keras
"""
layer_names = [x.name for x in self.keras_model.layers]
return layer_names
def predict(self, x, preprocess=False):
if preprocess:
return self.keras_model.predict(self.preprocess_input(x))
else:
return self.keras_model.predict(x)
class KerasModel():
WEIGHT_DIR = ""
def __init__(self, name):
self.name = name
self._model = {
'preprocess': unify_preprocess,
'default_input_size': 299,
}
self.sess = None
self.model = None
self.cleverhans_model = None
def load_weight(self, sess=None, checkpoint_path=''):
if self.model is None:
self._load_model(self.weight_path)
else:
self.model.load_weights(self.weight_path)
print("loaded keras model weights from ", self.weight_path)
def _load_model(self, path):
if os.path.exists(path):
self.model = load_model(path)
self.cleverhans_model = KerasModelWrapper(self.model)
print("loaded keras model from ", path)
else:
print("keras model path not exit", path)
return self.model
def _input_resize(self, imgs):
default_input_size = self._model['default_input_size']
imgs = tf.image.resize_images(imgs,
[default_input_size, default_input_size])
return imgs
def output_resize(self, imgs, size):
imgs = tf.image.resize_images(imgs, [size, size])
return imgs
def preprocess(self, imgs):
imgs = self._input_resize(imgs)
return self._model['preprocess'](imgs)
def predict_create_graph(self, batch_shape=None, use_prob=True, TOP_K=1):
if (use_prob == False):
print("Keras Model, use_prob==False not implemented!!")
if self.sess:
self.clear_session()
config = gpu_session_config()
self.sess = tf.Session(config=config)
with self.sess.as_default():
self.load_weight(self.sess)
def predict_batch(self, X, Y=None):
with self.sess.as_default():
X = self.preprocess(X)
ypred = self.model.predict_on_batch(X)
# ypred = ypred.argmax(1)
if Y is not None:
return ypred, None, None
else:
return ypred
def evaluate_generator(self, generator, batch_shape=None, use_prob=True):
total_ypred = []
total_correct = 0
total_size = 0
p = Profile(self.name + 'evaluate_generator ')
self.predict_create_graph()
for _, X, Y in generator:
ypred = self.predict_batch(X)
total_ypred = total_ypred + [ypred]
total_correct += X[ypred.argmax(1) == Y.argmax(1)].shape[0]
total_size += X.shape[0]
# print(total_correct, total_size)
total_accuracy = float(total_correct / total_size)
p.stop()
return np.concatenate(total_ypred), None, total_accuracy
def clear_session(self):
K.clear_session()
del self.model
self.model = None
if self.sess:
self.sess.close()
self.sess = None
tf.reset_default_graph()
def reload(self):
self.clear_session()
self.model = self._load_model(self.weight_path)
def get_logits(self, x, nb_classes):
return self.cleverhans_model.get_logits(x)
def get_probs(self, x, nb_classes):
return self.cleverhans_model.get_probs(x)
def blackbox_attack(input_path,
out_path,
batch_size=128,
learning_rate=0.001,
data_aug=10,
nb_epochs_s=10,
lmbda=0.1,
aug_batch_size=512):
"""
Black-box attack from arxiv.org/abs/1602.02697
:return: a dictionary with:
* black-box model accuracy on test set
* substitute model accuracy on test set
* black-box model accuracy on adversarial examples transferred
from the substitute model
"""
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Dictionary used to keep track and return key accuracies
accuracies = {}
# Create TF session
sess = tf.Session()
# data prep
train_datagen = ImageDataGenerator(preprocessing_function=preprocess_input,
rotation_range=30,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(
preprocessing_function=preprocess_input, )
IM_WIDTH, IM_HEIGHT = 299, 299
BATCH_SIZE = 32
train_generator = train_datagen.flow_from_directory(
'{}/train'.format(input_path),
target_size=(IM_WIDTH, IM_HEIGHT),
batch_size=BATCH_SIZE,
)
validation_generator = test_datagen.flow_from_directory(
'{}/eval'.format(input_path),
target_size=(IM_WIDTH, IM_HEIGHT),
batch_size=BATCH_SIZE,
)
# Get data (only one batch)
x_train, y_train = train_generator.next()
x_test, y_test = validation_generator.next()
# Initialize substitute training set reserved for adversary
X_sub = x_test
Y_sub = np.argmax(y_test, axis=1)
# Obtain Image parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
# Seed random number generator so tutorial is reproducible
rng = np.random.RandomState([2017, 8, 30])
# Simulate the black-box model locally
# You could replace this by a remote labeling API for instance
print("Setting up the black-box model.")
model = load_model(
'/data/pycon18/src/discriminative/inceptionv3-ft120_910acc.model')
kmodel = KerasModelWrapper(model)
bbox_preds = kmodel.get_probs(x)
# Train substitute using method from https://arxiv.org/abs/1602.02697
print("Training the substitute model.")
train_sub_out = train_sub(sess, x, y, bbox_preds, X_sub, Y_sub, nb_classes,
nb_epochs_s, batch_size, learning_rate, data_aug,
lmbda, aug_batch_size, rng)
model_sub, preds_sub = train_sub_out
# Evaluate the substitute model on clean test examples
eval_params = {'batch_size': batch_size}
acc = model_eval(sess, x, y, preds_sub, x_test, y_test, args=eval_params)
accuracies['sub'] = acc
# Initialize the Fast Gradient Sign Method (FGSM) attack object.
fgsm_par = {'eps': 0.3, 'ord': np.inf, 'clip_min': 0., 'clip_max': 1.}
fgsm = FastGradientMethod(model_sub, sess=sess)
# Craft adversarial examples using the substitute
eval_params = {'batch_size': batch_size}
x_adv_sub = fgsm.generate(x, **fgsm_par)
# Evaluate the accuracy of the "black-box" model on adversarial examples
accuracy = model_eval(sess,
x,
y,
kmodel.get_probs(x_adv_sub),
x_test,
y_test,
args=eval_params)
print(
'Test accuracy of oracle on adversarial examples generated using the substitute: '
+ str(accuracy))
accuracies['bbox_on_sub_adv_ex'] = accuracy
adv_images = sess.run(x_adv_sub, feed_dict={x: x_test})
original_pred = model.predict(x_test)
attack_pred = model.predict(adv_images)
# Save the images only if they cheat the oracle
img_to_save = []
for op, ap, x_img, adv_img in zip(np.argmax(original_pred, axis=1),
np.argmax(attack_pred, axis=1), x_test,
adv_images):
if op != ap:
img_to_save.append((x_img, adv_img))
x_filenames = ['file_{}.jpg'.format(i) for i in range(len(img_to_save))]
x_filenames_attack = [
'file_{}_attack.jpg'.format(i) for i in range(len(img_to_save))
]
save_images(np.array([x[0] for x in img_to_save]), x_filenames, out_path)
save_images(np.array([x[1] for x in img_to_save]), x_filenames_attack,
out_path)
return accuracies
def cifar10_eval_attacks(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
sweep_eps=SWEEP_EPS,
targeted=TARGETED,
model_key='model_1_a',
attacker_keys='clean',
eval_model_keys=None,
threat_model='white_box',
generate_examples=True):
"""
CIFAR10 cleverhans training
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param model_key: name of the keras model to be loaded and tested
:param attacker_key: name or list of names to be loaded
and used to attack the model
:return: an AccuracyReport object
"""
if threat_model == 'white_box':
eval_model_keys = [
model_key,
]
attacker_partition = ''
defender_partition = ''
if threat_model == 'black_box':
attacker_partition = 'A'
defender_partition = 'B'
if not isinstance(eval_model_keys, list):
raise ValueError('eval_model_keys must be list for black_box')
#TODO: add white-box info to meta-data
""" v<the eval model
"model_1_g": { v< the surrogate model
"advgan_b->model_1_e": {
"model_acc": "saved_models/model_1_cifar10_ResNet20_v2\\pickle\\model_1_g_advgan_b_model_acc.p",
"target_acc": "saved_models/model_1_cifar10_ResNet20_v2\\pickle\\model_1_g_advgan_b_target_acc.p",
"attack_stats": {
"L1": 127.04542236328125,
"L2": 2.9744277954101563,
"Linf": 0.2539639711380005,
"%pix": 93.39645385742188,
"num_batches": 20,
"time": "97.7us"
"threat_model":"black_box"
"""
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
sess = tf.Session()
K.set_learning_phase(0)
## Create TF session and set as Keras backend session
K.set_session(sess)
# Get CIFAR10 data
data = CIFAR10(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
dataset_size = data.x_train.shape[0]
dataset_train = data.to_tensorflow()[0]
#dataset_train = dataset_train.map(
# lambda x, y: (random_shift(random_horizontal_flip(x)), y), 4)
#dataset_train = dataset_train.batch(batch_size)
#dataset_train = dataset_train.prefetch(16)
#x_train, y_train = data.get_set('train')
x_test, y_test = data.get_set('test')
#nb_train = x_train.shape[0]
nb_test = x_test.shape[0]
# Use Image Parameters
img_rows, img_cols, nchannels = x_test.shape[1:4]
nb_classes = y_test.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
y_target = tf.placeholder(tf.float32, shape=(None, nb_classes))
meta = read_from_meta()
model_meta = meta['model'][model_key]
filename = model_meta['file_name'].replace('CIFAR10',
'CIFAR10' + attacker_partition)
keras_model = tf.keras.models.load_model(
filepath=model_meta['folder_path'] + '/' + filename,
custom_objects=custom_object())
model = KerasModelWrapper(keras_model)
attacker_keys = list(attacker_keys)
report = dict()
for attacker_key in attacker_keys:
# Create a new model and train it to be robust to Attacker
#keras_model = c10load.load_model(version=2,subtract_pixel_mean=True)
attacker_meta = meta['attacker'][attacker_key]
attack_type = attacker_meta['attack_type']
attack_params = {}
attack_params.update(meta['attacker']['default']['attack_params'])
attack_params.update(attacker_meta['attack_params'])
if 'spsa' in attacker_key:
eval_par = {'batch_size': 1}
else:
eval_par = {'batch_size': attack_params['batch_size']}
for k, v in attack_params.items():
if isinstance(v, str):
attack_params[k] = eval(v)
#define attacker
if attack_type == 'advgan' or 'g+' in attack_type:
if 'meta_key' in attacker_meta.keys():
folderpath = meta['advgan'][
attacker_meta['meta_key']]['train_params']['output_folder']
attack_params.update({
'generator_filepath':
os.path.join(folderpath, 'generator.hd5'),
'custom_objects':
custom_object()
})
else:
raise NotImplementedError(
"Must provide attacker meta with existing meta_key")
standard_attackers = {
'cwl2': cha.CarliniWagnerL2,
'fgsm': cha.FastGradientMethod,
'pgd': cha.MadryEtAl,
'jsma': cha.SaliencyMapMethod,
'stm': cha.SpatialTransformationMethod,
'advgan': cha.AdvGAN,
'spsa': cha.SPSA,
'g+pgd': cha.GanInformedPGD,
'g+spsa': cha.GanInformedSPSA
#'g+fgsm':cha.GanInformedFGM
}
if attack_type in standard_attackers.keys():
attacker = standard_attackers[attack_type](model, sess=sess)
elif attack_type == None or attack_type == 'clean':
attacker = None
else:
print(attack_type + ' is not a valid attack type')
pkl_folderpath = os.path.join(model_meta['folder_path'], 'pickle',
attacker_key)
if not os.path.isdir(pkl_folderpath):
os.makedirs(pkl_folderpath)
########
if targeted:
# get target labels
target_test = np.repeat(range(nb_classes), nb_test)
x_test_shuf = np.array(np.tile(x_test, (nb_classes, 1, 1, 1)))
y_test_shuf = np.array(np.tile(y_test, (nb_classes, 1)))
y_target_test_shuf = tf.keras.utils.to_categorical(
target_test, nb_classes)
#do not shuffle
#shuffle_in_unison(x_test_shuf,y_test_shuf,y_target_test_shuf)
x_test_by_t_o = [[None] * nb_classes for n in range(nb_classes)]
y_test_by_t_o = [[None] * nb_classes for n in range(nb_classes)]
y_target_test_by_t_o = [[None] * nb_classes
for n in range(nb_classes)]
nb_test_by_t_o = np.zeros((nb_classes + 1, nb_classes + 1))
print(y_target_test_shuf)
for t in range(nb_classes):
for o in range(nb_classes):
if t == o:
continue
index = np.logical_and(y_target_test_shuf[:, t],
y_test_shuf[:, o])
nb_test_by_t_o[t, o] = np.count_nonzero(index)
x_test_by_t_o[t][o] = x_test_shuf[index]
y_test_by_t_o[t][o] = y_test_shuf[index]
y_target_test_by_t_o[t][o] = y_target_test_shuf[index]
np.testing.assert_array_equal(y_target_test_by_t_o[0][1],
y_target_test_by_t_o[0][2],
err_msg='',
verbose=True)
nb_test_by_t_o[nb_classes, :] = np.sum(nb_test_by_t_o, axis=0)
nb_test_by_t_o[:, nb_classes] = np.sum(nb_test_by_t_o, axis=1)
attack_params.update({'y_target': y_target})
def model_eval_wrapper(preds,
acc_target='original_class',
adv_x=None):
if acc_target == 'original_class':
acc_target = y_test_by_t_o
elif acc_target == 'target_class':
acc_target = y_target_test_by_t_o
else:
raise ValueError('invalid value for accuracy_target: ' +
acc_target)
accuracy_by_t_o = np.zeros((nb_classes + 1, nb_classes + 1))
orig_accuracy_by_t_o = np.zeros(
(nb_classes + 1, nb_classes + 1))
for t in range(nb_classes + 1):
for o in range(nb_classes):
if t == o:
continue
row_scale = nb_test_by_t_o[t, o] / nb_test_by_t_o[
t, nb_classes]
col_scale = nb_test_by_t_o[t, o] / nb_test_by_t_o[
nb_classes, o]
if t < nb_classes:
feed = {
y_target:
y_target_test_by_t_o[t][o]
[:eval_par['batch_size'], :]
}
if generate_examples:
assert adv_x is not None, 'adv_x tensor must be supplied when generating examples'
pickle_x_file = os.path.join(
pkl_folderpath, pickle_file_head +
"x_test_targeted_{}_{}.p".format(t, o))
if os.path.exists(pickle_x_file):
adv_x_test = pickle.load(
open(pickle_x_file, "rb"))
else:
adv_x_test = gen_np(
sess, x_test_by_t_o[t][o], x, adv_x,
y_target_test_by_t_o[t][o], y_target)
pickle.dump(adv_x_test,
open(pickle_x_file, "wb"))
accuracy_by_t_o[t, o] = model_eval(
sess,
adv_x,
y,
preds,
adv_x_test,
acc_target[t][o],
args=eval_par)
orig_accuracy_by_t_o[t, o] = model_eval(
sess,
adv_x,
y,
preds,
x_test_by_t_o[t][o],
acc_target[t][o],
args=eval_par)
else:
accuracy_by_t_o[t, o] = model_eval(
sess,
x,
y,
preds,
x_test_by_t_o[t][o],
acc_target[t][o],
feed=feed,
args=eval_par)
accuracy_by_t_o[
nb_classes,
o] += accuracy_by_t_o[t, o] * col_scale
orig_accuracy_by_t_o[
nb_classes,
o] += orig_accuracy_by_t_o[t, o] * col_scale
accuracy_by_t_o[
t, nb_classes] += accuracy_by_t_o[t, o] * row_scale
orig_accuracy_by_t_o[
t,
nb_classes] += orig_accuracy_by_t_o[t,
o] * row_scale
if adv_x is not None:
# fill diagonal with original accuracies
for o in range(nb_classes):
accuracy_by_t_o[o,
o] = orig_accuracy_by_t_o[nb_classes,
o]
return accuracy_by_t_o
else:
x_test_shuf = x_test
y_test_shuf = y_test
def attack(x, attack_params=attack_params):
if attacker:
return attacker.generate(x, **attack_params)
else:
return x
def gen_np(sess, X, x, adv_x, Y_target=None, y_target=None):
#inputs:
# sess (required) : tf session
# X (required) : numpy input data
# x (required) : placeholder for model input
# adv_x (required) : tensor for generator output
# Y_target (optional) : optional numpy array speccifying the target class
# y_target (optional) : optional placeholder for the target inputs
#outputs:
#
if attacker:
with sess.as_default():
_batch_size = eval_par['batch_size']
nb_x = X.shape[0]
nb_batches = int(np.ceil(float(nb_x) / _batch_size))
assert nb_batches * _batch_size >= nb_x
adv_x_np = np.zeros((0, ) + X.shape[1:], dtype=X.dtype)
for batch in range(nb_batches):
start = batch * _batch_size
end = min(nb_x, start + _batch_size)
feed_dict = {x: X[start:end]}
if not Y_target is None:
feed_dict.update({y_target: Y_target[start:end]})
adv_x_cur = adv_x.eval(feed_dict=feed_dict)
adv_x_np = np.concatenate([adv_x_np, adv_x_cur],
axis=0)
assert end >= nb_x
return adv_x_np
else:
return x
def attack_stats_eval(x, adv_x, num_batches=1):
# Return attack info
with sess.as_default():
_batch_size = eval_par['batch_size']
_as_eval = dict()
cum_time = 0.
attack_stats = attack_statistics(x, adv_x)
for batch in range(num_batches):
feed_dict = {
x:
x_test_shuf[batch * _batch_size:(batch + 1) *
_batch_size],
y:
y_test_shuf[batch * _batch_size:(batch + 1) *
_batch_size]
}
if targeted:
feed_dict.update({
y_target:
y_target_test_shuf[batch *
_batch_size:(batch + 1) *
_batch_size]
})
_as = sess.run(attack_stats, feed_dict=feed_dict)
if batch == 0:
_as_eval = deepcopy(_as)
else:
_as_eval = {k: v + _as[k] for k, v in _as_eval.items()}
t_1 = time.process_time()
adv_x.eval(feed_dict=feed_dict)
t_2 = time.process_time()
cum_time += t_2 - t_1
cum_time /= num_batches * _batch_size
_as_eval = {k: v / num_batches for k, v in _as_eval.items()}
_as_eval.update({
'num_batches': num_batches,
'time': metric_convert(cum_time, 's')
})
return _as_eval
report.update({attacker_key: {'model_acc': {}}})
for eval_model_key in eval_model_keys:
#Sweep over models to evaluate on. "White Box" attacks
#only have one eval_model_key "Black Box" attack may
#have several eval_model_key "defenses"
report_view = report[attacker_key]
if threat_model == 'white_box':
assert model_key == eval_model_key, (
'for white_box attacks, ',
'generating model and eval model must be the same')
eval_model = model
elif threat_model == 'black_box':
#add black box eval model to report and update report head
if not 'black_box' in report_view.keys():
report_view.update(
{'black_box': {
eval_model_key: {
'model_acc': {}
}
}})
else:
report_view['black_box'].update(
{eval_model_key: {
'model_acc': {}
}})
report_view = report_view['black_box'][eval_model_key]
#load eval model trained on defense dataset
eval_model_meta = meta['model'][eval_model_key]
filename = eval_model_meta['file_name'].replace(
'CIFAR10', 'CIFAR10' + defender_partition)
keras_model = tf.keras.models.load_model(
filepath=eval_model_meta['folder_path'] + '/' + filename,
custom_objects=custom_object())
eval_model = KerasModelWrapper(keras_model)
#evaluate model on clean examples
preds = eval_model.get_logits(x)
model_acc = model_eval(sess,
x,
y,
preds,
x_test,
y_test,
args=eval_par)
print('Test accuracy on clean examples %0.4f\n' % model_acc)
report_view.update({'clean_model_acc': model_acc})
t1 = 0
#sweep epsilon
if sweep_eps and attack_type != 'clean':
max_eps = 2 * attack_params['eps']
if 'eps_iter' in attack_params.keys():
max_eps_iter = 2 * attack_params['eps_iter']
epsilons = np.linspace(1 / 255, max_eps,
min(int(max_eps * 255), 16))
sweep_e = dict()
for e in epsilons:
scaled_e = str(int(e * 255))
t1 = time.time()
attack_params.update({'eps': e})
if 'eps_iter' in attack_params.keys():
attack_params.update(
{'eps_iter': max_eps_iter * e / max_eps})
adv_x = attack(x, attack_params)
attack_stats_cur = attack_stats_eval(x, adv_x, 1)
preds_adv = eval_model.get_probs(adv_x)
if targeted:
model_acc = model_eval_wrapper(
preds_adv,
acc_target='original_class',
adv_x=adv_x)
target_acc = model_eval_wrapper(
preds_adv, acc_target='target_class', adv_x=adx_x)
pickle_file_head = '{}_{}_{}_'.format(
model_key, attacker_key, e)
pickle_m_file = os.path.join(
pkl_folderpath, pickle_file_head + "model_acc.p")
pickle_t_file = os.path.join(
pkl_folderpath, pickle_file_head + "target_acc.p")
pickle.dump(model_acc, open(pickle_m_file, "wb"))
pickle.dump(target_acc, open(pickle_t_file, "wb"))
sweep_e.update({
scaled_e: {
'model_acc': pickle_m_file,
'target_acc': pickle_t_file,
'attack_stats': attack_stats_cur
}
})
else:
if generate_examples:
pickle_x_file = os.path.join(
pkl_folderpath,
pickle_file_head + "x_test_untargeted.p")
if os.path.exists(pickle_x_file):
adv_x_test = pickle.load(
open(pickle_x_file, "rb"))
else:
adv_x_test = gen_np(sess, x_test, x, adv_x)
pickle.dump(adv_x_test,
open(pickle_x_file, "wb"))
model_acc = model_eval(sess,
adv_x,
y,
preds,
adv_x_test,
y_test,
args=eval_par)
else:
model_acc = model_eval(sess,
x,
y,
preds,
x_test,
y_test,
args=eval_par)
sweep_e.update({
scaled_e: {
'model_acc': model_acc,
'attack_stats': attack_stats_cur
}
})
print('Epsilon %.2f, accuracy on adversarial' % e,
'examples %0.4f\n' % model_acc)
print(sweep_e[scaled_e])
report_view.update({'sweep_eps': sweep_e})
t2 = time.time()
else:
if 'eps' in attack_params:
cond_eps = attack_params['eps']
else:
cond_eps = 'N/A'
print('evaluating {}->{} examples on {} (single epsilon: {})'.
format(attacker_key, model_key, eval_model_key,
cond_eps))
t1 = time.time()
adv_x = attack(x, attack_params)
preds_adv = eval_model.get_probs(adv_x)
pickle_file_head = '{}_{}_'.format(model_key, attacker_key)
if targeted:
model_acc = model_eval_wrapper(preds_adv,
acc_target='original_class',
adv_x=adv_x)
target_acc = model_eval_wrapper(preds_adv,
acc_target='target_class',
adv_x=adv_x)
if threat_model == 'black_box':
pickle_m_file = os.path.join(
pkl_folderpath,
pickle_file_head + eval_model_key + "_model_acc.p")
pickle_t_file = os.path.join(
pkl_folderpath, pickle_file_head + eval_model_key +
"_target_acc.p")
else:
pickle_m_file = os.path.join(
pkl_folderpath, pickle_file_head + "_model_acc.p")
pickle_t_file = os.path.join(
pkl_folderpath, pickle_file_head + "_target_acc.p")
pickle.dump(model_acc, open(pickle_m_file, "wb"))
pickle.dump(target_acc, open(pickle_t_file, "wb"))
report_view.update({
'model_acc':
pickle_m_file,
'target_acc':
pickle_t_file,
'attack_stats':
attack_stats_eval(x, adv_x, 20)
})
else:
if generate_examples:
pickle_x_file = os.path.join(
pkl_folderpath,
pickle_file_head + "x_test_untargeted.p")
if os.path.exists(pickle_x_file):
adv_x_test = pickle.load(open(pickle_x_file, "rb"))
else:
adv_x_test = gen_np(sess, x_test, x, adv_x)
pickle.dump(adv_x_test, open(pickle_x_file, "wb"))
#evaluate on self and, if black box, all other eval models
model_acc = model_eval(sess,
adv_x,
y,
preds_adv,
adv_x_test,
y_test,
args=eval_par)
else:
model_acc = model_eval(sess,
x,
y,
preds_adv,
x_test,
y_test,
args=eval_par)
report_view.update({
'model_acc':
model_acc,
'attack_stats':
attack_stats_eval(x, adv_x, 20)
})
t2 = time.time()
if targeted:
print('Test accuracy on adversarial examples %0.4f\n' %
model_acc[nb_classes, nb_classes])
print('Target accuracy on adversarial examples %0.4f\n' %
target_acc[nb_classes, nb_classes])
else:
print('Test accuracy on adversarial examples %0.4f\n' %
model_acc)
print("Took", t2 - t1, "seconds")
return report
