class TestMomentumIterativeMethod(TestBasicIterativeMethod):
def setUp(self):
super(TestMomentumIterativeMethod, self).setUp()
import tensorflow as tf
# The world's simplest neural network
def my_model(x):
W1 = tf.constant([[1.5, .3], [-2, 0.3]], dtype=tf.float32)
h1 = tf.nn.sigmoid(tf.matmul(x, W1))
W2 = tf.constant([[-2.4, 1.2], [0.5, -2.3]], dtype=tf.float32)
res = tf.nn.softmax(tf.matmul(h1, W2))
return res
self.sess = tf.Session()
self.model = my_model
self.attack = MomentumIterativeMethod(self.model, sess=self.sess)
def test_generate_np_can_be_called_with_different_decay_factor(self):
x_val = np.random.rand(100, 2)
x_val = np.array(x_val, dtype=np.float32)
for dacay_factor in [0.0, 0.5, 1.0]:
x_adv = self.attack.generate_np(x_val, eps=0.5, ord=np.inf,
dacay_factor=dacay_factor,
clip_min=-5.0, clip_max=5.0)
delta = np.max(np.abs(x_adv - x_val), axis=1)
self.assertClose(delta, 0.5)
class TestMomentumIterativeMethod(TestBasicIterativeMethod):
def setUp(self):
super(TestMomentumIterativeMethod, self).setUp()
self.sess = tf.Session()
self.model = SimpleModel()
self.attack = MomentumIterativeMethod(self.model, sess=self.sess)
def test_generate_np_can_be_called_with_different_decay_factor(self):
x_val = np.random.rand(100, 2)
x_val = np.array(x_val, dtype=np.float32)
for dacay_factor in [0.0, 0.5, 1.0]:
x_adv = self.attack.generate_np(x_val, eps=0.5, ord=np.inf,
dacay_factor=dacay_factor,
clip_min=-5.0, clip_max=5.0)
delta = np.max(np.abs(x_adv - x_val), axis=1)
self.assertClose(delta, 0.5)
class TestMomentumIterativeMethod(TestBasicIterativeMethod):
def setUp(self):
super(TestMomentumIterativeMethod, self).setUp()
self.sess = tf.Session()
self.model = SimpleModel()
self.attack = MomentumIterativeMethod(self.model, sess=self.sess)
def test_generate_np_can_be_called_with_different_decay_factor(self):
x_val = np.random.rand(100, 2)
x_val = np.array(x_val, dtype=np.float32)
for dacay_factor in [0.0, 0.5, 1.0]:
x_adv = self.attack.generate_np(x_val, eps=0.5, ord=np.inf,
dacay_factor=dacay_factor,
clip_min=-5.0, clip_max=5.0)
delta = np.max(np.abs(x_adv - x_val), axis=1)
self.assertClose(delta, 0.5)
print("The Test accuracy is: {}".format(acc))
#################################### Adversarial Attack (MIM) ###################################
wrap = KerasModelWrapper(keras_model)
mim = MomentumIterativeMethod(wrap, back='tf', sess=sess)
mim_params = {
'eps': 0.7,
'eps_iter': 0.7,
'nb_iter': 10,
'y_target': None,
'ord': np.inf,
'clip_min': 0.,
'clip_max': 1.
}
adv_x = mim.generate_np(x_test, **mim_params)
adv_conf = keras_model.predict(adv_x)
adv_pred = np.argmax(adv_conf, axis=1)
adv_acc = np.mean(np.equal(adv_pred, y_test))
print("The adversarial accuracy is: {}".format(adv_acc))
###################################### Original Image ##########################################
x_sample = x_test[7010].reshape(28, 28)
plt.imshow(x_sample, cmap='Blues')
plt.show()
###################################### Adversarial Image ########################################
adv_x_sample = adv_x[7010].reshape(28, 28)
plt.imshow(adv_x_sample, cmap='Blues')
plt.show()
print("MIM")
mim_params = {
'eps': float(sys.argv[1]),
'eps_iter': 0.01,
'nb_iter': 500,
'ord': np.inf,
'clip_min': 0.,
'clip_max': 1.
}
mim_source = MomentumIterativeMethod(wrap_source, sess=sess)
X_adv_source = np.zeros((len(indices_test), 32, 32, 3))
for i in np.arange(0, len(indices_test), 500):
X_adv_source[i:(i + 500)] = mim_source.generate_np(
X_test[indices_test[i:(i + 500)]], **mim_params)
print("metrics source model")
print(metrics(model_source, X_adv_source, X_test, pred_source, indices_test))
print("metrics base model")
print(metrics(model, X_adv_source, X_test, pred_base, indices_test))
pred_source_adv = np.argmax(model_source.predict(X_adv_source), axis=1)
pred_adv_basefromsource = np.argmax(model.predict(X_adv_source), axis=1)
agree_func(indices_test, pred_adv_basefromsource, pred_source_adv, pred_base,
pred_source)
print(" ")
####################################
#MIM Diverse
print("\n\n")
def mnist_tutorial_adv_train(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
viz_enabled=VIZ_ENABLED,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
source_samples=SOURCE_SAMPLES,
learning_rate=LEARNING_RATE,
attack_iterations=ATTACK_ITERATIONS,
model_path=MODEL_PATH,
targeted=TARGETED,
noise_output=NOISE_OUTPUT):
"""
MNIST tutorial for Adversarial Training
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:param model_path: path to the model file
:param targeted: should we run a targeted attack? or untargeted?
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session
sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get MNIST test data
mnist = MNIST(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
x_train, y_train = mnist.get_set('train')
x_test, y_test = mnist.get_set('test')
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
nb_filters = 64
# Define TF model graph
model = ModelBasicCNN('model1', nb_classes, nb_filters)
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate,
'filename': os.path.split(model_path)[-1]
}
rng = np.random.RandomState([2017, 8, 30])
# check if we've trained before, and if we have, use that pre-trained model
if os.path.exists(model_path + ".meta"):
tf_model_load(sess, model_path)
else:
train(sess, loss, x_train, y_train, args=train_params, rng=rng)
saver = tf.train.Saver()
saver.save(sess, model_path)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
assert x_test.shape[0] == test_end - test_start, x_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using FGSM - BIM - MIM approach
###########################################################################
nb_adv_per_sample = str(nb_classes - 1) if targeted else '1'
print('Crafting ' + str(source_samples) + ' * ' + nb_adv_per_sample +
' adversarial examples')
print("This could take some time ...")
# Instantiate a CW attack object
fgsm = FastGradientMethod(model, sess=sess)
bim = BasicIterativeMethod(model, sess=sess)
mim = MomentumIterativeMethod(model, sess=sess)
if viz_enabled:
assert source_samples == nb_classes
idxs = [
np.where(np.argmax(y_test, axis=1) == i)[0][0]
for i in range(nb_classes)
]
if targeted:
if viz_enabled:
# Initialize our array for grid visualization
grid_shape = (nb_classes, 1, img_rows, img_cols, nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
adv_inputs = np.array([[instance] * nb_classes
for instance in x_test[idxs]],
dtype=np.float32)
else:
adv_inputs = np.array([[instance] * nb_classes
for instance in x_test[:source_samples]],
dtype=np.float32)
one_hot = np.zeros((nb_classes, nb_classes))
one_hot[np.arange(nb_classes), np.arange(nb_classes)] = 1
adv_inputs = adv_inputs.reshape(
(source_samples * nb_classes, img_rows, img_cols, nchannels))
adv_ys = np.array([one_hot] * source_samples,
dtype=np.float32).reshape(
(source_samples * nb_classes, nb_classes))
yname = "y_target"
else:
if viz_enabled:
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols,
nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
adv_inputs = x_test[idxs]
else:
adv_inputs = x_test[:source_samples]
adv_ys = None
yname = "y"
fgsm_params = {'eps': 0.3, 'clip_min': 0., 'clip_max': 1.}
bim_params = {
'eps': 0.3,
'clip_min': 0.,
'clip_max': 1.,
'nb_iter': 50,
'eps_iter': .01
}
mim_params = {
'eps': 0.3,
'clip_min': 0.,
'clip_max': 1.,
'nb_iter': 50,
'eps_iter': .01
}
adv_fgsm = fgsm.generate_np(adv_inputs, **fgsm_params)
adv_bim = bim.generate_np(adv_inputs, **bim_params)
adv_mim = mim.generate_np(adv_inputs, **mim_params)
eval_params = {'batch_size': np.minimum(nb_classes, source_samples)}
if targeted:
adv_fgsm_accuracy = model_eval(sess,
x,
y,
preds,
adv_fgsm,
adv_ys,
args=eval_params)
adv_bim_accuracy = model_eval(sess,
x,
y,
preds,
adv_bim,
adv_ys,
args=eval_params)
adv_mim_accuracy = model_eval(sess,
x,
y,
preds,
adv_mim,
adv_ys,
args=eval_params)
else:
if viz_enabled:
err_fgsm = model_eval(sess,
x,
y,
preds,
adv_fgsm,
y_test[idxs],
args=eval_params)
err_bim = model_eval(sess,
x,
y,
preds,
adv_bim,
y_test[idxs],
args=eval_params)
err_mim = model_eval(sess,
x,
y,
preds,
adv_mim,
y_test[idxs],
args=eval_params)
adv_fgsm_accuracy = 1 - err_fgsm
adv_bim_accuracy = 1 - err_bim
adv_mim_accuracy = 1 - err_mim
else:
err_fgsm = model_eval(sess,
x,
y,
preds,
adv_fgsm,
y_test[:source_samples],
args=eval_params)
err_bim = model_eval(sess,
x,
y,
preds,
adv_bim,
y_test[:source_samples],
args=eval_params)
err_mim = model_eval(sess,
x,
y,
preds,
adv_mim,
y_test[:source_samples],
args=eval_params)
adv_fgsm_accuracy = 1 - err_fgsm
adv_bim_accuracy = 1 - err_bim
adv_mim_accuracy = 1 - err_mim
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
print('Avg. rate of successful adv. (FGSM) examples {0:.4f}'.format(
adv_fgsm_accuracy))
report.clean_train_adv_fgsm_eval = 1. - adv_fgsm_accuracy
print('Avg. rate of successful adv. (BIM) examples {0:.4f}'.format(
adv_bim_accuracy))
report.clean_train_adv_bim_eval = 1. - adv_bim_accuracy
print('Avg. rate of successful adv. (MIM) examples {0:.4f}'.format(
adv_mim_accuracy))
report.clean_train_adv_mim_eval = 1. - adv_mim_accuracy
# Compute the average distortion introduced by the algorithm
percent_perturbed_fgsm = np.mean(
np.sum((adv_fgsm - adv_inputs)**2, axis=(1, 2, 3))**.5)
print('Avg. L_2 norm of (FGSM) perturbations {0:.4f}'.format(
percent_perturbed_fgsm))
percent_perturbed_bim = np.mean(
np.sum((adv_bim - adv_inputs)**2, axis=(1, 2, 3))**.5)
print('Avg. L_2 norm of (BIM) perturbations {0:.4f}'.format(
percent_perturbed_bim))
percent_perturbed_mim = np.mean(
np.sum((adv_mim - adv_inputs)**2, axis=(1, 2, 3))**.5)
print('Avg. L_2 norm of (MIM) perturbations {0:.4f}'.format(
percent_perturbed_mim))
###########################################################################
# Adversarial Training
###########################################################################
model2 = ModelBasicCNN('model2', nb_classes, nb_filters)
fgsm2 = FastGradientMethod(model, sess=sess)
# bim2 = BasicIterativeMethod(model, sess=sess)
# mim2 = MomentumIterativeMethod(model, sess=sess)
def attack_fgsm(x):
return fgsm2.generate(adv_inputs, **fgsm_params)
# def attack_bim(x):
# return bim2.generate(adv_inputs, **bim_params)
# def attack_mim(x):
# return mim2.generate(adv_inputs, **mim_params)
preds2 = model2.get_logits(x)
loss2_fgsm = CrossEntropy(model2, smoothing=0.1, attack=attack_fgsm)
# loss2_bim = CrossEntropy(model2, smoothing=0.1, attack=attack_bim)
# loss2_mim = CrossEntropy(model2, smoothing=0.1, attack=attack_mim)
train(sess, loss2_fgsm, x_train, y_train, args=train_params, rng=rng)
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds2, x_test, y_test, args=eval_params)
assert x_test.shape[0] == test_end - test_start, x_test.shape
print('Test accuracy on adversarial fgsm test examples: {0}'.format(
accuracy))
report.clean_train_clean_eval = accuracy
print("Defined TensorFlow model graph.")
adv_fgsm_accuracy = model_eval(sess,
x,
y,
preds,
adv_fgsm,
adv_ys,
args=eval_params)
adv_bim_accuracy = model_eval(sess,
x,
y,
preds,
adv_bim,
adv_ys,
args=eval_params)
adv_mim_accuracy = model_eval(sess,
x,
y,
preds,
adv_mim,
adv_ys,
args=eval_params)
# Close TF session
sess.close()
return report
####################################
#MIM
print("\n\n")
print("MIM")
mim_params = {
'eps': float(sys.argv[1]),
'eps_iter': 0.01,
'nb_iter': 1000,
'ord': np.inf,
'clip_min': 0.,
'clip_max': 1.
}
mim_source = MomentumIterativeMethod(wrap_source, sess=sess)
X_adv_source = mim_source.generate_np(X_test[indices_test], **mim_params)
print("metrics source model")
print(metrics(model_source, X_adv_source, X_test, pred_source, indices_test))
print("metrics base model")
print(metrics(model, X_adv_source, X_test, pred_base, indices_test))
pred_source_adv = np.argmax(model_source.predict(X_adv_source), axis=1)
pred_adv_basefromsource = np.argmax(model.predict(X_adv_source), axis=1)
agree_func(indices_test, pred_adv_basefromsource, pred_source_adv, pred_base,
pred_source)
print(" ")
####################################
#MIM Diverse
print("\n\n")
def mnist_tutorial_mim(train_start=0, train_end=60000, test_start=0,
test_end=10000, viz_enabled=VIZ_ENABLED,
nb_epochs=NB_EPOCHS, batch_size=BATCH_SIZE,
source_samples=SOURCE_SAMPLES,
learning_rate=LEARNING_RATE,
attack_iterations=ATTACK_ITERATIONS,
model_path=MODEL_PATH,
targeted=TARGETED,
noise_output=NOISE_OUTPUT):
"""
MNIST tutorial for Momentum Iterative Method's attack
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:param model_path: path to the model file
:param targeted: should we run a targeted attack? or untargeted?
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session
sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get MNIST test data
mnist = MNIST(train_start=train_start, train_end=train_end,
test_start=test_start, test_end=test_end)
x_train, y_train = mnist.get_set('train')
x_test, y_test = mnist.get_set('test')
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols,
nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
nb_filters = 64
# Define TF model graph
model = ModelBasicCNN('model1', nb_classes, nb_filters)
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate,
'filename': os.path.split(model_path)[-1]
}
rng = np.random.RandomState([2017, 8, 30])
# check if we've trained before, and if we have, use that pre-trained model
if os.path.exists(model_path + ".meta"):
tf_model_load(sess, model_path)
else:
train(sess, loss, x_train, y_train, args=train_params, rng=rng)
saver = tf.train.Saver()
saver.save(sess, model_path)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
assert x_test.shape[0] == test_end - test_start, x_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using Momentum Iterative Method's approach
###########################################################################
nb_adv_per_sample = str(nb_classes - 1) if targeted else '1'
print('Crafting ' + str(source_samples) + ' * ' + nb_adv_per_sample +
' adversarial examples')
print("This could take some time ...")
# Instantiate a MIM attack object
mim = MomentumIterativeMethod(model, sess=sess)
if viz_enabled:
assert source_samples == nb_classes
idxs = [np.where(np.argmax(y_test, axis=1) == i)[0][0]
for i in range(nb_classes)]
if targeted:
if viz_enabled:
# Initialize our array for grid visualization
grid_shape = (nb_classes, 1, img_rows, img_cols,
nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
adv_inputs = np.array(
[[instance] * nb_classes for instance in x_test[idxs]],
dtype=np.float32)
else:
adv_inputs = np.array(
[[instance] * nb_classes for
instance in x_test[:source_samples]], dtype=np.float32)
one_hot = np.zeros((nb_classes, nb_classes))
one_hot[np.arange(nb_classes), np.arange(nb_classes)] = 1
adv_inputs = adv_inputs.reshape(
(source_samples * nb_classes, img_rows, img_cols, nchannels))
adv_ys = np.array([one_hot] * source_samples,
dtype=np.float32).reshape((source_samples *
nb_classes, nb_classes))
else:
if viz_enabled:
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols, nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
adv_inputs = x_test[idxs]
else:
adv_inputs = x_test[:source_samples]
adv_ys = None
mim_params = {'eps': 0.3, 'clip_min': 0., 'clip_max': 1.,
'nb_iter': 50,
'eps_iter': .01}
adv = mim.generate_np(adv_inputs,
**mim_params)
eval_params = {'batch_size': np.minimum(nb_classes, source_samples)}
if targeted:
adv_accuracy = model_eval(
sess, x, y, preds, adv, adv_ys, args=eval_params)
else:
if viz_enabled:
err = model_eval(sess, x, y, preds, adv, y_test[idxs], args=eval_params)
adv_accuracy = 1 - err
else:
err = model_eval(sess, x, y, preds, adv, y_test[:source_samples],
args=eval_params)
adv_accuracy = 1 - err
if viz_enabled:
for i in range(nb_classes):
if noise_output:
image = adv[i * nb_classes] - adv_inputs[i * nb_classes]
else:
image = adv[i * nb_classes]
grid_viz_data[i, 0] = image
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
print('Avg. rate of successful adv. examples {0:.4f}'.format(adv_accuracy))
report.clean_train_adv_eval = 1. - adv_accuracy
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(np.sum((adv - adv_inputs)**2,
axis=(1, 2, 3))**.5)
print('Avg. L_2 norm of perturbations {0:.4f}'.format(percent_perturbed))
# Close TF session
sess.close()
def save_visual(data, path):
"""
Modified version of cleverhans.plot.pyplot
"""
figure = plt.figure()
# figure.canvas.set_window_title('Cleverhans: Grid Visualization')
# Add the images to the plot
num_cols = data.shape[0]
num_rows = data.shape[1]
num_channels = data.shape[4]
for y in range(num_rows):
for x in range(num_cols):
figure.add_subplot(num_rows, num_cols, (x + 1) + (y * num_cols))
plt.axis('off')
if num_channels == 1:
plt.imshow(data[x, y, :, :, 0], cmap='gray')
else:
plt.imshow(data[x, y, :, :, :])
# Draw the plot and return
plt.savefig(path)
return figure
# Finally, block & display a grid of all the adversarial examples
if viz_enabled:
if noise_output:
image_name = "output/mim_mnist_noise.png"
else:
image_name = "output/mim_mnist.png"
_ = save_visual(grid_viz_data, image_name)
return report
#comp_func_transfer(X_adv_auto, indices_test, pred_base, pred_auto, model_auto, model)
#comp_func_transfer(X_adv_ce, indices_test, pred_base, pred_ce, model_ce, model)
#comp_func_transfer(X_adv_rob, indices_test, pred_base, pred_rob, model_rob, model)
###################################
#MIM
print("\n\n")
print("MIM")
mim_stacked = MomentumIterativeMethod(wrap_stacked, sess=sess)
mim_auto = MomentumIterativeMethod(wrap_auto, sess=sess)
mim_ce = MomentumIterativeMethod(wrap_ce, sess=sess)
mim_rob = MomentumIterativeMethod(wrap_rob, sess=sess)
X_adv_stacked = np.zeros((len(indices_test), 32, 32, 3))
for i in np.arange(0, len(indices_test), 500):
X_adv_stacked[i:(i + 500)] = mim_stacked.generate_np(
X_test[indices_test[i:(i + 500)]], **mim_params)
X_adv_auto = np.zeros((len(indices_test), 32, 32, 3))
for i in np.arange(0, len(indices_test), 500):
X_adv_auto[i:(i + 500)] = mim_auto.generate_np(
X_test[indices_test[i:(i + 500)]], **mim_params)
X_adv_ce = np.zeros((len(indices_test), 32, 32, 3))
for i in np.arange(0, len(indices_test), 500):
X_adv_ce[i:(i + 500)] = mim_ce.generate_np(
X_test[indices_test[i:(i + 500)]], **mim_params)
X_adv_rob = np.zeros((len(indices_test), 32, 32, 3))
for i in np.arange(0, len(indices_test), 500):
X_adv_rob[i:(i + 500)] = mim_rob.generate_np(
X_test[indices_test[i:(i + 500)]], **mim_params)
'clip_min': 0}
elif attack_method == 'CW':
attack = CarliniWagnerL2(KerasModelWrapper(model), sess=sess)
params = {'confidence': eps}
data = data[eps_iter:eps_iter+decay_factor]
labels = labels[eps_iter:eps_iter+decay_factor]
# generate adversarial examples
# attack = MomentumIterativeMethod(KerasModelWrapper(model), sess=sess)
# for decay_factor in [.5]:
x_advs = []
succ_idxs = []
sep = 500
for i in range(0, len(data), sep):
_end = min(len(data), i+sep)
x_adv = attack.generate_np(data[i:_end], **params)
x_adv = (np.array(x_adv)*255.).astype(np.uint8)
# print(x_adv.shape, data[i:_end].shape)
attack_sign = (np.argmax(model.predict(x_adv.astype(np.float32)/255.), axis=1)!=labels[i:_end])
# print(np.array(attack_sign).shape)
x_advs.append(x_adv[attack_sign])
# print(np.array(x_advs).shape, np.vstack(x_advs).shape)
succ_idxs.append(np.arange(i, _end)[attack_sign])
# print(np.hstack(succ_idxs).shape)
# print(len(labels), succ_idxs)
np.save('advs/mnist_'+attack_method+'_'+sign+'_advs_show.npy', np.vstack(x_advs))
np.save('advs/mnist_'+attack_method+'_'+sign+'_advs_label.npy', labels[np.hstack(succ_idxs)])
np.save('advs/mnist_'+attack_method+'_'+sign+'_advs_label.npy', np.hstack(succ_idxs))
print('model acc under aes:', 1-len(idxs)/len(data))
pred_base, pred_stacked, pred_auto, pred_ce, pred_rob)
#comp_func_transfer(X_adv_stacked, indices_test, pred_base, pred_stacked, model_stacked, model)
#comp_func_transfer(X_adv_auto, indices_test, pred_base, pred_auto, model_auto, model)
#comp_func_transfer(X_adv_ce, indices_test, pred_base, pred_ce, model_ce, model)
#comp_func_transfer(X_adv_rob, indices_test, pred_base, pred_rob, model_rob, model)
####################################
#MIM
print("\n\n")
print("MIM")
mim_stacked = MomentumIterativeMethod(wrap_stacked, sess=sess)
mim_auto = MomentumIterativeMethod(wrap_auto, sess=sess)
mim_ce = MomentumIterativeMethod(wrap_ce, sess=sess)
mim_rob = MomentumIterativeMethod(wrap_rob, sess=sess)
X_adv_stacked = mim_stacked.generate_np(X_test[indices_test], **mim_params)
X_adv_auto = mim_auto.generate_np(X_test[indices_test], **mim_params)
X_adv_ce = mim_ce.generate_np(X_test[indices_test], **mim_params)
X_adv_rob = mim_rob.generate_np(X_test[indices_test], **mim_params)
comp_func(X_adv_stacked, X_adv_auto, X_adv_ce, X_adv_rob, indices_test,
pred_base, pred_stacked, pred_auto, pred_ce, pred_rob)
#comp_func_transfer(X_adv_stacked, indices_test, pred_base, pred_stacked, model_stacked, model)
#comp_func_transfer(X_adv_auto, indices_test, pred_base, pred_auto, model_auto, model)
#comp_func_transfer(X_adv_ce, indices_test, pred_base, pred_ce, model_ce, model)
#comp_func_transfer(X_adv_rob, indices_test, pred_base, pred_rob, model_rob, model)
####################################
#MIML2
print("\n\n")
print("MIMl2")