def setUp(self):
# Set master seed
master_seed(1234)
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
learning = tf.placeholder(tf.bool)
# Define the tensorflow graph
conv = tf.layers.conv2d(input_ph, 16, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(fc, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
# Create classifier and fit
self.classifier = TFClassifier((0, 1), input_ph, logits, output_ph,
train, loss, learning, self.sess)
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
self.classifier.fit(x_train, y_train, batch_size=100, nb_epochs=3)
def _create_tfclassifier():
"""
To create a simple TFClassifier for testing.
:return:
"""
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(fc, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Create the classifier
tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss,
None, sess)
return tfc
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
model = Model(**model_kwargs)
input_ph = model.x_input
labels_ph = model.y_input
training_ph = tf.placeholder(tf.bool, shape=())
tf_sess = tf.Session()
tf_sess.run(tf.global_variables_initializer())
if weights_file:
saver = tf.train.Saver()
filepath = maybe_download_weights_from_s3(weights_file)
tar = tarfile.open(filepath)
tar.extractall(path=SAVED_MODEL_DIR)
tar.close()
model_file = tf.train.latest_checkpoint(SAVED_MODEL_DIR +
'/free_checkpoint_tf')
saver.restore(tf_sess, model_file)
wrapped_model = TFClassifier(input_ph=input_ph,
output=model.pre_softmax,
labels_ph=labels_ph,
loss=model.xent,
learning=training_ph,
sess=tf_sess,
clip_values=(0, 255),
**wrapper_kwargs)
return wrapped_model
def _cnn_mnist_tf(input_shape):
labels_tf = tf.placeholder(tf.float32, [None, 10])
inputs_tf = tf.placeholder(tf.float32, [None] + list(input_shape))
# Define the tensorflow graph
conv = tf.layers.conv2d(inputs_tf, 4, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(fc, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=labels_tf))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train_tf = optimizer.minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
classifier = TFClassifier((0, 1),
inputs_tf,
logits,
loss=loss,
train=train_tf,
output_ph=labels_tf,
sess=sess)
return classifier
def get_madry_model(model_kwargs, wrapper_kwargs, weights_file=None):
model = make_madry_model(**model_kwargs)
input_ph = model.x_input
labels_ph = model.y_input
training_ph = tf.placeholder(tf.bool, shape=())
# Restore the checkpoint
saver = tf.train.Saver()
tf_sess = tf.Session()
saved_model_dir = paths.DockerPaths().saved_model_dir
filepath = os.path.join(saved_model_dir, weights_file)
model_file = tf.train.latest_checkpoint(filepath)
saver.restore(tf_sess, model_file)
wrapped_model = TFClassifier(
input_ph=input_ph,
output=model.pre_softmax,
labels_ph=labels_ph,
loss=model.xent,
learning=training_ph,
sess=tf_sess,
clip_values=(0, 255),
**wrapper_kwargs
)
return wrapped_model
def get_classifier_tf():
"""
Standard Tensorflow classifier for unit testing.
The following hyper-parameters were used to obtain the weights and biases:
learning_rate: 0.01
batch size: 10
number of epochs: 2
optimizer: tf.train.AdamOptimizer
:return: TFClassifier, tf.Session()
"""
import tensorflow as tf
from art.classifiers import TFClassifier
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(
input_ph,
1,
7,
activation=tf.nn.relu,
kernel_initializer=_tf_weights_loader('MNIST', 'W', 'CONV2D'),
bias_initializer=_tf_weights_loader('MNIST', 'B', 'CONV2D'))
conv = tf.layers.max_pooling2d(conv, 4, 4)
flattened = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(
flattened,
10,
kernel_initializer=_tf_weights_loader('MNIST', 'W', 'DENSE'),
bias_initializer=_tf_weights_loader('MNIST', 'B', 'DENSE'))
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Train the classifier
tfc = TFClassifier(clip_values=(0, 1),
input_ph=input_ph,
logits=logits,
output_ph=output_ph,
train=train,
loss=loss,
learning=None,
sess=sess)
return tfc, sess
def get_iris_classifier_tf():
"""
Standard Tensorflow classifier for unit testing.
The following hyper-parameters were used to obtain the weights and biases:
- learning_rate: 0.01
- batch size: 5
- number of epochs: 200
- optimizer: tf.train.AdamOptimizer
The model is trained of 70% of the dataset, and 30% of the training set is used as validation split.
:return: The trained model for Iris dataset and the session.
:rtype: `tuple(TFClassifier, tf.Session)`
"""
import tensorflow as tf
from art.classifiers import TFClassifier
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 4])
output_ph = tf.placeholder(tf.int32, shape=[None, 3])
# Define the tensorflow graph
dense1 = tf.layers.dense(
input_ph,
10,
kernel_initializer=_tf_weights_loader('IRIS', 'W', 'DENSE1'),
bias_initializer=_tf_weights_loader('IRIS', 'B', 'DENSE1'))
dense2 = tf.layers.dense(
dense1,
10,
kernel_initializer=_tf_weights_loader('IRIS', 'W', 'DENSE2'),
bias_initializer=_tf_weights_loader('IRIS', 'B', 'DENSE2'))
logits = tf.layers.dense(
dense2,
3,
kernel_initializer=_tf_weights_loader('IRIS', 'W', 'DENSE3'),
bias_initializer=_tf_weights_loader('IRIS', 'B', 'DENSE3'))
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
# Tensorflow session and initialization
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Train the classifier
tfc = TFClassifier(clip_values=(0, 1),
input_ph=input_ph,
logits=logits,
output_ph=output_ph,
train=None,
loss=loss,
learning=None,
sess=sess,
channel_index=1)
return tfc, sess
def CIFAR(**args):
# Get data for cifar
x_train, y_train, x_test, y_test, x_val, y_val, _min, _max = load_train_eval_test('cifar')
# Params
params = cifar_param
save_path = params.save_path
# Load CNN
if args['cnn_path'] is not None:
if args['last_name'] is not None:
sess_cnn, cnn = load_cnn(params.cnn, args['cnn_path'] + args['last_name'], load_last=False)
else:
sess_cnn, cnn = load_cnn(params.cnn, args['cnn_path'], load_last=True)
classifier_cnn = KerasClassifier(model=cnn.model, clip_values=(_min, _max))
# Load VQ
if args['vq_path'] is not None:
path = args['vq_path']
vq_hard = load_vq(path, config, args['last_name'])
classifer_e = TFClassifier(input_ph=vq_hard.input_x, output = vq_hard.pre_e, labels_ph= vq_hard.y_true,clip_values=(_min, _max), sess=vq_hard.sess, loss=vq_hard.loss_c2)
vq_models = [vq_hard]
# Black box test
paths = ['save_val/attacks/CIFAR_pool3_BIM_copy_x_test_advs.npy', 'save_val/attacks/CIFAR_pool3_BIM_large_copy_x_test_advs.npy',\
'save_val/attacks/CIFAR_pool3_FGSM_copy_x_test_advs.npy', 'save_val/attacks/CIFAR_pool3_FGSM_large_copy_x_test_advs.npy']
for index, black_flag in enumerate([args['BIM_black'], args['BIM_black_large'], args['FGSM_black'], args['FGSM_black_large']]):
if black_flag:
print(paths[index])
attack_file_path = paths[index]
if args['cnn_path'] is not None:
accs = cnn_black_box_test(attack_file_path, cnn, y_test, x_test)
if args['vq_path'] is not None:
vq_accs_q, vq_accs_e = vq_black_box_test(attack_file_path, [vq_hard], y_test, x_test)
# White box
attacks = []
if args['BIM_white']:
attacks.append(BasicIterativeMethod)
if args['FGSM_white']:
attacks.append(FastGradientMethod)
if args['CIM_white']:
attacks.append(CarliniLInfMethod)
if attacks != []:
eps = eval(args['eps'])
if args['cnn_path'] is not None:
cnn_white_box_test(eps, attacks, [sess_cnn], [classifier_cnn], [cnn.model], ['cnn'], x_test, y_test)
if args['vq_path'] is not None:
vq_white_box_test(eps, attacks, [vq_hard.sess], [classifer_e], vq_models, ['VQ'], x_test, y_test
)
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build a TFClassifier
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(fc, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Train the classifier
tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss,
None, sess)
tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=2)
# Attack
# TODO Launch with all possible attacks
attack_params = {
"attacker": "newtonfool",
"attacker_params": {
"max_iter": 5
}
}
up = UniversalPerturbation(tfc)
x_train_adv = up.generate(x_train, **attack_params)
self.assertTrue((up.fooling_rate >= 0.2) or not up.converged)
x_test_adv = x_test + up.v
self.assertFalse((x_test == x_test_adv).all())
train_y_pred = np.argmax(tfc.predict(x_train_adv), axis=1)
test_y_pred = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == test_y_pred).all())
self.assertFalse((np.argmax(y_train, axis=1) == train_y_pred).all())
def test_loss_gradient(self):
# Get MNIST
(_, _), (x_test, y_test), _, _ = load_mnist()
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
# Test gradient
tfc = TFClassifier(None, self._input_ph, self._logits, self._output_ph,
None, self._loss, None, self._sess)
grads = tfc.loss_gradient(x_test, y_test)
self.assertTrue(np.array(grads.shape == (NB_TEST, 28, 28, 1)).all())
self.assertTrue(np.sum(grads) != 0)
def cw2(self, sess):
from art.classifiers import TFClassifier
tfc = TFClassifier(clip_values=(0, 1),
input_ph=self.x_image,
logits=self.pre_softmax,
output_ph=self.y_input,
train=None,
loss=self.xent,
learning=None,
sess=sess)
return tfc
def test_class_gradient(self):
# Get MNIST
(_, _), (x_test, y_test), _, _ = load_mnist()
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
# Test gradient
tfc = TFClassifier((0, 1), self._input_ph, self._logits, None, None,
None, None, self._sess)
grads = tfc.class_gradient(x_test)
self.assertTrue(
np.array(grads.shape == (NB_TEST, 10, 28, 28, 1)).all())
self.assertTrue(np.sum(grads) != 0)
tf.reset_default_graph()
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build a TFClassifier
# Define input and output placeholders
self._input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
self._output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(self._input_ph, 4, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
self._logits = tf.layers.dense(fc, 10)
# Train operator
self._loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(logits=self._logits, onehot_labels=self._output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
self._train = optimizer.minimize(self._loss)
# Tensorflow session and initialization
self._sess = tf.Session()
self._sess.run(tf.global_variables_initializer())
# Get MNIST
batch_size, nb_train, nb_test = 100, 1000, 10
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:nb_train], y_train[:nb_train]
x_test, y_test = x_test[:nb_test], y_test[:nb_test]
# Train the classifier
tfc = TFClassifier((0, 1), self._input_ph, self._logits, self._output_ph, self._train, self._loss, None,
self._sess)
tfc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=2)
# Attack
nf = NewtonFool(tfc)
nf.set_params(max_iter=5)
x_test_adv = nf.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
y_pred = tfc.predict(x_test)
y_pred_adv = tfc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= y_pred_adv_max).all())
def test_failure_attack(self):
"""
Test the corner case when attack is failed.
:return:
"""
# Build a TFClassifier
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(fc, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Train the classifier
tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss,
None, sess)
tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=10)
# Failure attack
cl2m = CarliniL2Method(classifier=tfc,
targeted=True,
max_iter=0,
binary_search_steps=0,
learning_rate=0,
initial_const=1)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = cl2m.generate(x_test, **params)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
np.testing.assert_almost_equal(x_test, x_test_adv, 3)
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
labels_ph = tf.placeholder(tf.int32, shape=[None, 10])
training_ph = tf.placeholder(tf.bool, shape=())
x = tf.layers.conv2d(input_ph,
filters=4,
kernel_size=(5, 5),
activation=tf.nn.relu)
x = tf.layers.max_pooling2d(x, 2, 2)
x = tf.layers.conv2d(x,
filters=10,
kernel_size=(5, 5),
activation=tf.nn.relu)
x = tf.layers.max_pooling2d(x, 2, 2)
x = tf.layers.flatten(x)
x = tf.layers.dense(x, 100, activation=tf.nn.relu)
logits = tf.layers.dense(x, 10)
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=labels_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train_op = optimizer.minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if weights_file:
# Load Model using preferred save/restore method
filepath = maybe_download_weights_from_s3(weights_file)
tar = tarfile.open(filepath)
tar.extractall(path=paths.runtime_paths().saved_model_dir)
tar.close()
# Restore variables...
wrapped_model = TFClassifier(clip_values=(0.0, 1.0),
input_ph=input_ph,
output=logits,
labels_ph=labels_ph,
train=train_op,
loss=loss,
learning=training_ph,
sess=sess,
**wrapper_kwargs)
return wrapped_model
def test_fit_predict(self):
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN]
x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST]
# Test fit and predict
tfc = TFClassifier(None, self._input_ph, self._logits, self._output_ph,
self._train, self._loss, None, self._sess)
tfc.fit(x_train, y_train, batch_size=100, nb_epochs=1)
preds = tfc.predict(x_test)
preds_class = np.argmax(preds, axis=1)
trues_class = np.argmax(y_test, axis=1)
acc = np.sum(preds_class == trues_class) / len(trues_class)
print("\nAccuracy: %.2f%%" % (acc * 100))
self.assertGreater(acc, 0.1)
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None):
input_ph = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
labels_ph = tf.placeholder(tf.int32, shape=[None, 10])
training_ph = tf.placeholder(tf.bool, shape=())
# Conditional for handling training phase or inference phase
output = tf.cond(
training_ph,
true_fn=lambda: _training_pass(input_ph),
false_fn=lambda: _inference_pass(input_ph),
)
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=output,
onehot_labels=labels_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.003)
train_op = optimizer.minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if weights_file:
# Load Model using preferred save/restore method
filepath = maybe_download_weights_from_s3(weights_file)
tar = tarfile.open(filepath)
tar.extractall(path=paths.runtime_paths().saved_model_dir)
tar.close()
# Restore variables...
wrapped_model = TFClassifier(clip_values=(0.0, 1.0),
input_ph=input_ph,
output=output,
labels_ph=labels_ph,
train=train_op,
loss=loss,
learning=training_ph,
sess=sess,
**wrapper_kwargs)
return wrapped_model
def __call__(self, session):
inputs = tf.compat.v1.placeholder(tf.float32,
shape=[
None, self._image_height,
self._image_width,
self._n_channels
])
target_ys = tf.compat.v1.placeholder(tf.float32,
shape=[None, self._n_classes])
logits = self.calculate_logits(inputs)
target_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=target_ys,
logits=logits))
restorer = tf.compat.v1.train.Saver()
restorer.restore(session, self._checkpoint_path)
return TFClassifier(clip_values=(-1, 1),
input_ph=inputs,
output=logits,
labels_ph=target_ys,
sess=session,
loss=target_loss)
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build a TFClassifier
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(fc, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Get MNIST
(x_train, y_train), (x_test, _) = self.mnist
# Train the classifier
tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss,
None, sess)
tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=2)
# Attack
# import time
nf = NewtonFool(tfc, max_iter=5)
# print("Test Tensorflow....")
# starttime = time.clock()
# x_test_adv = nf.generate(x_test, batch_size=1)
# self.assertFalse((x_test == x_test_adv).all())
# endtime = time.clock()
# print(1, endtime - starttime)
# starttime = time.clock()
# x_test_adv = nf.generate(x_test, batch_size=10)
# endtime = time.clock()
# print(10, endtime - starttime)
# starttime = time.clock()
x_test_adv = nf.generate(x_test, batch_size=100)
# endtime = time.clock()
# print(100, endtime - starttime)
#
# starttime = time.clock()
# x_test_adv = nf.generate(x_test, batch_size=1000)
# endtime = time.clock()
# print(1000, endtime - starttime)
self.assertFalse((x_test == x_test_adv).all())
y_pred = tfc.predict(x_test)
y_pred_adv = tfc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= y_pred_adv_max).all())
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build a TFClassifier
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
flattened = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(flattened, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Train the classifier
tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss,
None, sess)
tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=2)
# Attack
attack_params = {
"max_translation": 10.0,
"num_translations": 3,
"max_rotation": 30.0,
"num_rotations": 3
}
attack_st = SpatialTransformation(tfc)
x_train_adv = attack_st.generate(x_train, **attack_params)
self.assertTrue(abs(x_train_adv[0, 8, 13, 0] - 0.8066048) <= 0.01)
# self.assertTrue(abs(attack_st.fooling_rate - 0.948) <= 0.01)
self.assertTrue(attack_st.attack_trans_x == -3)
self.assertTrue(attack_st.attack_trans_y == -3)
self.assertTrue(attack_st.attack_rot == -30.0)
x_test_adv = attack_st.generate(x_test)
self.assertTrue(abs(x_test_adv[0, 14, 14, 0] - 0.6941315) <= 0.01)
sess.close()
tf.reset_default_graph()
# x_test = x_test[0:20, :]
# y_test = y_test[0:20]
data_path = config['data_path']
cifar = cifar10_input.CIFAR10Data(data_path)
x_test = cifar.eval_data.xs[0:50, :]
y_test = cifar.eval_data.ys[0:50]
# print(x_test.shape)
# print(min_pixel_value)
# print(max_pixel_value)
with tf.Session() as sess:
saver.restore(sess, model_ckpt)
classifier = TFClassifier(input_ph=input_ph,
logits=logits,
sess=sess,
loss=loss,
output_ph=labels_ph)
predictions = classifier.predict(x_test)
print(x_test[0])
# print(predictions)
print(np.argmax(predictions, axis=1))
accuracy = np.sum(
np.argmax(predictions, axis=1) == y_test) / len(y_test)
print('Accuracy on benign test examples: {}%'.format(accuracy * 100))
# FGSM
attack = FastGradientMethod(classifier=classifier,
eps=epsilon,
def test_nb_classes(self):
# Start to test
tfc = TFClassifier(None, self._input_ph, self._logits, self._output_ph,
self._train, None, None, self._sess)
self.assertTrue(tfc.nb_classes == 10)
x = tf.layers.max_pooling2d(x, 2, 2)
x = tf.layers.conv2d(x, filters=10, kernel_size=5, activation=tf.nn.relu)
x = tf.layers.max_pooling2d(x, 2, 2)
x = tf.layers.flatten(x)
x = tf.layers.dense(x, 100, activation=tf.nn.relu)
logits = tf.layers.dense(x, 10)
loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=labels_ph))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
#print('success')
# Step 3: Create the ART classifier
classifier = TFClassifier(clip_values=(min_pixel_value, max_pixel_value), input_ph=input_ph, output=logits,
labels_ph=labels_ph, train=train, loss=loss, learning=None, sess=sess)
# Step 4: Train the ART classifier
classifier.fit(x_train, y_train, batch_size=64, nb_epochs=3)
print('successful')
# Step 5: Evaluate the ART classifier on benign test examples
predictions = classifier.predict(x_test)
accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)
print('Accuracy on benign test examples: {}%'.format(accuracy * 100))
# Step 6: Generate adversarial test examples
attack = FastGradientMethod(classifier=classifier, eps=0.2)
x_test_adv = attack.generate(x=x_test)
# Step 7: Evaluate the ART classifier on adversarial test examples
def test_tfclassifier(self):
"""
First test with the TFClassifier.
:return:
"""
# Build a TFClassifier
# Define input and output placeholders
input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
output_ph = tf.placeholder(tf.int32, shape=[None, 10])
# Define the tensorflow graph
conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu)
conv = tf.layers.max_pooling2d(conv, 2, 2)
fc = tf.contrib.layers.flatten(conv)
# Logits layer
logits = tf.layers.dense(fc, 10)
# Train operator
loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(logits=logits,
onehot_labels=output_ph))
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train = optimizer.minimize(loss)
# Tensorflow session and initialization
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Get MNIST
(x_train, y_train), (x_test, y_test) = self.mnist
# Train the classifier
tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss,
None, sess)
tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=10)
# First attack
clinfm = CarliniLInfMethod(classifier=tfc,
targeted=True,
max_iter=10,
eps=0.5)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = clinfm.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', target)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target == y_pred_adv).any())
# Second attack
clinfm = CarliniLInfMethod(classifier=tfc,
targeted=False,
max_iter=10,
eps=0.5)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = clinfm.generate(x_test, **params)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', target)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(sum(target != y_pred_adv) / float(len(target))))
self.assertTrue((target != y_pred_adv).any())
# Third attack
clinfm = CarliniLInfMethod(classifier=tfc,
targeted=False,
max_iter=10,
eps=0.5)
params = {}
x_test_adv = clinfm.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(x_test), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', y_pred)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(sum(y_pred != y_pred_adv) / float(len(y_pred))))
self.assertTrue((y_pred != y_pred_adv).any())
# First attack without batching
clinfmwob = CarliniLInfMethod(classifier=tfc,
targeted=True,
max_iter=10,
eps=0.5,
batch_size=1)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = clinfmwob.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', target)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(sum(target == y_pred_adv) / float(len(target))))
self.assertTrue((target == y_pred_adv).any())
# Second attack without batching
clinfmwob = CarliniLInfMethod(classifier=tfc,
targeted=False,
max_iter=10,
eps=0.5,
batch_size=1)
params = {'y': random_targets(y_test, tfc.nb_classes)}
x_test_adv = clinfmwob.generate(x_test, **params)
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', target)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(sum(target != y_pred_adv) / float(len(target))))
self.assertTrue((target != y_pred_adv).any())
# Third attack without batching
clinfmwob = CarliniLInfMethod(classifier=tfc,
targeted=False,
max_iter=10,
eps=0.5,
batch_size=1)
params = {}
x_test_adv = clinfmwob.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(x_test), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
logger.debug('CW0 Target: %s', y_pred)
logger.debug('CW0 Actual: %s', y_pred_adv)
logger.info('CW0 Success Rate: %.2f',
(sum(y_pred != y_pred_adv) / float(len(y_pred))))
self.assertTrue((y_pred != y_pred_adv).any())
def CIFAR_VQ(**args):
# Get data for cifar
x_train, y_train, x_test, y_test, x_val, y_val, _min, _max = load_train_eval_test(
'cifar')
x_val_adv = np.load('./save_val/cifar_01_pool3_x_val_adv.npy')
x_val = np.append(x_val, x_val_adv, axis=0)
y_val = np.append(y_val, y_val, axis=0)
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=
15, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False) # randomly flip images
datagen.fit(x_train)
# Params
params = cifar_param
save_path = params.save_path
# Adv train VQ
batch_size = 128
path = args['load_path']
vq_hard = load_vq(path, config, args['last_name'])
vq_hard.build_path()
classifer_e = TFClassifier(input_ph=vq_hard.input_x,
output=vq_hard.pre_e,
labels_ph=vq_hard.y_true,
clip_values=(_min, _max),
sess=vq_hard.sess,
loss=vq_hard.loss_c2)
classifer_q_path = TFClassifier(input_ph=vq_hard.input_x,
output=vq_hard.pre_path,
labels_ph=vq_hard.y_true,
clip_values=(_min, _max),
sess=vq_hard.sess,
loss=vq_hard.loss_path)
if args['PGD'] == False:
vq_attacks = [
FastGradientMethod(classifer_e,
eps=args['eps'],
batch_size=batch_size)
]
else:
vq_attacks = [
ProjectedGradientDescent(classifer_e,
eps=args['eps'],
eps_step=0.01,
batch_size=batch_size)
]
# eps = 0.02
# vq_attacks = [FastGradientMethod(classifer_e, eps = eps, batch_size = batch_size),FastGradientMethod(classifer_q_path, eps = eps, batch_size = batch_size)]
vq_hard.params['save_path'] = path + '/adv_eps_' + str(
args['eps']) + '_ratio_' + str(args['ratio']) + '_PGD_' + str(
args['PGD']) + '/'
if os.path.exists(vq_hard.params['save_path']) == False:
os.mkdir(vq_hard.params['save_path'])
else:
i = 0
while os.path.exists(vq_hard.params['save_path']):
vq_hard.params['save_path'] = vq_hard.params[
'save_path'][:-1] + '_' + str(i) + '/'
os.mkdir(vq_hard.params['save_path'])
print('Save at', vq_hard.params['save_path'])
vq_hard.params[
'save_best'] = 'q' if args['save_best'] is None else args['save_best']
vq_hard.params['early_stop'] = args['early_stop']
adv_train_vq(vq_hard,
vq_attacks,
x_train,
y_train,
x_val,
y_val,
ratio=args['ratio'],
epochs=args['epochs'],
datagen=datagen)
def MNIST_VQ(**args):
# Get val data
x_train, y_train, x_test, y_test, x_val, y_val, _min, _max = load_train_eval_test(
'mnist', fashion=args['fashion'])
if args['fashion'] == False:
x_val_adv = np.load('./save_val/mnist_x_val_adv_vgg.npy')
else:
x_val_adv = np.load('./save_val/fashion_x_val_adv_vgg.npy')
x_val = np.append(x_val, x_val_adv, axis=0)
y_val = np.append(y_val, y_val, axis=0)
# Params
if args['fashion'] == False:
params = mnist_param
else:
params = fashion_param
save_path = params.save_path
# Adv train VQ
batch_size = 128
path = args['load_path']
vq_hard = load_vq(path, config, args['last_name'])
classifer_e = TFClassifier(input_ph=vq_hard.input_x,
output=vq_hard.pre_e,
labels_ph=vq_hard.y_true,
clip_values=(_min, _max),
sess=vq_hard.sess,
loss=vq_hard.loss_c2)
if args['PGD'] == False:
vq_attacks = [
FastGradientMethod(classifer_e,
eps=args['eps'],
batch_size=batch_size)
]
else:
vq_attacks = [
ProjectedGradientDescent(classifer_e,
eps=args['eps'],
eps_step=0.01,
batch_size=batch_size)
]
vq_hard.params['save_path'] = path + '/adv_eps_' + str(
args['eps']) + '_ratio_' + str(args['ratio']) + '_PGD_' + str(
args['PGD']) + '/'
if os.path.exists(vq_hard.params['save_path']) == False:
os.mkdir(vq_hard.params['save_path'])
else:
i = 0
while os.path.exists(vq_hard.params['save_path']):
vq_hard.params['save_path'] = vq_hard.params[
'save_path'][:-1] + '_' + str(i) + '/'
os.mkdir(vq_hard.params['save_path'])
print('Save at', vq_hard.params['save_path'])
vq_hard.params[
'save_best'] = 'q' if args['save_best'] is None else args['save_best']
vq_hard.params['early_stop'] = args['early_stop']
adv_train_vq(vq_hard,
vq_attacks,
x_train,
y_train,
x_val,
y_val,
ratio=args['ratio'],
epochs=args['epochs'])
def test_input_shape(self):
# Start to test
tfc = TFClassifier(None, self._input_ph, self._logits, self._output_ph,
self._train, None, None, self._sess)
self.assertTrue(np.array(tfc.input_shape == (28, 28, 1)).all())