def calculate_accuracy_adv_spatial_smoothing(sess,
x,
y,
predictions,
y_test,
X_test,
X_test_adv,
eval_param=None):
print("\n===Calculating the accuracy with feature squeezing...")
for width in range(1, 11):
# height = width
for height in range(1, 11):
X_squeezed = median_filter_np(X_test, width, height)
X_adv_squeezed = median_filter_np(X_test_adv, width, height)
accuracy_leg, _ = model_test_eval(sess,
x,
y,
predictions,
X_squeezed,
y_test,
args=eval_param)
accuracy_mal, _ = model_test_eval(sess,
x,
y,
predictions,
X_adv_squeezed,
y_test,
args=eval_param)
print(
"Width: %2d, Height: %2d, Accuracy_legitimate: %.2f, Accuracy_malicious: %.2f"
% (width, height, accuracy_leg, accuracy_mal))
def calculate_accuracy_adv_feature_squeezing(sess,
x,
y,
logits,
logits_bin,
X_adv,
y_test,
eval_params=None):
accuracy, pred = model_test_eval(sess,
x,
y,
logits,
X_adv,
y_test,
args=eval_params)
print('Test accuracy of white-box on adversarial '
'examples: {:.3f}'.format(accuracy))
accuracy, pred = model_test_eval(sess,
x,
y,
logits_bin,
X_adv,
y_test,
args=eval_params)
print('Test accuracy of white-box on feature squeezing adversarial '
'examples: {:.3f}'.format(accuracy))
def eval(mu, sigma, add_dropout, batch_size):
"""
Evaluate a GTSRB model
:param mu: A 0-D Tensor or Python value of type `dtype`. The mean of the
truncated normal distribution.
:param sigma: A 0-D Tensor or Python value of type `dtype`. The standard deviation
of the truncated normal distribution.
:param add_dropout: if or not use dropout.
:param batch_size: size of training batches
:return: a dictionary with:
* model training accuracy and loss on training data
* model validating accuracy and loss on validation data
* accuracy on test dataset by class lebel
"""
np.random.seed(0)
assert keras.backend.backend() == "tensorflow"
# # Create TF session and set as Keras backend session
sess = tf.Session()
keras.backend.set_session(sess)
# Get GTSRB data
_, _, X_test, y_test = load_data(train_data_dir, test_data_dir)
# One-Hot Encode
label_binarizer = LabelBinarizer()
y_test = label_binarizer.fit_transform(y_test)
print(y_test.shape)
# Define input and output TF placeholders
x = tf.placeholder(tf.float32, (None, 32, 32, 3))
y = tf.placeholder(tf.int32, (None))
one_hot_y = tf.one_hot(y, 43)
# model = LeNet(model_dir)
# logits, _ = model(x, add_dropout)
model = LeNet(model_dir, mu, sigma)
logits = model(x)
print('loading LeNet ...')
# Parameters required by evaluating
eval_params = {'batch_size': batch_size}
accuracy, pred = model_test_eval(sess,
x,
one_hot_y,
logits,
X_test,
y_test,
args=eval_params)
print('Test accuracy of LeNet on legitimate test '
'examples: {:.3f}'.format(accuracy))
def eval(mu, sigma, add_dropout, batch_size):
"""
Evaluate a GTSRB model
:param mu: A 0-D Tensor or Python value of type `dtype`. The mean of the
truncated normal distribution.
:param sigma: A 0-D Tensor or Python value of type `dtype`. The standard deviation
of the truncated normal distribution.
:param add_dropout: if or not use dropout.
:param batch_size: size of training batches
:return: a dictionary with:
* model training accuracy and loss on training data
* model validating accuracy and loss on validation data
* accuracy on test dataset by class lebel
"""
np.random.seed(0)
assert keras.backend.backend() == "tensorflow"
# # Create TF session and set as Keras backend session
sess = tf.Session()
keras.backend.set_session(sess)
# Get MNIST data
mnist = input_data.read_data_sets('/tmp/', one_hot=True, reshape=False)
X_train, y_train, X_test, y_test = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
# One-Hot Encode
# label_binarizer = LabelBinarizer()
# y_test = label_binarizer.fit_transform(y_test)
# print(y_test.shape)
# Define input and output TF placeholders
x = tf.placeholder(tf.float32, (None, 28, 28, 1))
y = tf.placeholder(tf.float32, (None, 10))
# one_hot_y = tf.one_hot(y, 43)
model = MNIST(model_dir)
logits, _ = model(x, add_dropout)
print('loading MNIST ...')
# Parameters required by evaluating
eval_params = {'batch_size': batch_size}
accuracy, pred = model_test_eval(sess,
x,
y,
logits,
X_test,
y_test,
args=eval_params)
print('Test accuracy of MNIST on legitimate test '
'examples: {:.3f}'.format(accuracy))
def gtsrb_whitebox(attack, mu, sigma, batch_size, eps, steps, kappa, alpha):
"""
GTSRB tutorial for the white-box attack.
:param attack: attack type(FGSM, I-FGSM, Carlini)
:param mu: A 0-D Tensor or Python value of type `dtype`. The mean of the
truncated normal distribution.
:param sigma: A 0-D Tensor or Python value of type `dtype`. The standard deviation
of the truncated normal distribution.
:param dropout: dropout value
:param add_dropout: if or not dropout.
:param batch_size: size of training batches
:param eps: the epsilon (input variation parameter)
:return: a dictionary with:
* white-box model accuracy on test set
* white-box model accuracy on adversarial examples transferred
from the substitute model
"""
np.random.seed(0)
assert keras.backend.backend() == "tensorflow"
# # Create TF session and set as Keras backend session
sess = tf.Session()
keras.backend.set_session(sess)
# Get GTSRB data
_, _, X_test, y_test = load_data(train_data_dir, test_data_dir)
# One-Hot Encode
label_binarizer = LabelBinarizer()
y_test = label_binarizer.fit_transform(y_test)
# Define input and output TF placeholders
x = tf.placeholder(tf.float32, (None, 32, 32, 3))
y = tf.placeholder(tf.int32, (None))
one_hot_y = tf.one_hot(y, 43)
print('Preparing the white-box model LeNet.')
model = LeNet(model_dir, mu, sigma)
logits = model(x)
# Parameters required by evaluating
eval_params = {'batch_size': batch_size}
accuracy, pred = model_test_eval(sess,
x,
one_hot_y,
logits,
X_test,
y_test,
args=eval_params)
print('Test accuracy of white-box on legitimate test '
'examples: {:.3f}'.format(accuracy))
model = AlexNet(model_dir, mu, sigma)
logits = model(x)
# take the random step in the RAND+FGSM
if attack == "rand_fgsm":
X_test = np.clip(
X_test + alpha * np.sign(np.random.randn(*X_test.shape)), 0.0, 1.0)
eps -= alpha
grad = generate_gradient(x, logits, one_hot_y, loss='training')
# FGSM and RAND_FGSM one-shot attack
if attack in ["fgsm", "rand_fgsm"]:
adv_x = fgsm(x, grad, eps, clipping=True)
if attack == 'ifgsm':
adv_x = iteration_fgsm(model,
x,
one_hot_y,
steps,
eps,
loss='training')
if attack == 'Carlini':
X_test = X_test[0:128]
Y_test = y_test[0:128]
# cli = CarliniL2(sess, model, targeted=False, max_iterations=1000, confidence=0, batch_size=1)
cli = CarliniLi(sess, model, targeted=False, max_iterations=1000)
X_adv = cli.attack(X_test, Y_test)
r = np.clip(X_adv - X_test, -eps, eps)
X_adv = X_test + r
accuracy, pred = model_test_eval(sess,
x,
one_hot_y,
logits,
X_adv,
y_test,
args=eval_params)
print('Test accuracy of white-box on adversarial examples crafted by '
'Carlini Attack: {:.3f}'.format(accuracy))
display_leg_adv_sample(X_test, X_adv)
return
# compute the adversarial examples and evaluate
X_adv = batch_eval(sess, [x, y], [adv_x], [X_test, y_test],
args=eval_params)[0]
accuracy, pred = model_test_eval(sess,
x,
one_hot_y,
logits,
X_adv,
y_test,
args=eval_params)
print('Test accuracy of white-box on adversarial '
'examples: {:.3f}'.format(accuracy))
display_leg_adv_sample(X_test, X_adv)