def test_update_image_classification_sw(art_warning, fix_get_mnist_subset, image_dl_estimator):
try:
from art.attacks.evasion import ProjectedGradientDescent
classifier, _ = image_dl_estimator(from_logits=False)
swd = SummaryWriterDefault(summary_writer=True, ind_1=True, ind_2=True, ind_3=True, ind_4=True)
attack = ProjectedGradientDescent(
estimator=classifier, max_iter=10, eps=0.3, eps_step=0.03, batch_size=5, verbose=False, summary_writer=swd
)
(x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset
attack.generate(x=x_train_mnist, y=y_train_mnist)
assert all(attack.summary_writer.i_1 == [False, False, False, False, False])
if np.ndim(attack.summary_writer.i_2) != 0:
assert len(attack.summary_writer.i_2) == 5
np.testing.assert_almost_equal(attack.summary_writer.i_3["0"], np.array([0.0, 0.0, 0.0, 0.0, 0.0]))
np.testing.assert_almost_equal(attack.summary_writer.i_4["0"], np.array([0.0, 0.0, 0.0, 0.0, 0.0]))
except ARTTestException as e:
art_warning(e)
def test_binary_keras_instantiation_and_attack_pgd(art_warning):
tf.compat.v1.disable_eager_execution()
try:
x, y = sklearn.datasets.make_classification(n_samples=10000,
n_features=20,
n_informative=5,
n_redundant=2,
n_repeated=0,
n_classes=2)
train_x, test_x, train_y, test_y = sklearn.model_selection.train_test_split(
x, y, test_size=0.2)
train_x = train_x.astype(np.float32)
test_x = test_x.astype(np.float32)
model = tf.keras.models.Sequential([
tf.keras.layers.Dense(128,
activation=tf.nn.relu,
input_shape=(20, )),
tf.keras.layers.Dense(1, activation=tf.nn.sigmoid),
])
model.summary()
model.compile(optimizer=tf.optimizers.Adam(),
loss="binary_crossentropy",
metrics=["accuracy"])
classifier = KerasClassifier(model=model)
classifier.fit(train_x, train_y, nb_epochs=5)
pred = classifier.predict(test_x)
attack = ProjectedGradientDescent(estimator=classifier, eps=0.5)
x_test_adv = attack.generate(x=test_x)
adv_predictions = classifier.predict(x_test_adv)
assert (adv_predictions != pred).any()
except ARTTestException as e:
art_warning(e)
def test_pgd(self):
from art.attacks.evasion import ProjectedGradientDescent
attack = ProjectedGradientDescent(estimator=self.obj_detect,
max_iter=2)
x_test_adv = attack.generate(x=self.x_test, y=self.y_test)
np.testing.assert_raises(AssertionError, np.testing.assert_array_equal,
x_test_adv, self.x_test)
def test_update_image_classification_bool_str(art_warning, fix_get_mnist_subset, image_dl_estimator, summary_writer):
try:
from art.attacks.evasion import ProjectedGradientDescent
classifier, _ = image_dl_estimator(from_logits=False)
attack = ProjectedGradientDescent(
estimator=classifier,
max_iter=10,
eps=0.3,
eps_step=0.03,
batch_size=5,
verbose=False,
summary_writer=summary_writer,
)
(x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset
attack.generate(x=x_train_mnist, y=y_train_mnist)
except ARTTestException as e:
art_warning(e)
def test_update_image_object_detection_sw(art_warning, fix_get_mnist_subset, fix_get_rcnn):
try:
from art.attacks.evasion import ProjectedGradientDescent
frcnn = fix_get_rcnn
swd = SummaryWriterDefault(summary_writer=True, ind_1=False, ind_2=True, ind_3=True, ind_4=True)
attack = ProjectedGradientDescent(
estimator=frcnn, max_iter=10, eps=0.3, eps_step=0.03, batch_size=5, verbose=False, summary_writer=swd
)
(x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset
attack.generate(x=x_train_mnist, y=y_train_mnist)
if np.ndim(attack.summary_writer.i_2) != 0:
assert len(attack.summary_writer.i_2) == 5
np.testing.assert_almost_equal(attack.summary_writer.i_3["0"], np.array([0.2265982]))
np.testing.assert_almost_equal(attack.summary_writer.i_4["0"], np.array([0.0, 0.0, 0.0, 0.0, 0.0]))
except ARTTestException as e:
art_warning(e)
x_i_new = x_i[0:num_frames_min, :, :, :]
x_list_new.append(x_i_new)
x = np.asarray(x_list_new, dtype=float)
y_pred = pgt.predict(x=x, y_init=y_init)
##################
# evasion attack #
##################
from art.attacks.evasion import ProjectedGradientDescent
attack = ProjectedGradientDescent(estimator=pgt, eps=eps, eps_step=eps_step, batch_size=1, max_iter=20)
x_adv = attack.generate(x=x, y=y_pred)
y_pred_adv = pgt.predict(x=x_adv, y_init=y_init)
if x.dtype == object:
for i in range(x.shape[0]):
print("L_inf:", np.max(np.abs(x_adv[i] - x[i])))
else:
print("L_inf:", np.max(np.abs(x_adv - x)))
################################
# visualise adversarial images #
################################
x_vis = x_adv
y_vis = y_pred_adv
def natual(eps):
# Step 1: Load the MNIST dataset
(x_train,
y_train), (x_test,
y_test), min_pixel_value, max_pixel_value = load_mnist()
# Step 2: Create the model
import tensorflow as tf
from tensorflow.keras import Model
from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPool2D
class TensorFlowModel(Model):
"""
Standard TensorFlow model for unit testing.
"""
def __init__(self):
super(TensorFlowModel, self).__init__()
self.conv1 = Conv2D(filters=4, kernel_size=5, activation="relu")
self.conv2 = Conv2D(filters=10, kernel_size=5, activation="relu")
self.maxpool = MaxPool2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid",
data_format=None)
self.flatten = Flatten()
self.dense1 = Dense(100, activation="relu")
self.logits = Dense(10, activation="linear")
def call(self, x):
"""
Call function to evaluate the model.
:param x: Input to the model
:return: Prediction of the model
"""
x = self.conv1(x)
x = self.maxpool(x)
x = self.conv2(x)
x = self.maxpool(x)
x = self.flatten(x)
x = self.dense1(x)
x = self.logits(x)
return x
optimizer = tf.keras.optimizers.Adam(learning_rate=0.01)
def train_step(model, images, labels):
with tf.GradientTape() as tape:
predictions = model(images, training=True)
loss = loss_object(labels, predictions)
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
model = TensorFlowModel()
loss_object = tf.keras.losses.CategoricalCrossentropy(from_logits=True)
# Step 3: Create the ART classifier
classifier = TensorFlowV2Classifier(
model=model,
loss_object=loss_object,
train_step=train_step,
nb_classes=10,
input_shape=(28, 28, 1),
clip_values=(0, 1),
)
# Step 4: Train the ART classifier
classifier.fit(x_train, y_train, batch_size=64, nb_epochs=10)
# Step 5: Evaluate the ART classifier on benign test examples
predictions = classifier.predict(x_test)
accuracy = np.sum(
np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(
y_test)
print("Accuracy on benign test examples: {}%".format(accuracy * 100))
# Step 6: Generate adversarial test examples
attack = ProjectedGradientDescent(estimator=classifier,
eps=eps,
eps_step=eps / 3,
max_iter=20)
x_test_adv = attack.generate(x=x_test)
# Step 7: Evaluate the ART classifier on adversarial test examples
predictions = classifier.predict(x_test_adv)
accuracy = np.sum(
np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(
y_test)
print("Accuracy on adversarial test examples: {}%".format(accuracy * 100))
def main():
args = parse_option()
print(args)
# check args
if args.loss not in LOSS_NAMES:
raise ValueError('Unsupported loss function type {}'.format(args.loss))
if args.optimizer == 'adam':
optimizer1 = tf.keras.optimizers.Adam(lr=args.lr_1)
elif args.optimizer == 'lars':
from lars_optimizer import LARSOptimizer
# not compatible with tf2
optimizer1 = LARSOptimizer(
args.lr_1,
exclude_from_weight_decay=['batch_normalization', 'bias'])
elif args.optimizer == 'sgd':
optimizer1 = tfa.optimizers.SGDW(learning_rate=args.lr_1,
momentum=0.9,
weight_decay=1e-4)
optimizer2 = tf.keras.optimizers.Adam(lr=args.lr_2)
model_name = '{}_model-bs_{}-lr_{}'.format(args.loss, args.batch_size_1,
args.lr_1)
# 0. Load data
if args.data == 'mnist':
mnist = tf.keras.datasets.mnist
elif args.data == 'fashion_mnist':
mnist = tf.keras.datasets.fashion_mnist
print('Loading {} data...'.format(args.data))
(_, y_train), (_, y_test) = mnist.load_data()
# x_train, x_test = x_train / 255.0, x_test / 255.0
# x_train = x_train.reshape(-1, 28*28).astype(np.float32)
# x_test = x_test.reshape(-1, 28*28).astype(np.float32)
(x_train, _), (x_test, _), _, _ = load_mnist()
# print(x_train[0][0])
print(x_train.shape, x_test.shape)
# simulate low data regime for training
# n_train = x_train.shape[0]
# shuffle_idx = np.arange(n_train)
# np.random.shuffle(shuffle_idx)
# x_train = x_train[shuffle_idx][:args.n_data_train]
# y_train = y_train[shuffle_idx][:args.n_data_train]
# print('Training dataset shapes after slicing:')
print(x_train.shape, y_train.shape)
train_ds = tf.data.Dataset.from_tensor_slices(
(x_train, y_train)).shuffle(5000).batch(args.batch_size_1)
train_ds2 = tf.data.Dataset.from_tensor_slices(
(x_train, y_train)).shuffle(5000).batch(args.batch_size_2)
test_ds = tf.data.Dataset.from_tensor_slices(
(x_test, y_test)).batch(args.batch_size_1)
# 1. Stage 1: train encoder with multiclass N-pair loss
encoder = Encoder(normalize=True, activation=args.activation)
projector = Projector(args.projection_dim,
normalize=True,
activation=args.activation)
if args.loss == 'max_margin':
def loss_func(z, y):
return losses.max_margin_contrastive_loss(z,
y,
margin=args.margin,
metric=args.metric)
elif args.loss == 'npairs':
loss_func = losses.multiclass_npairs_loss
elif args.loss == 'sup_nt_xent':
def loss_func(z, y):
return losses.supervised_nt_xent_loss(
z,
y,
temperature=args.temperature,
base_temperature=args.base_temperature)
elif args.loss.startswith('triplet'):
triplet_kind = args.loss.split('-')[1]
def loss_func(z, y):
return losses.triplet_loss(z,
y,
kind=triplet_kind,
margin=args.margin)
train_loss = tf.keras.metrics.Mean(name='train_loss')
test_loss = tf.keras.metrics.Mean(name='test_loss')
# tf.config.experimental_run_functions_eagerly(True)
@tf.function
# train step for the contrastive loss
def train_step_stage1(x, y):
'''
x: data tensor, shape: (batch_size, data_dim)
y: data labels, shape: (batch_size, )
'''
with tf.GradientTape() as tape:
r = encoder(x, training=True)
z = projector(r, training=True)
# print("z", z, "y", y)
loss = loss_func(z, y)
gradients = tape.gradient(
loss, encoder.trainable_variables + projector.trainable_variables)
optimizer1.apply_gradients(
zip(gradients,
encoder.trainable_variables + projector.trainable_variables))
train_loss(loss)
@tf.function
def test_step_stage1(x, y):
r = encoder(x, training=False)
z = projector(r, training=False)
t_loss = loss_func(z, y)
test_loss(t_loss)
print('Stage 1 training ...')
for epoch in range(args.epoch):
# Reset the metrics at the start of the next epoch
train_loss.reset_states()
test_loss.reset_states()
for x, y in train_ds:
train_step_stage1(x, y)
for x_te, y_te in test_ds:
test_step_stage1(x_te, y_te)
template = 'Epoch {}, Loss: {}, Test Loss: {}'
# print(template.format(epoch + 1,
# train_loss.result(),
# test_loss.result()))
if args.draw_figures:
# projecting data with the trained encoder, projector
x_tr_proj = projector(encoder(x_train))
x_te_proj = projector(encoder(x_test))
# convert tensor to np.array
x_tr_proj = x_tr_proj.numpy()
x_te_proj = x_te_proj.numpy()
print(x_tr_proj.shape, x_te_proj.shape)
# check learned embedding using PCA
pca = PCA(n_components=2)
pca.fit(x_tr_proj)
x_te_proj_pca = pca.transform(x_te_proj)
x_te_proj_pca_df = pd.DataFrame(x_te_proj_pca, columns=['PC1', 'PC2'])
x_te_proj_pca_df['label'] = y_test
# PCA scatter plot
fig, ax = plt.subplots()
ax = sns.scatterplot('PC1',
'PC2',
data=x_te_proj_pca_df,
palette='tab10',
hue='label',
linewidth=0,
alpha=0.6,
ax=ax)
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
title = 'Data: {}\nEmbedding: {}\nbatch size: {}; LR: {}'.format(
args.data, LOSS_NAMES[args.loss], args.batch_size_1, args.lr_1)
ax.set_title(title)
fig.savefig('figs/PCA_plot_{}_{}_embed.png'.format(
args.data, model_name))
# density plot for PCA
g = sns.jointplot('PC1', 'PC2', data=x_te_proj_pca_df, kind="hex")
plt.subplots_adjust(top=0.95)
g.fig.suptitle(title)
g.savefig('figs/Joint_PCA_plot_{}_{}_embed.png'.format(
args.data, model_name))
# Stage 2: freeze the learned representations and then learn a classifier
# on a linear layer using a softmax loss
softmax = SoftmaxPred()
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_acc = tf.keras.metrics.SparseCategoricalAccuracy(name='train_ACC')
test_loss = tf.keras.metrics.Mean(name='test_loss')
test_acc = tf.keras.metrics.SparseCategoricalAccuracy(name='test_ACC')
cce_loss_obj = tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True)
@tf.function
# train step for the 2nd stage
def train_step(model, x, y):
'''
x: data tensor, shape: (batch_size, data_dim)
y: data labels, shape: (batch_size, )
'''
with tf.GradientTape() as tape:
r = model.layers[0](x, training=False)
y_preds = model.layers[1](r, training=True)
loss = cce_loss_obj(y, y_preds)
# freeze the encoder, only train the softmax layer
gradients = tape.gradient(loss, model.layers[1].trainable_variables)
optimizer2.apply_gradients(
zip(gradients, model.layers[1].trainable_variables))
train_loss(loss)
train_acc(y, y_preds)
@tf.function
def test_step(x, y):
r = encoder(x, training=False)
y_preds = softmax(r, training=False)
t_loss = cce_loss_obj(y, y_preds)
test_loss(t_loss)
test_acc(y, y_preds)
if args.write_summary:
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
train_log_dir = 'logs/{}/{}/{}/train'.format(model_name, args.data,
current_time)
test_log_dir = 'logs/{}/{}/{}/test'.format(model_name, args.data,
current_time)
train_summary_writer = tf.summary.create_file_writer(train_log_dir)
test_summary_writer = tf.summary.create_file_writer(test_log_dir)
print('Stage 2 training ...')
model = tf.keras.Sequential([encoder, softmax])
loss_object = tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True)
classifier = TensorFlowV2Classifier(
model=model,
loss_object=loss_object,
train_step=train_step,
nb_classes=10,
input_shape=(28, 28, 1),
clip_values=(0, 1),
)
# classifier.fit(x_train, y_train, batch_size=256, nb_epochs=20)
for epoch in range(args.epoch):
# Reset the metrics at the start of the next epoch
train_loss.reset_states()
train_acc.reset_states()
test_loss.reset_states()
test_acc.reset_states()
for x, y in train_ds2:
train_step(model, x, y)
if args.write_summary:
with train_summary_writer.as_default():
tf.summary.scalar('loss', train_loss.result(), step=epoch)
tf.summary.scalar('accuracy', train_acc.result(), step=epoch)
for x_te, y_te in test_ds:
test_step(x_te, y_te)
if args.write_summary:
with test_summary_writer.as_default():
tf.summary.scalar('loss', test_loss.result(), step=epoch)
tf.summary.scalar('accuracy', test_acc.result(), step=epoch)
template = 'Epoch {}, Loss: {}, Acc: {}, Test Loss: {}, Test Acc: {}'
print(
template.format(epoch + 1, train_loss.result(),
train_acc.result() * 100, test_loss.result(),
test_acc.result() * 100))
predictions = classifier.predict(x_test)
print(predictions.shape, y_test.shape)
accuracy = np.sum(np.argmax(predictions, axis=1) == y_test) / len(y_test)
print("Accuracy on benign test examples: {}%".format(accuracy * 100))
print('Stage 3 attacking ...')
attack = ProjectedGradientDescent(estimator=classifier,
eps=args.eps,
eps_step=args.eps / 3,
max_iter=20)
x_test_adv = attack.generate(x=x_test)
print('Stage 4 attacking ...')
predictions = classifier.predict(x_test_adv)
accuracy = np.sum(np.argmax(predictions, axis=1) == y_test) / len(y_test)
print("Accuracy on adversarial test examples: {}%".format(accuracy * 100))
natual(args.eps)
def main():
# Create ART object detector
frcnn = PyTorchFasterRCNN(clip_values=(0, 255),
attack_losses=[
"loss_classifier", "loss_box_reg",
"loss_objectness", "loss_rpn_box_reg"
])
# Load image 1
image_0 = cv2.imread("./10best-cars-group-cropped-1542126037.jpg")
image_0 = cv2.cvtColor(image_0, cv2.COLOR_BGR2RGB) # Convert to RGB
print("image_0.shape:", image_0.shape)
# Load image 2
image_1 = cv2.imread("./banner-diverse-group-of-people-2.jpg")
image_1 = cv2.cvtColor(image_1, cv2.COLOR_BGR2RGB) # Convert to RGB
image_1 = cv2.resize(image_1,
dsize=(image_0.shape[1], image_0.shape[0]),
interpolation=cv2.INTER_CUBIC)
print("image_1.shape:", image_1.shape)
# Stack images
image = np.stack([image_0, image_1], axis=0).astype(np.float32)
print("image.shape:", image.shape)
for i in range(image.shape[0]):
plt.axis("off")
plt.title("image {}".format(i))
plt.imshow(image[i].astype(np.uint8), interpolation="nearest")
plt.show()
# Make prediction on benign samples
predictions = frcnn.predict(x=image)
for i in range(image.shape[0]):
print("\nPredictions image {}:".format(i))
# Process predictions
predictions_class, predictions_boxes, predictions_class = extract_predictions(
predictions[i])
# Plot predictions
plot_image_with_boxes(img=image[i].copy(),
boxes=predictions_boxes,
pred_cls=predictions_class)
# Create and run attack
eps = 32
attack = ProjectedGradientDescent(estimator=frcnn,
eps=eps,
eps_step=2,
max_iter=10)
image_adv = attack.generate(x=image, y=None)
print("\nThe attack budget eps is {}".format(eps))
print("The resulting maximal difference in pixel values is {}.".format(
np.amax(np.abs(image - image_adv))))
for i in range(image_adv.shape[0]):
plt.axis("off")
plt.title("image_adv {}".format(i))
plt.imshow(image_adv[i].astype(np.uint8), interpolation="nearest")
plt.show()
predictions_adv = frcnn.predict(x=image_adv)
for i in range(image.shape[0]):
print("\nPredictions adversarial image {}:".format(i))
# Process predictions
predictions_adv_class, predictions_adv_boxes, predictions_adv_class = extract_predictions(
predictions_adv[i])
# Plot predictions
plot_image_with_boxes(img=image_adv[i].copy(),
boxes=predictions_adv_boxes,
pred_cls=predictions_adv_class)
max_iter=40,
targeted=False,
num_random_init=5,
batch_size=32,
)
# Step 4: Create the trainer object - AdversarialTrainerFBFPyTorch
# if you have apex installed, change use_amp to True
epsilon = 8.0 / 255.0
trainer = AdversarialTrainerFBFPyTorch(classifier, eps=epsilon, use_amp=False)
# Build a Keras image augmentation object and wrap it in ART
art_datagen = PyTorchDataGenerator(iterator=dataloader,
size=x_train.shape[0],
batch_size=128)
# Step 5: fit the trainer
trainer.fit_generator(art_datagen, nb_epochs=30)
x_test_pred = np.argmax(classifier.predict(x_test), axis=1)
print(
"Accuracy on benign test samples after adversarial training: %.2f%%" %
(np.sum(x_test_pred == np.argmax(y_test, axis=1)) / x_test.shape[0] * 100))
x_test_attack = attack.generate(x_test)
x_test_attack_pred = np.argmax(classifier.predict(x_test_attack), axis=1)
print(
"Accuracy on original PGD adversarial samples after adversarial training: %.2f%%"
% (np.sum(x_test_attack_pred == np.argmax(y_test, axis=1)) /
x_test.shape[0] * 100))
)
# Create a toy Keras CNN architecture & wrap it under ART interface
classifier = KerasClassifier(build_model(),
clip_values=(0, 1),
use_logits=False)
# Create attack for adversarial trainer; here, we use 2 attacks, both crafting adv examples on the target model
pgd = ProjectedGradientDescent(classifier,
eps=8,
eps_step=2,
max_iter=10,
num_random_init=20)
# Create some adversarial samples for evaluation
x_test_pgd = pgd.generate(x_test)
# Create adversarial trainer and perform adversarial training
adv_trainer = AdversarialTrainer(classifier, attacks=pgd, ratio=1.0)
adv_trainer.fit_generator(art_datagen, nb_epochs=83)
# Evaluate the adversarially trained model on clean test set
labels_true = np.argmax(y_test, axis=1)
labels_test = np.argmax(classifier.predict(x_test), axis=1)
print("Accuracy test set: %.2f%%" %
(np.sum(labels_test == labels_true) / x_test.shape[0] * 100))
# Evaluate the adversarially trained model on original adversarial samples
labels_pgd = np.argmax(classifier.predict(x_test_pgd), axis=1)
print("Accuracy on original PGD adversarial samples: %.2f%%" %
(np.sum(labels_pgd == labels_true) / x_test.shape[0] * 100))
def plot_attacks_acc(classifier, x, y, path_fig, dataset, title):
'''
Description:
This function takes in a classifier model and a list of images with labels and creates
a plot showing how the accuracy of model on the dataset decreases as attack strength (perturbation size)
increases for 3 different attacks (FGSM, PGD, BIM).
:param classifier: model to be evaluated
:param x: list of images to be predicted on
:param y: labels of images
:param path_fig: path to save the plot figure
:param dataset: name of dataset (e.g. mnist, cifar, ddsm, brain_mri, lidc)
:param title: title to define plot figure
:return: Figure will be saved with title
'''
if dataset == 'ddsm':
eps_range = [0.00001, 0.00005, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.00125, 0.0015, 0.00175, 0.002, 0.0025, 0.003, 0.0035, 0.004, 0.0045, 0.005, 0.0055, 0.006, 0.007, 0.008]
step_size = 0.001
elif dataset == 'brain_mri':
eps_range = [0.0001, 0.0005, 0.001, 0.0013, 0.0016, 0.002, 0.00225, 0.0025, 0.00275, 0.003, 0.00325, 0.0035, 0.00375, 0.004, 0.0045, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.012]
step_size = 0.001
elif dataset == 'mnist':
eps_range = [0.0001, 0.01, 0.02, 0.05, 0.075, 0.1, 0.125, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.5]
step_size = 0.1
elif dataset == 'cifar':
eps_range = [0.0001, 0.001, 0.002, 0.003, 0.004, 0.005, 0.007, 0.009, 0.01, 0.015, 0.02, 0.03, 0.04, 0.05]
step_size = 0.01
elif dataset == 'lidc':
eps_range = [0.0001, 0.0003, 0.0006, 0.0008, 0.001, 0.00125, 0.0015, 0.00175, 0.002, 0.0023, 0.0026, 0.0028, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.011, 0.012, 0.013, 0.014, 0.015, 0.016, 0.017, 0.018, 0.019, 0.02]
step_size = 0.001
nb_correct_fgsm = []
nb_correct_pgd = []
nb_correct_bim = []
for eps in eps_range:
attacker_fgsm = FastGradientMethod(classifier, eps=eps)
attacker_pgd = ProjectedGradientDescent(classifier, eps=eps, eps_step=eps/4, max_iter=10,
num_random_init=5)
attacker_bim = BasicIterativeMethod(classifier, eps=eps, eps_step=eps/10, max_iter=10)
x_fgsm = attacker_fgsm.generate(x)
x_pgd = attacker_pgd.generate(x)
x_bim = attacker_bim.generate(x)
x_pred_fgsm = np.argmax(classifier.predict(x_fgsm), axis=1)
nb_correct_fgsm += [np.sum(x_pred_fgsm == np.argmax(y, axis=1))]
x_pred_pgd = np.argmax(classifier.predict(x_pgd), axis=1)
nb_correct_pgd += [np.sum(x_pred_pgd == np.argmax(y, axis=1))]
x_pred_bim = np.argmax(classifier.predict(x_bim), axis=1)
nb_correct_bim += [np.sum(x_pred_bim == np.argmax(y, axis=1))]
fig, ax = plt.subplots()
ax.plot(np.array(eps_range) / step_size, 100 * np.array(nb_correct_fgsm) / y.shape[0], 'b--', label='FGSM')
ax.plot(np.array(eps_range) / step_size, 100 * np.array(nb_correct_pgd) / y.shape[0], 'r--', label='PGD')
ax.plot(np.array(eps_range) / step_size, 100 * np.array(nb_correct_bim) / y.shape[0], 'g--', label='BIM')
legend = ax.legend(loc='upper right', shadow=True, fontsize='large')
legend.get_frame().set_facecolor('#FFFFFF')
if dataset == 'mnist':
plt.xlabel('Perturbation (x ' + '$10^{-1}$' + ')')
elif dataset == 'cifar':
plt.xlabel('Perturbation (x ' + '$10^{-2}$' + ')')
else:
plt.xlabel('Perturbation (x ' + '$10^{-3}$' + ')')
plt.ylabel('Accuracy (%)')
plt.savefig(path_fig + dataset + '/' + title + '.png')
plt.clf()
data = [np.array(eps_range), np.array(nb_correct_fgsm) / y.shape[0], np.array(nb_correct_pgd) / y.shape[0], np.array(nb_correct_bim) / y.shape[0]]
out = csv.writer(open(path_csv + dataset + '/' + title + '.csv', "w"), delimiter=',', quoting=csv.QUOTE_ALL)
out.writerows(zip(*data))
return 0
accuracy = np.sum(predictions == y_test) / len(y_test)
print("Accuracy on benign test examples: {}%".format(accuracy * 100))
# Step 5: Generate adversarial test examples
# attack = FastGradientMethod(estimator=classifier, eps=0.1)
# x_test_adv = attack.generate(x=x_test)
# adv_crafter = DeepFool(classifier, nb_grads=args.nb_grads)
# pgd
adv_crafter_untargeted = ProjectedGradientDescent(classifier,
eps=args.eps,
eps_step=args.eps_step,
max_iter=args.max_iter)
print("PGD:Craft attack on untargeted training examples")
x_test_adv = adv_crafter_untargeted.generate(x_test)
adv_crafter_targeted = ProjectedGradientDescent(classifier,
targeted=True,
eps=args.eps_step,
eps_step=args.eps_step,
max_iter=args.max_iter)
print("PGD:Craft attack on targeted training examples")
targets = random_targets(y_test, nb_classes=10)
x_test_adv_targeted = adv_crafter_targeted.generate(x_test, **{"y": targets})
#auto pgd
auto_adv_crafter_untargeted = AutoProjectedGradientDescent(
classifier, eps=args.eps, eps_step=args.eps_step, max_iter=args.max_iter)
print("AutoPGD:Craft attack on untargeted training examples")
x_test_auto_adv = auto_adv_crafter_untargeted.generate(x_test)
tiny_vgg.compile(optimizer=optimizer, loss=loss_object)
classifier = KerasClassifier(model=tiny_vgg,
clip_values=(0, 1),
use_logits=False)
attack = ProjectedGradientDescent(estimator=classifier,
eps=16 / 255,
eps_step=1 / 255,
norm="inf",
max_iter=200)
#attack = CarliniLInfMethod(classifier,
# confidence=0.8, targeted=False, learning_rate=0.001)
x_test_adv = attack.generate(x=x_test)
outputs = classifier.predict(x_test_adv)
preds = np.argmax(outputs, axis=1)
trues = np.argmax(y_test, axis=1)
accuracy = np.sum(preds == trues) / len(y_test)
print("Accuracy on adversarial test examples: {}%".format(accuracy * 100))
print("Ixs that worked: ")
print(np.where(preds != trues))
## Save a few of x_test_adv, please work:
for i in range(len(preds)):
x = (x_test_adv[i] * 255).astype(np.uint8)
im = Image.fromarray(x)
im.save(ADV_IMAGE_SAVE_LOCATION + '/x_adv_' + str(i) + '.jpeg')
clip_values=(0, 1),
)
print("Creating adversarial attack object...\n")
pgd = ProjectedGradientDescent(adv_classifier,
norm=np.inf,
eps=eps,
eps_step=0.001,
targeted=False,
batch_size=2048,
num_random_init=27)
print("Generating adversarial samples...\n")
logger.info("Craft attack on training examples")
x_train_adv = pgd.generate(train_data)
save_samples(x_train_adv, 'pgd_train', exp)
logger.info("=" * 50)
logger.info("Craft attack test examples")
x_test_adv = pgd.generate(test_data)
save_samples(x_test_adv, 'pgd_test', exp)
logger.info("=" * 50)
print("Evaluating adversarial samples on clean model...\n")
preds = np.argmax(adv_classifier.predict(x_test_adv), axis=1)
acc = np.sum(
preds == non_encoded_test_labels) / non_encoded_test_labels.shape[0]
logger.info("Classifier before adversarial training")
logger.info("Accuracy on adversarial samples: %.2f%%", (acc * 100))
logger.info("=" * 50)
0.0001, 0.0003, 0.0006, 0.0007, 0.0008, 0.0009, 0.001, 0.00125, 0.0015,
0.00175, 0.002, 0.0023, 0.0026, 0.0028, 0.003, 0.004, 0.005, 0.006,
0.007, 0.008, 0.009, 0.01, 0.011, 0.012, 0.013, 0.014, 0.015, 0.016,
0.017, 0.018, 0.019, 0.02
]
# evaluate sensitivity scores of each image
test_eps_scores = [1] * x_test.shape[0]
for eps in eps_range:
attacker = ProjectedGradientDescent(classifier,
eps=eps,
eps_step=eps / 4,
max_iter=max_iter,
num_random_init=num_random_init)
x_test_adv = attacker.generate(x_test)
for i in range(x_test.shape[0]):
img = np.expand_dims(x_test[i], axis=0)
adv_img = np.expand_dims(x_test_adv[i], axis=0)
pred = np.argmax(classifier.predict(img))
pred_adv = np.argmax(classifier.predict(adv_img))
if test_eps_scores[i] == 1:
if pred != pred_adv:
test_eps_scores[i] = eps
np.save(path + dataset + '/test_eps_scores.npy', test_eps_scores)
test_eps_scores = np.load(path + dataset + '/test_eps_scores.npy')
test_eps_freq = [0] * x_test.shape[0]
for eps_score in test_eps_scores:
for i in range(len(eps_range)):