def run_mnist_adv(num_epochs=NUM_EPOCHS, batch_size=BATCH_SIZE,
testing=False, learning_rate=LEARNING_RATE):
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# set random seed
tf.set_random_seed(42)
# can use gpu
config = tf.ConfigProto( device_count = {'GPU': 1 , 'CPU': 1} )
# Create TF session and set Keras backend session as TF
sess = tf.Session(config=config)
keras.backend.set_session(sess)
# Get MNIST test data
mnist = MNIST()
x_train, y_train = mnist.get_set("train")
x_test, y_test = mnist.get_set("test")
# Obtain image params
n_rows, n_cols, n_channels = x_train.shape[1:4]
n_classes = y_train.shape[1]
# define TF model graph
model = ConvNet((n_rows, n_cols, n_channels), n_classes)
model(model.input)
wrap = KerasModelWrapper(model)
fgsm = FastGradientMethod(wrap, sess=sess)
fgsm_params = {
'eps': 0.3,
'clip_min': 0.,
'clip_max': 1.
}
adv_acc_metric = get_adversarial_acc_metric(model, fgsm, fgsm_params)
model.compile(
optimizer=keras.optimizers.Adam(learning_rate),
loss='categorical_crossentropy',
metrics=['accuracy', adv_acc_metric]
)
# Train an MNIST model
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=num_epochs,
validation_data=(x_test, y_test),
verbose=1)
# Evaluate the accuracy on legitimate and adversarial test examples
_, acc, adv_acc = model.evaluate(x_test, y_test,
batch_size=batch_size,
verbose=0)
report.clean_train_clean_eval = acc
report.clean_train_adv_eval = adv_acc
print('Test accuracy on legitimate examples: %0.4f' % acc)
print('Test accuracy on adversarial examples: %0.4f\n' % adv_acc)
# Calculate training error
if testing:
_, train_acc, train_adv_acc = model.evaluate(x_train, y_train,
batch_size=batch_size,
verbose=0)
report.train_clean_train_clean_eval = train_acc
report.train_clean_train_adv_eval = train_adv_acc
print("Repeating the process, using adversarial training")
# Redefine Keras model
model_2 = ConvNet((n_rows, n_cols, n_channels), n_classes)
model_2(model_2.input)
wrap_2 = KerasModelWrapper(model_2)
fgsm_2 = FastGradientMethod(wrap_2, sess=sess)
# Use a loss function based on legitimate and adversarial examples
adv_loss_2 = get_adversarial_loss(model_2, fgsm_2, fgsm_params)
adv_acc_metric_2 = get_adversarial_acc_metric(model_2, fgsm_2, fgsm_params)
model_2.compile(
optimizer=keras.optimizers.Adam(learning_rate),
loss=adv_loss_2,
metrics=['accuracy', adv_acc_metric_2]
)
# Train an MNIST model
model_2.fit(x_train, y_train,
batch_size=batch_size,
epochs=num_epochs,
validation_data=(x_test, y_test),
verbose=1)
# Evaluate the accuracy on legitimate and adversarial test examples
_, acc, adv_acc = model_2.evaluate(x_test, y_test,
batch_size=batch_size,
verbose=0)
report.adv_train_clean_eval = acc
report.adv_train_adv_eval = adv_acc
print('Test accuracy on legitimate examples: %0.4f' % acc)
print('Test accuracy on adversarial examples: %0.4f\n' % adv_acc)
# Calculate training error
if testing:
_, train_acc, train_adv_acc = model_2.evaluate(x_train, y_train,
batch_size=batch_size,
verbose=0)
report.train_adv_train_clean_eval = train_acc
report.train_adv_train_adv_eval = train_adv_acc
return report
def train_zero_knowledge_gandef_model(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
smoke_test=True,
save=False,
testing=False,
backprop_through_attack=False,
num_threads=None):
"""
MNIST cleverhans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param train_batch_size: size of training batches
:param test_batch_size: size of testing batches
:param learning_rate: learning rate for training
:param save: if true, the final model will be saved
:param testing: if true, complete an AccuracyReport for unit tests
to verify that performance is adequate
:param backprop_through_attack: If True, backprop through adversarial
example construction process during
adversarial training.
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
if num_threads:
config_args = dict(intra_op_parallelism_threads=1)
else:
config_args = {}
sess = tf.Session(config=tf.ConfigProto(**config_args))
# Get MNIST test data
X_train, Y_train, X_test, Y_test = data_fashion_mnist()
if smoke_test:
X_train, Y_train, X_test, Y_test = X_train[:
256], Y_train[:
256], X_test[:
256], Y_test[:
256]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
y_soft = tf.placeholder(tf.float32, shape=(None, 10))
# Prepare optimizer
learning_rate = 1e-4
clf_opt = tf.train.AdamOptimizer(learning_rate)
dic_opt = tf.train.AdamOptimizer(learning_rate * 10)
# Train an MNIST model
train_params = {
'nb_epochs': 80,
'batch_size': 128,
'trade_off': 2,
'inner_epochs': 1
}
rng = np.random.RandomState([2017, 8, 30])
# Adversarial training
print("Start adversarial training")
zero_knowledge_gandef_model = make_zero_knowledge_gandef_model(
name="model_zero_knowledge_gandef")
aug_x = gaussian_augment(x, std=1)
preds_clean = zero_knowledge_gandef_model(x)
preds_aug = zero_knowledge_gandef_model(aug_x)
def cross_entropy(truth, preds, mean=True):
# Get the logits operator
op = preds.op
if op.type == "Softmax":
logits, = op.inputs
else:
logits = preds
# Calculate cross entropy loss
out = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=truth)
# Take average loss and return
if mean:
out = tf.reduce_mean(out)
return out
def sigmoid_entropy(truth, preds, mean=True):
# Get the logits operator
op = preds.op
if op.type == "Softmax":
logits, = op.inputs
else:
logits = preds
# Calculate cross entropy loss
out = tf.nn.sigmoid_cross_entropy_with_logits(logits=logits,
labels=truth)
# Take average loss and return
if mean:
out = tf.reduce_mean(out)
return out
# Perform and evaluate adversarial training
gan_train_v2(sess,
x,
y_soft,
preds_clean,
X_train,
Y_train,
loss_func=[cross_entropy, sigmoid_entropy],
optimizer=[clf_opt, dic_opt],
predictions_adv=preds_aug,
evaluate=None,
args=train_params,
rng=rng,
var_list=zero_knowledge_gandef_model.get_gan_params())
# Evaluate the accuracy of the MNIST model on Clean examples
preds_clean = zero_knowledge_gandef_model(x)
eval_params = {
'batch_size': 128,
'use_dic': False,
'is_clean': True,
'reject_threshold': 0.5
}
clean_acc = confident_model_eval(sess,
x,
y_soft,
preds_clean,
X_test,
Y_test,
args=eval_params)
print('Test accuracy on Clean test examples: %0.4f\n' % clean_acc)
report.adv_train_clean_eval = clean_acc
# Evaluate the accuracy of the MNIST model on FGSM examples
fgsm_params = {'eps': 0.6, 'clip_min': -1., 'clip_max': 1.}
fgsm_att = FastGradientMethod(zero_knowledge_gandef_model, sess=sess)
fgsm_adv = fgsm_att.generate(x, **fgsm_params)
preds_fgsm_adv = zero_knowledge_gandef_model(fgsm_adv)
eval_params = {
'batch_size': 128,
'use_dic': False,
'is_clean': False,
'reject_threshold': 0.5
}
fgsm_acc = confident_model_eval(sess,
x,
y_soft,
preds_fgsm_adv,
X_test,
Y_test,
args=eval_params)
print('Test accuracy on FGSM test examples: %0.4f\n' % fgsm_acc)
report.adv_train_adv_eval = fgsm_acc
# Evaluate the accuracy of the MNIST model on BIM examples
bim_params = {'eps': 0.6, 'eps_iter': 0.1, 'clip_min': -1., 'clip_max': 1.}
bim_att = BasicIterativeMethod(zero_knowledge_gandef_model, sess=sess)
bim_adv = bim_att.generate(x, **bim_params)
preds_bim_adv = zero_knowledge_gandef_model(bim_adv)
eval_params = {
'batch_size': 128,
'use_dic': False,
'is_clean': False,
'reject_threshold': 0.5
}
bim_acc = confident_model_eval(sess,
x,
y_soft,
preds_bim_adv,
X_test,
Y_test,
args=eval_params)
print('Test accuracy on BIM test examples: %0.4f\n' % bim_acc)
report.adv_train_adv_eval = bim_acc
# Evaluate the accuracy of the MNIST model on PGD examples
pgd_params = {
'eps': 0.6,
'eps_iter': 0.02,
'nb_iter': 40,
'clip_min': -1.,
'clip_max': 1.,
'rand_init': True
}
pgd_att = MadryEtAl(zero_knowledge_gandef_model, sess=sess)
pgd_adv = pgd_att.generate(x, **bim_params)
preds_pgd_adv = zero_knowledge_gandef_model(pgd_adv)
eval_params = {
'batch_size': 128,
'use_dic': False,
'is_clean': False,
'reject_threshold': 0.5
}
pgd_acc = confident_model_eval(sess,
x,
y_soft,
preds_pgd_adv,
X_test,
Y_test,
args=eval_params)
print('Test accuracy on PGD test examples: %0.4f\n' % pgd_acc)
report.adv_train_adv_eval = pgd_acc
# Save model
if save:
model_path = "models/zero_knowledge_gandef"
vars_to_save = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='model_zero_knowledge_gandef*')
assert len(vars_to_save) > 0
saver = tf.train.Saver(var_list=vars_to_save)
saver.save(sess, model_path)
print('Model saved\n')
else:
print('Model not saved\n')
def test_run_single_gpu_fgsm(self):
"""
Test the basic single GPU performance by comparing to the FGSM
tutorial.
"""
from cleverhans_tutorials import mnist_tutorial_tf
# Run the MNIST tutorial on a dataset of reduced size
flags = {
'train_start': 0,
'train_end': 5000,
'test_start': 0,
'test_end': 333,
'nb_epochs': 5,
'testing': True
}
report = mnist_tutorial_tf.mnist_tutorial(**flags)
# Run the multi-gpu trainer for clean training
flags.update({
'batch_size': 128,
'adam_lrn': 0.001,
'dataset': 'mnist',
'only_adv_train': False,
'eval_iters': 1,
'ngpu': 1,
'fast_tests': False,
'attack_type_train': '',
'save_dir': None,
'save_steps': 10000,
'attack_nb_iter_train': None,
'save': False,
'model_type': 'basic',
'attack_type_test': 'FGSM'
})
flags.update({'adv_train': False})
HParams = namedtuple('HParams', flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.compat.v1.set_random_seed(42)
with tf.compat.v1.variable_scope(None, 'runner'):
report_dict = run_trainer(hparams)
report_2 = AccuracyReport()
report_2.train_clean_train_clean_eval = report_dict['train']
report_2.clean_train_clean_eval = report_dict['test']
report_2.clean_train_adv_eval = report_dict['FGSM']
# Run the multi-gpu trainer for adversarial training
flags.update({'adv_train': True, 'attack_type_train': 'FGSM'})
HParams = namedtuple('HParams', flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.compat.v1.set_random_seed(42)
with tf.compat.v1.variable_scope(None, 'runner'):
report_dict = run_trainer(hparams)
report_2.train_adv_train_clean_eval = report_dict['train']
report_2.adv_train_clean_eval = report_dict['test']
report_2.adv_train_adv_eval = report_dict['FGSM']
self.assertClose(report.train_clean_train_clean_eval,
report_2.train_clean_train_clean_eval,
atol=5e-2)
self.assertClose(report.clean_train_clean_eval,
report_2.clean_train_clean_eval,
atol=2e-2)
self.assertClose(report.clean_train_adv_eval,
report_2.clean_train_adv_eval,
atol=5e-2)
self.assertClose(report.train_adv_train_clean_eval,
report_2.train_adv_train_clean_eval,
atol=1e-1)
self.assertClose(report.adv_train_clean_eval,
report_2.adv_train_clean_eval,
atol=2e-2)
self.assertClose(report.adv_train_adv_eval,
report_2.adv_train_adv_eval,
atol=1e-1)
def mnist_tutorial(train_start=0, train_end=60000, test_start=0,
test_end=10000, nb_epochs=NB_EPOCHS, batch_size=BATCH_SIZE,
learning_rate=LEARNING_RATE, testing=False,
label_smoothing=0.1):
"""
MNIST CleverHans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param testing: if true, training error is calculated
:param label_smoothing: float, amount of label smoothing for cross entropy
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Force TensorFlow to use single thread to improve reproducibility
config = tf.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1)
if keras.backend.image_data_format() != 'channels_last':
raise NotImplementedError("this tutorial requires keras to be configured to channels_last format")
# Create TF session and set as Keras backend session
sess = tf.Session(config=config)
keras.backend.set_session(sess)
# Get MNIST test data
mnist = MNIST(train_start=train_start, train_end=train_end,
test_start=test_start, test_end=test_end)
x_train, y_train = mnist.get_set('train')
x_test, y_test = mnist.get_set('test')
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Label smoothing
y_train -= label_smoothing * (y_train - 1. / nb_classes)
# Define Keras model
model = cnn_model(img_rows=img_rows, img_cols=img_cols,
channels=nchannels, nb_filters=64,
nb_classes=nb_classes)
print("Defined Keras model.")
# To be able to call the model in the custom loss, we need to call it once
# before, see https://github.com/tensorflow/tensorflow/issues/23769
model(model.input)
# Initialize the Fast Gradient Sign Method (FGSM) attack object
wrap = KerasModelWrapper(model)
fgsm = FastGradientMethod(wrap, sess=sess)
fgsm_params = {'eps': 0.3,
'clip_min': 0.,
'clip_max': 1.}
adv_acc_metric = get_adversarial_acc_metric(model, fgsm, fgsm_params)
model.compile(
optimizer=keras.optimizers.Adam(learning_rate),
loss='categorical_crossentropy',
metrics=['accuracy', adv_acc_metric]
)
# Train an MNIST model
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=nb_epochs,
validation_data=(x_test, y_test),
verbose=2)
# Evaluate the accuracy on legitimate and adversarial test examples
_, acc, adv_acc = model.evaluate(x_test, y_test,
batch_size=batch_size,
verbose=0)
report.clean_train_clean_eval = acc
report.clean_train_adv_eval = adv_acc
print('Test accuracy on legitimate examples: %0.4f' % acc)
print('Test accuracy on adversarial examples: %0.4f\n' % adv_acc)
# Calculate training error
if testing:
_, train_acc, train_adv_acc = model.evaluate(x_train, y_train,
batch_size=batch_size,
verbose=0)
report.train_clean_train_clean_eval = train_acc
report.train_clean_train_adv_eval = train_adv_acc
print("Repeating the process, using adversarial training")
# Redefine Keras model
model_2 = cnn_model(img_rows=img_rows, img_cols=img_cols,
channels=nchannels, nb_filters=64,
nb_classes=nb_classes)
model_2(model_2.input)
wrap_2 = KerasModelWrapper(model_2)
fgsm_2 = FastGradientMethod(wrap_2, sess=sess)
# Use a loss function based on legitimate and adversarial examples
adv_loss_2 = get_adversarial_loss(model_2, fgsm_2, fgsm_params)
adv_acc_metric_2 = get_adversarial_acc_metric(model_2, fgsm_2, fgsm_params)
model_2.compile(
optimizer=keras.optimizers.Adam(learning_rate),
loss=adv_loss_2,
metrics=['accuracy', adv_acc_metric_2]
)
# Train an MNIST model
model_2.fit(x_train, y_train,
batch_size=batch_size,
epochs=nb_epochs,
validation_data=(x_test, y_test),
verbose=2)
# Evaluate the accuracy on legitimate and adversarial test examples
_, acc, adv_acc = model_2.evaluate(x_test, y_test,
batch_size=batch_size,
verbose=0)
report.adv_train_clean_eval = acc
report.adv_train_adv_eval = adv_acc
print('Test accuracy on legitimate examples: %0.4f' % acc)
print('Test accuracy on adversarial examples: %0.4f\n' % adv_acc)
# Calculate training error
if testing:
_, train_acc, train_adv_acc = model_2.evaluate(x_train, y_train,
batch_size=batch_size,
verbose=0)
report.train_adv_train_clean_eval = train_acc
report.train_adv_train_adv_eval = train_adv_acc
return report
def helper_run_multi_gpu_madryetal(self, extra_flags=None):
"""
Compare the single GPU performance to multiGPU performance.
"""
# Run the trainers on a dataset of reduced size
flags = {
'train_start': 0,
'train_end': 5000,
'test_start': 0,
'test_end': 333,
'nb_epochs': 5,
'testing': True
}
# Run the multi-gpu trainer for adversarial training
flags.update({
'batch_size': 128,
'adam_lrn': 0.001,
'dataset': 'mnist',
'only_adv_train': False,
'eval_iters': 1,
'fast_tests': True,
'save_dir': None,
'save_steps': 10000,
'attack_nb_iter_train': 10,
'sync_step': None,
'adv_train': True,
'save': False,
'model_type': 'basic',
'attack_type_test': 'MadryEtAl_y'
})
if extra_flags is not None:
flags.update(extra_flags)
# Run the multi-gpu trainer for adversarial training using 2 gpus
# trainer_multigpu by default sets `allow_soft_placement=True`
flags.update({
'ngpu': 2,
'attack_type_train': 'MadryEtAl_y_multigpu',
'sync_step': 1
})
HParams = namedtuple('HParams', flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.compat.v1.set_random_seed(42)
with tf.compat.v1.variable_scope(None, 'runner'):
report_dict = run_trainer(hparams)
report_m = AccuracyReport()
report_m.train_adv_train_clean_eval = report_dict['train']
report_m.adv_train_clean_eval = report_dict['test']
report_m.adv_train_adv_eval = report_dict['MadryEtAl_y']
flags.update({'ngpu': 1, 'attack_type_train': 'MadryEtAl_y'})
hparams = HParams(**flags)
np.random.seed(42)
tf.compat.v1.set_random_seed(42)
with tf.compat.v1.variable_scope(None, 'runner'):
report_dict = run_trainer(hparams)
report_s = AccuracyReport()
report_s.train_adv_train_clean_eval = report_dict['train']
report_s.adv_train_clean_eval = report_dict['test']
report_s.adv_train_adv_eval = report_dict['MadryEtAl_y']
self.assertClose(report_s.train_adv_train_clean_eval,
report_m.train_adv_train_clean_eval,
atol=5e-2)
self.assertClose(report_s.adv_train_clean_eval,
report_m.adv_train_clean_eval,
atol=2e-2)
self.assertClose(report_s.adv_train_adv_eval,
report_m.adv_train_adv_eval,
atol=5e-2)
def test_run_single_gpu_fgsm(self):
"""
Test the basic single GPU performance by comparing to the FGSM
tutorial.
"""
from cleverhans_tutorials import mnist_tutorial_tf
# Run the MNIST tutorial on a dataset of reduced size
flags = {
"train_start": 0,
"train_end": 5000,
"test_start": 0,
"test_end": 333,
"nb_epochs": 5,
"testing": True,
}
report = mnist_tutorial_tf.mnist_tutorial(**flags)
# Run the multi-gpu trainer for clean training
flags.update(
{
"batch_size": 128,
"adam_lrn": 0.001,
"dataset": "mnist",
"only_adv_train": False,
"eval_iters": 1,
"ngpu": 1,
"fast_tests": False,
"attack_type_train": "",
"save_dir": None,
"save_steps": 10000,
"attack_nb_iter_train": None,
"save": False,
"model_type": "basic",
"attack_type_test": "FGSM",
}
)
flags.update({"adv_train": False})
HParams = namedtuple("HParams", flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.set_random_seed(42)
with tf.variable_scope(None, "runner"):
report_dict = run_trainer(hparams)
report_2 = AccuracyReport()
report_2.train_clean_train_clean_eval = report_dict["train"]
report_2.clean_train_clean_eval = report_dict["test"]
report_2.clean_train_adv_eval = report_dict["FGSM"]
# Run the multi-gpu trainer for adversarial training
flags.update({"adv_train": True, "attack_type_train": "FGSM"})
HParams = namedtuple("HParams", flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.set_random_seed(42)
with tf.variable_scope(None, "runner"):
report_dict = run_trainer(hparams)
report_2.train_adv_train_clean_eval = report_dict["train"]
report_2.adv_train_clean_eval = report_dict["test"]
report_2.adv_train_adv_eval = report_dict["FGSM"]
self.assertClose(
report.train_clean_train_clean_eval,
report_2.train_clean_train_clean_eval,
atol=5e-2,
)
self.assertClose(
report.clean_train_clean_eval, report_2.clean_train_clean_eval, atol=2e-2
)
self.assertClose(
report.clean_train_adv_eval, report_2.clean_train_adv_eval, atol=5e-2
)
self.assertClose(
report.train_adv_train_clean_eval,
report_2.train_adv_train_clean_eval,
atol=1e-1,
)
self.assertClose(
report.adv_train_clean_eval, report_2.adv_train_clean_eval, atol=2e-2
)
self.assertClose(
report.adv_train_adv_eval, report_2.adv_train_adv_eval, atol=1e-1
)
def helper_run_multi_gpu_madryetal(self, extra_flags=None):
"""
Compare the single GPU performance to multiGPU performance.
"""
# Run the trainers on a dataset of reduced size
flags = {
"train_start": 0,
"train_end": 5000,
"test_start": 0,
"test_end": 333,
"nb_epochs": 5,
"testing": True,
}
# Run the multi-gpu trainer for adversarial training
flags.update(
{
"batch_size": 128,
"adam_lrn": 0.001,
"dataset": "mnist",
"only_adv_train": False,
"eval_iters": 1,
"fast_tests": True,
"save_dir": None,
"save_steps": 10000,
"attack_nb_iter_train": 10,
"sync_step": None,
"adv_train": True,
"save": False,
"model_type": "basic",
"attack_type_test": "MadryEtAl_y",
}
)
if extra_flags is not None:
flags.update(extra_flags)
# Run the multi-gpu trainer for adversarial training using 2 gpus
# trainer_multigpu by default sets `allow_soft_placement=True`
flags.update(
{"ngpu": 2, "attack_type_train": "MadryEtAl_y_multigpu", "sync_step": 1}
)
HParams = namedtuple("HParams", flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.set_random_seed(42)
with tf.variable_scope(None, "runner"):
report_dict = run_trainer(hparams)
report_m = AccuracyReport()
report_m.train_adv_train_clean_eval = report_dict["train"]
report_m.adv_train_clean_eval = report_dict["test"]
report_m.adv_train_adv_eval = report_dict["MadryEtAl_y"]
flags.update({"ngpu": 1, "attack_type_train": "MadryEtAl_y"})
hparams = HParams(**flags)
np.random.seed(42)
tf.set_random_seed(42)
with tf.variable_scope(None, "runner"):
report_dict = run_trainer(hparams)
report_s = AccuracyReport()
report_s.train_adv_train_clean_eval = report_dict["train"]
report_s.adv_train_clean_eval = report_dict["test"]
report_s.adv_train_adv_eval = report_dict["MadryEtAl_y"]
self.assertClose(
report_s.train_adv_train_clean_eval,
report_m.train_adv_train_clean_eval,
atol=5e-2,
)
self.assertClose(
report_s.adv_train_clean_eval, report_m.adv_train_clean_eval, atol=2e-2
)
self.assertClose(
report_s.adv_train_adv_eval, report_m.adv_train_adv_eval, atol=5e-2
)
def test_run_single_gpu_fgsm(self):
"""
Test the basic single GPU performance by comparing to the FGSM
tutorial.
"""
from cleverhans_tutorials import mnist_tutorial_tf
# Run the MNIST tutorial on a dataset of reduced size
flags = {'train_start': 0,
'train_end': 5000,
'test_start': 0,
'test_end': 333,
'nb_epochs': 5,
'testing': True}
report = mnist_tutorial_tf.mnist_tutorial(**flags)
# Run the multi-gpu trainer for clean training
flags.update({'batch_size': 128, 'adam_lrn': 0.001,
'dataset': 'mnist', 'only_adv_train': False,
'eval_iters': 1, 'ngpu': 1, 'fast_tests': False,
'attack_type_train': '',
'save_dir': None, 'save_steps': 10000,
'attack_nb_iter_train': None, 'save': False,
'model_type': 'basic', 'attack_type_test': 'FGSM'})
flags.update({'adv_train': False})
HParams = namedtuple('HParams', flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.set_random_seed(42)
with tf.variable_scope(None, 'runner'):
report_dict = run_trainer(hparams)
report_2 = AccuracyReport()
report_2.train_clean_train_clean_eval = report_dict['train']
report_2.clean_train_clean_eval = report_dict['test']
report_2.clean_train_adv_eval = report_dict['FGSM']
# Run the multi-gpu trainer for adversarial training
flags.update({'adv_train': True,
'attack_type_train': 'FGSM',
})
HParams = namedtuple('HParams', flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.set_random_seed(42)
with tf.variable_scope(None, 'runner'):
report_dict = run_trainer(hparams)
report_2.train_adv_train_clean_eval = report_dict['train']
report_2.adv_train_clean_eval = report_dict['test']
report_2.adv_train_adv_eval = report_dict['FGSM']
self.assertClose(report.train_clean_train_clean_eval,
report_2.train_clean_train_clean_eval,
atol=5e-2)
self.assertClose(report.clean_train_clean_eval,
report_2.clean_train_clean_eval,
atol=2e-2)
self.assertClose(report.clean_train_adv_eval,
report_2.clean_train_adv_eval,
atol=5e-2)
self.assertClose(report.train_adv_train_clean_eval,
report_2.train_adv_train_clean_eval,
atol=1e-1)
self.assertClose(report.adv_train_clean_eval,
report_2.adv_train_clean_eval,
atol=2e-2)
self.assertClose(report.adv_train_adv_eval,
report_2.adv_train_adv_eval,
atol=1e-1)
def helper_run_multi_gpu_madryetal(self, extra_flags=None):
"""
Compare the single GPU performance to multiGPU performance.
"""
# Run the trainers on a dataset of reduced size
flags = {'train_start': 0,
'train_end': 5000,
'test_start': 0,
'test_end': 333,
'nb_epochs': 5,
'testing': True}
# Run the multi-gpu trainer for adversarial training
flags.update({'batch_size': 128, 'adam_lrn': 0.001,
'dataset': 'mnist', 'only_adv_train': False,
'eval_iters': 1, 'fast_tests': True,
'save_dir': None, 'save_steps': 10000,
'attack_nb_iter_train': 10, 'sync_step': None,
'adv_train': True,
'save': False,
'model_type': 'basic',
'attack_type_test': 'MadryEtAl_y',
})
if extra_flags is not None:
flags.update(extra_flags)
# Run the multi-gpu trainer for adversarial training using 2 gpus
# trainer_multigpu by default sets `allow_soft_placement=True`
flags.update({'ngpu': 2,
'attack_type_train': 'MadryEtAl_y_multigpu',
'sync_step': 1})
HParams = namedtuple('HParams', flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.set_random_seed(42)
with tf.variable_scope(None, 'runner'):
report_dict = run_trainer(hparams)
report_m = AccuracyReport()
report_m.train_adv_train_clean_eval = report_dict['train']
report_m.adv_train_clean_eval = report_dict['test']
report_m.adv_train_adv_eval = report_dict['MadryEtAl_y']
flags.update({'ngpu': 1,
'attack_type_train': 'MadryEtAl_y',
})
hparams = HParams(**flags)
np.random.seed(42)
tf.set_random_seed(42)
with tf.variable_scope(None, 'runner'):
report_dict = run_trainer(hparams)
report_s = AccuracyReport()
report_s.train_adv_train_clean_eval = report_dict['train']
report_s.adv_train_clean_eval = report_dict['test']
report_s.adv_train_adv_eval = report_dict['MadryEtAl_y']
self.assertClose(report_s.train_adv_train_clean_eval,
report_m.train_adv_train_clean_eval,
atol=5e-2)
self.assertClose(report_s.adv_train_clean_eval,
report_m.adv_train_clean_eval,
atol=2e-2)
self.assertClose(report_s.adv_train_adv_eval,
report_m.adv_train_adv_eval,
atol=5e-2)
