def test_mnist_tutorial_tf(self):
import tensorflow as tf
from cleverhans_tutorials import mnist_tutorial_tf
# Run the MNIST tutorial on a dataset of reduced size
test_dataset_indices = {
"train_start": 0,
"train_end": 5000,
"test_start": 0,
"test_end": 333,
"nb_epochs": 2,
"testing": True,
}
g = tf.Graph()
with g.as_default():
np.random.seed(42)
report = mnist_tutorial_tf.mnist_tutorial(num_threads=1,
**test_dataset_indices)
# Check accuracy values contained in the AccuracyReport object
self.assertGreater(report.train_clean_train_clean_eval, 0.97)
self.assertLess(report.train_clean_train_adv_eval, 0.05)
self.assertGreater(report.train_adv_train_clean_eval, 0.93)
self.assertGreater(report.train_adv_train_adv_eval, 0.4)
# Check that the tutorial is deterministic (seeded properly)
atol_fac = 2e-2 if HAS_GPU else 1e-6
g = tf.Graph()
with g.as_default():
np.random.seed(42)
report_2 = mnist_tutorial_tf.mnist_tutorial(num_threads=1,
**test_dataset_indices)
self.assertClose(
report.train_clean_train_clean_eval,
report_2.train_clean_train_clean_eval,
atol=atol_fac * 1,
)
self.assertClose(
report.train_clean_train_adv_eval,
report_2.train_clean_train_adv_eval,
atol=atol_fac * 1,
)
self.assertClose(
report.train_adv_train_clean_eval,
report_2.train_adv_train_clean_eval,
atol=atol_fac * 1,
)
self.assertClose(
report.train_adv_train_adv_eval,
report_2.train_adv_train_adv_eval,
atol=atol_fac * 1,
)
def test_mnist_tutorial_tf(self):
np.random.seed(42)
import tensorflow as tf
tf.set_random_seed(42)
from cleverhans_tutorials import mnist_tutorial_tf
# Run the MNIST tutorial on a dataset of reduced size
test_dataset_indices = {
'train_start': 0,
'train_end': 5000,
'test_start': 0,
'test_end': 333,
'nb_epochs': 2,
'testing': True
}
report = mnist_tutorial_tf.mnist_tutorial(**test_dataset_indices)
# Check accuracy values contained in the AccuracyReport object
self.assertGreater(report.train_clean_train_clean_eval, 0.97)
self.assertLess(report.train_clean_train_adv_eval, 0.029)
self.assertGreater(report.train_adv_train_clean_eval, 0.95)
self.assertGreater(report.train_adv_train_adv_eval, 0.4)
# Check that the tutorial is deterministic (seeded properly)
if LooseVersion(tf.__version__) >= LooseVersion('1.1.0'):
atol_fac = 1
else:
atol_fac = 2
report_2 = mnist_tutorial_tf.mnist_tutorial(**test_dataset_indices)
self.assertClose(report.train_clean_train_clean_eval,
report_2.train_clean_train_clean_eval,
atol=atol_fac * 5e-3)
self.assertClose(report.train_clean_train_adv_eval,
report_2.train_clean_train_adv_eval,
atol=atol_fac * 5e-3)
self.assertClose(report.train_adv_train_clean_eval,
report_2.train_adv_train_clean_eval,
atol=atol_fac * 2e-2)
self.assertClose(report.train_adv_train_adv_eval,
report_2.train_adv_train_adv_eval,
atol=atol_fac * 2e-1)
def test_mnist_tutorial_tf(self):
import tensorflow as tf
from cleverhans_tutorials import mnist_tutorial_tf
# Run the MNIST tutorial on a dataset of reduced size
test_dataset_indices = {'train_start': 0,
'train_end': 5000,
'test_start': 0,
'test_end': 333,
'nb_epochs': 2,
'testing': True}
g = tf.Graph()
with g.as_default():
np.random.seed(42)
report = mnist_tutorial_tf.mnist_tutorial(num_threads=1,
**test_dataset_indices)
# Check accuracy values contained in the AccuracyReport object
self.assertGreater(report.train_clean_train_clean_eval, 0.97)
self.assertLess(report.train_clean_train_adv_eval, 0.05)
self.assertGreater(report.train_adv_train_clean_eval, 0.93)
self.assertGreater(report.train_adv_train_adv_eval, 0.4)
# Check that the tutorial is deterministic (seeded properly)
atol_fac = 2e-2 if HAS_GPU else 1e-6
g = tf.Graph()
with g.as_default():
np.random.seed(42)
report_2 = mnist_tutorial_tf.mnist_tutorial(num_threads=1,
**test_dataset_indices)
self.assertClose(report.train_clean_train_clean_eval,
report_2.train_clean_train_clean_eval,
atol=atol_fac * 1)
self.assertClose(report.train_clean_train_adv_eval,
report_2.train_clean_train_adv_eval,
atol=atol_fac * 1)
self.assertClose(report.train_adv_train_clean_eval,
report_2.train_adv_train_clean_eval,
atol=atol_fac * 1)
self.assertClose(report.train_adv_train_adv_eval,
report_2.train_adv_train_adv_eval,
atol=atol_fac * 1)
def test_mnist_tutorial_tf(self):
np.random.seed(42)
import tensorflow as tf
tf.set_random_seed(42)
from cleverhans_tutorials import mnist_tutorial_tf
# Run the MNIST tutorial on a dataset of reduced size
test_dataset_indices = {
'train_start': 0,
'train_end': 5000,
'test_start': 0,
'test_end': 333,
'nb_epochs': 2,
'testing': True
}
report = mnist_tutorial_tf.mnist_tutorial(**test_dataset_indices)
# Check accuracy values contained in the AccuracyReport object
self.assertTrue(report.train_clean_train_clean_eval > 0.90)
self.assertTrue(report.train_clean_train_adv_eval < 0.05)
self.assertTrue(report.train_adv_train_clean_eval > 0.90)
self.assertTrue(report.train_adv_train_adv_eval > 0.3)
def test_run_single_gpu_fgsm(self):
"""
Test the basic single GPU performance by comparing to the FGSM
tutorial.
"""
from cleverhans_tutorials import mnist_tutorial_tf
# Run the MNIST tutorial on a dataset of reduced size
flags = {
'train_start': 0,
'train_end': 5000,
'test_start': 0,
'test_end': 333,
'nb_epochs': 5,
'testing': True
}
report = mnist_tutorial_tf.mnist_tutorial(**flags)
# Run the multi-gpu trainer for clean training
flags.update({
'batch_size': 128,
'adam_lrn': 0.001,
'dataset': 'mnist',
'only_adv_train': False,
'eval_iters': 1,
'ngpu': 1,
'fast_tests': False,
'attack_type_train': '',
'save_dir': None,
'save_steps': 10000,
'attack_nb_iter_train': None,
'save': False,
'model_type': 'basic',
'attack_type_test': 'FGSM'
})
flags.update({'adv_train': False})
HParams = namedtuple('HParams', flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.compat.v1.set_random_seed(42)
with tf.compat.v1.variable_scope(None, 'runner'):
report_dict = run_trainer(hparams)
report_2 = AccuracyReport()
report_2.train_clean_train_clean_eval = report_dict['train']
report_2.clean_train_clean_eval = report_dict['test']
report_2.clean_train_adv_eval = report_dict['FGSM']
# Run the multi-gpu trainer for adversarial training
flags.update({'adv_train': True, 'attack_type_train': 'FGSM'})
HParams = namedtuple('HParams', flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.compat.v1.set_random_seed(42)
with tf.compat.v1.variable_scope(None, 'runner'):
report_dict = run_trainer(hparams)
report_2.train_adv_train_clean_eval = report_dict['train']
report_2.adv_train_clean_eval = report_dict['test']
report_2.adv_train_adv_eval = report_dict['FGSM']
self.assertClose(report.train_clean_train_clean_eval,
report_2.train_clean_train_clean_eval,
atol=5e-2)
self.assertClose(report.clean_train_clean_eval,
report_2.clean_train_clean_eval,
atol=2e-2)
self.assertClose(report.clean_train_adv_eval,
report_2.clean_train_adv_eval,
atol=5e-2)
self.assertClose(report.train_adv_train_clean_eval,
report_2.train_adv_train_clean_eval,
atol=1e-1)
self.assertClose(report.adv_train_clean_eval,
report_2.adv_train_clean_eval,
atol=2e-2)
self.assertClose(report.adv_train_adv_eval,
report_2.adv_train_adv_eval,
atol=1e-1)
def test_run_single_gpu_fgsm(self):
"""
Test the basic single GPU performance by comparing to the FGSM
tutorial.
"""
from cleverhans_tutorials import mnist_tutorial_tf
# Run the MNIST tutorial on a dataset of reduced size
flags = {
"train_start": 0,
"train_end": 5000,
"test_start": 0,
"test_end": 333,
"nb_epochs": 5,
"testing": True,
}
report = mnist_tutorial_tf.mnist_tutorial(**flags)
# Run the multi-gpu trainer for clean training
flags.update(
{
"batch_size": 128,
"adam_lrn": 0.001,
"dataset": "mnist",
"only_adv_train": False,
"eval_iters": 1,
"ngpu": 1,
"fast_tests": False,
"attack_type_train": "",
"save_dir": None,
"save_steps": 10000,
"attack_nb_iter_train": None,
"save": False,
"model_type": "basic",
"attack_type_test": "FGSM",
}
)
flags.update({"adv_train": False})
HParams = namedtuple("HParams", flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.set_random_seed(42)
with tf.variable_scope(None, "runner"):
report_dict = run_trainer(hparams)
report_2 = AccuracyReport()
report_2.train_clean_train_clean_eval = report_dict["train"]
report_2.clean_train_clean_eval = report_dict["test"]
report_2.clean_train_adv_eval = report_dict["FGSM"]
# Run the multi-gpu trainer for adversarial training
flags.update({"adv_train": True, "attack_type_train": "FGSM"})
HParams = namedtuple("HParams", flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.set_random_seed(42)
with tf.variable_scope(None, "runner"):
report_dict = run_trainer(hparams)
report_2.train_adv_train_clean_eval = report_dict["train"]
report_2.adv_train_clean_eval = report_dict["test"]
report_2.adv_train_adv_eval = report_dict["FGSM"]
self.assertClose(
report.train_clean_train_clean_eval,
report_2.train_clean_train_clean_eval,
atol=5e-2,
)
self.assertClose(
report.clean_train_clean_eval, report_2.clean_train_clean_eval, atol=2e-2
)
self.assertClose(
report.clean_train_adv_eval, report_2.clean_train_adv_eval, atol=5e-2
)
self.assertClose(
report.train_adv_train_clean_eval,
report_2.train_adv_train_clean_eval,
atol=1e-1,
)
self.assertClose(
report.adv_train_clean_eval, report_2.adv_train_clean_eval, atol=2e-2
)
self.assertClose(
report.adv_train_adv_eval, report_2.adv_train_adv_eval, atol=1e-1
)
def test_run_single_gpu_fgsm(self):
"""
Test the basic single GPU performance by comparing to the FGSM
tutorial.
"""
from cleverhans_tutorials import mnist_tutorial_tf
# Run the MNIST tutorial on a dataset of reduced size
flags = {'train_start': 0,
'train_end': 5000,
'test_start': 0,
'test_end': 333,
'nb_epochs': 5,
'testing': True}
report = mnist_tutorial_tf.mnist_tutorial(**flags)
# Run the multi-gpu trainer for clean training
flags.update({'batch_size': 128, 'adam_lrn': 0.001,
'dataset': 'mnist', 'only_adv_train': False,
'eval_iters': 1, 'ngpu': 1, 'fast_tests': False,
'attack_type_train': '',
'save_dir': None, 'save_steps': 10000,
'attack_nb_iter_train': None, 'save': False,
'model_type': 'basic', 'attack_type_test': 'FGSM'})
flags.update({'adv_train': False})
HParams = namedtuple('HParams', flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.set_random_seed(42)
with tf.variable_scope(None, 'runner'):
report_dict = run_trainer(hparams)
report_2 = AccuracyReport()
report_2.train_clean_train_clean_eval = report_dict['train']
report_2.clean_train_clean_eval = report_dict['test']
report_2.clean_train_adv_eval = report_dict['FGSM']
# Run the multi-gpu trainer for adversarial training
flags.update({'adv_train': True,
'attack_type_train': 'FGSM',
})
HParams = namedtuple('HParams', flags.keys())
hparams = HParams(**flags)
np.random.seed(42)
tf.set_random_seed(42)
with tf.variable_scope(None, 'runner'):
report_dict = run_trainer(hparams)
report_2.train_adv_train_clean_eval = report_dict['train']
report_2.adv_train_clean_eval = report_dict['test']
report_2.adv_train_adv_eval = report_dict['FGSM']
self.assertClose(report.train_clean_train_clean_eval,
report_2.train_clean_train_clean_eval,
atol=5e-2)
self.assertClose(report.clean_train_clean_eval,
report_2.clean_train_clean_eval,
atol=2e-2)
self.assertClose(report.clean_train_adv_eval,
report_2.clean_train_adv_eval,
atol=5e-2)
self.assertClose(report.train_adv_train_clean_eval,
report_2.train_adv_train_clean_eval,
atol=1e-1)
self.assertClose(report.adv_train_clean_eval,
report_2.adv_train_clean_eval,
atol=2e-2)
self.assertClose(report.adv_train_adv_eval,
report_2.adv_train_adv_eval,
atol=1e-1)