def test_mnist_tutorial_tf(self): import tensorflow as tf from cleverhans_tutorials import mnist_tutorial_tf # Run the MNIST tutorial on a dataset of reduced size test_dataset_indices = { "train_start": 0, "train_end": 5000, "test_start": 0, "test_end": 333, "nb_epochs": 2, "testing": True, } g = tf.Graph() with g.as_default(): np.random.seed(42) report = mnist_tutorial_tf.mnist_tutorial(num_threads=1, **test_dataset_indices) # Check accuracy values contained in the AccuracyReport object self.assertGreater(report.train_clean_train_clean_eval, 0.97) self.assertLess(report.train_clean_train_adv_eval, 0.05) self.assertGreater(report.train_adv_train_clean_eval, 0.93) self.assertGreater(report.train_adv_train_adv_eval, 0.4) # Check that the tutorial is deterministic (seeded properly) atol_fac = 2e-2 if HAS_GPU else 1e-6 g = tf.Graph() with g.as_default(): np.random.seed(42) report_2 = mnist_tutorial_tf.mnist_tutorial(num_threads=1, **test_dataset_indices) self.assertClose( report.train_clean_train_clean_eval, report_2.train_clean_train_clean_eval, atol=atol_fac * 1, ) self.assertClose( report.train_clean_train_adv_eval, report_2.train_clean_train_adv_eval, atol=atol_fac * 1, ) self.assertClose( report.train_adv_train_clean_eval, report_2.train_adv_train_clean_eval, atol=atol_fac * 1, ) self.assertClose( report.train_adv_train_adv_eval, report_2.train_adv_train_adv_eval, atol=atol_fac * 1, )
def test_mnist_tutorial_tf(self): np.random.seed(42) import tensorflow as tf tf.set_random_seed(42) from cleverhans_tutorials import mnist_tutorial_tf # Run the MNIST tutorial on a dataset of reduced size test_dataset_indices = { 'train_start': 0, 'train_end': 5000, 'test_start': 0, 'test_end': 333, 'nb_epochs': 2, 'testing': True } report = mnist_tutorial_tf.mnist_tutorial(**test_dataset_indices) # Check accuracy values contained in the AccuracyReport object self.assertGreater(report.train_clean_train_clean_eval, 0.97) self.assertLess(report.train_clean_train_adv_eval, 0.029) self.assertGreater(report.train_adv_train_clean_eval, 0.95) self.assertGreater(report.train_adv_train_adv_eval, 0.4) # Check that the tutorial is deterministic (seeded properly) if LooseVersion(tf.__version__) >= LooseVersion('1.1.0'): atol_fac = 1 else: atol_fac = 2 report_2 = mnist_tutorial_tf.mnist_tutorial(**test_dataset_indices) self.assertClose(report.train_clean_train_clean_eval, report_2.train_clean_train_clean_eval, atol=atol_fac * 5e-3) self.assertClose(report.train_clean_train_adv_eval, report_2.train_clean_train_adv_eval, atol=atol_fac * 5e-3) self.assertClose(report.train_adv_train_clean_eval, report_2.train_adv_train_clean_eval, atol=atol_fac * 2e-2) self.assertClose(report.train_adv_train_adv_eval, report_2.train_adv_train_adv_eval, atol=atol_fac * 2e-1)
def test_mnist_tutorial_tf(self): import tensorflow as tf from cleverhans_tutorials import mnist_tutorial_tf # Run the MNIST tutorial on a dataset of reduced size test_dataset_indices = {'train_start': 0, 'train_end': 5000, 'test_start': 0, 'test_end': 333, 'nb_epochs': 2, 'testing': True} g = tf.Graph() with g.as_default(): np.random.seed(42) report = mnist_tutorial_tf.mnist_tutorial(num_threads=1, **test_dataset_indices) # Check accuracy values contained in the AccuracyReport object self.assertGreater(report.train_clean_train_clean_eval, 0.97) self.assertLess(report.train_clean_train_adv_eval, 0.05) self.assertGreater(report.train_adv_train_clean_eval, 0.93) self.assertGreater(report.train_adv_train_adv_eval, 0.4) # Check that the tutorial is deterministic (seeded properly) atol_fac = 2e-2 if HAS_GPU else 1e-6 g = tf.Graph() with g.as_default(): np.random.seed(42) report_2 = mnist_tutorial_tf.mnist_tutorial(num_threads=1, **test_dataset_indices) self.assertClose(report.train_clean_train_clean_eval, report_2.train_clean_train_clean_eval, atol=atol_fac * 1) self.assertClose(report.train_clean_train_adv_eval, report_2.train_clean_train_adv_eval, atol=atol_fac * 1) self.assertClose(report.train_adv_train_clean_eval, report_2.train_adv_train_clean_eval, atol=atol_fac * 1) self.assertClose(report.train_adv_train_adv_eval, report_2.train_adv_train_adv_eval, atol=atol_fac * 1)
def test_mnist_tutorial_tf(self): np.random.seed(42) import tensorflow as tf tf.set_random_seed(42) from cleverhans_tutorials import mnist_tutorial_tf # Run the MNIST tutorial on a dataset of reduced size test_dataset_indices = { 'train_start': 0, 'train_end': 5000, 'test_start': 0, 'test_end': 333, 'nb_epochs': 2, 'testing': True } report = mnist_tutorial_tf.mnist_tutorial(**test_dataset_indices) # Check accuracy values contained in the AccuracyReport object self.assertTrue(report.train_clean_train_clean_eval > 0.90) self.assertTrue(report.train_clean_train_adv_eval < 0.05) self.assertTrue(report.train_adv_train_clean_eval > 0.90) self.assertTrue(report.train_adv_train_adv_eval > 0.3)
def test_run_single_gpu_fgsm(self): """ Test the basic single GPU performance by comparing to the FGSM tutorial. """ from cleverhans_tutorials import mnist_tutorial_tf # Run the MNIST tutorial on a dataset of reduced size flags = { 'train_start': 0, 'train_end': 5000, 'test_start': 0, 'test_end': 333, 'nb_epochs': 5, 'testing': True } report = mnist_tutorial_tf.mnist_tutorial(**flags) # Run the multi-gpu trainer for clean training flags.update({ 'batch_size': 128, 'adam_lrn': 0.001, 'dataset': 'mnist', 'only_adv_train': False, 'eval_iters': 1, 'ngpu': 1, 'fast_tests': False, 'attack_type_train': '', 'save_dir': None, 'save_steps': 10000, 'attack_nb_iter_train': None, 'save': False, 'model_type': 'basic', 'attack_type_test': 'FGSM' }) flags.update({'adv_train': False}) HParams = namedtuple('HParams', flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.compat.v1.set_random_seed(42) with tf.compat.v1.variable_scope(None, 'runner'): report_dict = run_trainer(hparams) report_2 = AccuracyReport() report_2.train_clean_train_clean_eval = report_dict['train'] report_2.clean_train_clean_eval = report_dict['test'] report_2.clean_train_adv_eval = report_dict['FGSM'] # Run the multi-gpu trainer for adversarial training flags.update({'adv_train': True, 'attack_type_train': 'FGSM'}) HParams = namedtuple('HParams', flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.compat.v1.set_random_seed(42) with tf.compat.v1.variable_scope(None, 'runner'): report_dict = run_trainer(hparams) report_2.train_adv_train_clean_eval = report_dict['train'] report_2.adv_train_clean_eval = report_dict['test'] report_2.adv_train_adv_eval = report_dict['FGSM'] self.assertClose(report.train_clean_train_clean_eval, report_2.train_clean_train_clean_eval, atol=5e-2) self.assertClose(report.clean_train_clean_eval, report_2.clean_train_clean_eval, atol=2e-2) self.assertClose(report.clean_train_adv_eval, report_2.clean_train_adv_eval, atol=5e-2) self.assertClose(report.train_adv_train_clean_eval, report_2.train_adv_train_clean_eval, atol=1e-1) self.assertClose(report.adv_train_clean_eval, report_2.adv_train_clean_eval, atol=2e-2) self.assertClose(report.adv_train_adv_eval, report_2.adv_train_adv_eval, atol=1e-1)
def test_run_single_gpu_fgsm(self): """ Test the basic single GPU performance by comparing to the FGSM tutorial. """ from cleverhans_tutorials import mnist_tutorial_tf # Run the MNIST tutorial on a dataset of reduced size flags = { "train_start": 0, "train_end": 5000, "test_start": 0, "test_end": 333, "nb_epochs": 5, "testing": True, } report = mnist_tutorial_tf.mnist_tutorial(**flags) # Run the multi-gpu trainer for clean training flags.update( { "batch_size": 128, "adam_lrn": 0.001, "dataset": "mnist", "only_adv_train": False, "eval_iters": 1, "ngpu": 1, "fast_tests": False, "attack_type_train": "", "save_dir": None, "save_steps": 10000, "attack_nb_iter_train": None, "save": False, "model_type": "basic", "attack_type_test": "FGSM", } ) flags.update({"adv_train": False}) HParams = namedtuple("HParams", flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.set_random_seed(42) with tf.variable_scope(None, "runner"): report_dict = run_trainer(hparams) report_2 = AccuracyReport() report_2.train_clean_train_clean_eval = report_dict["train"] report_2.clean_train_clean_eval = report_dict["test"] report_2.clean_train_adv_eval = report_dict["FGSM"] # Run the multi-gpu trainer for adversarial training flags.update({"adv_train": True, "attack_type_train": "FGSM"}) HParams = namedtuple("HParams", flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.set_random_seed(42) with tf.variable_scope(None, "runner"): report_dict = run_trainer(hparams) report_2.train_adv_train_clean_eval = report_dict["train"] report_2.adv_train_clean_eval = report_dict["test"] report_2.adv_train_adv_eval = report_dict["FGSM"] self.assertClose( report.train_clean_train_clean_eval, report_2.train_clean_train_clean_eval, atol=5e-2, ) self.assertClose( report.clean_train_clean_eval, report_2.clean_train_clean_eval, atol=2e-2 ) self.assertClose( report.clean_train_adv_eval, report_2.clean_train_adv_eval, atol=5e-2 ) self.assertClose( report.train_adv_train_clean_eval, report_2.train_adv_train_clean_eval, atol=1e-1, ) self.assertClose( report.adv_train_clean_eval, report_2.adv_train_clean_eval, atol=2e-2 ) self.assertClose( report.adv_train_adv_eval, report_2.adv_train_adv_eval, atol=1e-1 )
def test_run_single_gpu_fgsm(self): """ Test the basic single GPU performance by comparing to the FGSM tutorial. """ from cleverhans_tutorials import mnist_tutorial_tf # Run the MNIST tutorial on a dataset of reduced size flags = {'train_start': 0, 'train_end': 5000, 'test_start': 0, 'test_end': 333, 'nb_epochs': 5, 'testing': True} report = mnist_tutorial_tf.mnist_tutorial(**flags) # Run the multi-gpu trainer for clean training flags.update({'batch_size': 128, 'adam_lrn': 0.001, 'dataset': 'mnist', 'only_adv_train': False, 'eval_iters': 1, 'ngpu': 1, 'fast_tests': False, 'attack_type_train': '', 'save_dir': None, 'save_steps': 10000, 'attack_nb_iter_train': None, 'save': False, 'model_type': 'basic', 'attack_type_test': 'FGSM'}) flags.update({'adv_train': False}) HParams = namedtuple('HParams', flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.set_random_seed(42) with tf.variable_scope(None, 'runner'): report_dict = run_trainer(hparams) report_2 = AccuracyReport() report_2.train_clean_train_clean_eval = report_dict['train'] report_2.clean_train_clean_eval = report_dict['test'] report_2.clean_train_adv_eval = report_dict['FGSM'] # Run the multi-gpu trainer for adversarial training flags.update({'adv_train': True, 'attack_type_train': 'FGSM', }) HParams = namedtuple('HParams', flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.set_random_seed(42) with tf.variable_scope(None, 'runner'): report_dict = run_trainer(hparams) report_2.train_adv_train_clean_eval = report_dict['train'] report_2.adv_train_clean_eval = report_dict['test'] report_2.adv_train_adv_eval = report_dict['FGSM'] self.assertClose(report.train_clean_train_clean_eval, report_2.train_clean_train_clean_eval, atol=5e-2) self.assertClose(report.clean_train_clean_eval, report_2.clean_train_clean_eval, atol=2e-2) self.assertClose(report.clean_train_adv_eval, report_2.clean_train_adv_eval, atol=5e-2) self.assertClose(report.train_adv_train_clean_eval, report_2.train_adv_train_clean_eval, atol=1e-1) self.assertClose(report.adv_train_clean_eval, report_2.adv_train_clean_eval, atol=2e-2) self.assertClose(report.adv_train_adv_eval, report_2.adv_train_adv_eval, atol=1e-1)