def test_mnist_blackbox(self):
import tensorflow as tf
from cleverhans_tutorials import mnist_blackbox
# Run the MNIST tutorial on a dataset of reduced size, reduced number
# of data augmentations, increased substitute holdout for faster runtime.
mnist_blackbox_args = {'train_start': 0,
'train_end': 5000,
'test_start': 0,
'test_end': 2000,
'data_aug': 1,
'holdout': 1000,
'nb_epochs': 2,
'nb_epochs_s': 6}
g = tf.Graph()
with g.as_default():
np.random.seed(42)
report = mnist_blackbox.mnist_blackbox(**mnist_blackbox_args)
# Check accuracy values contained in the AccuracyReport object
self.assertTrue(report['bbox'] > 0.7, report['bbox'])
self.assertTrue(report['sub'] > 0.7, report['sub'])
self.assertTrue(report['bbox_on_sub_adv_ex'] < 0.3,
report['bbox_on_sub_adv_ex'])
g = tf.Graph()
with g.as_default():
np.random.seed(42)
report_2 = mnist_blackbox.mnist_blackbox(**mnist_blackbox_args)
atol_fac = 1e-2 if HAS_GPU else 1e-6
self.assertClose(report['bbox'], report_2['bbox'], atol=atol_fac * 1)
self.assertClose(report['sub'], report_2['sub'], atol=atol_fac * 1)
self.assertClose(report['bbox_on_sub_adv_ex'],
report_2['bbox_on_sub_adv_ex'], atol=atol_fac * 1)
def test_mnist_blackbox(self):
from cleverhans_tutorials import mnist_blackbox
np.random.seed(42)
import tensorflow as tf
tf.set_random_seed(42)
# Run the MNIST tutorial on a dataset of reduced size, reduced number
# of data augmentations, increased substitute holdout for faster runtime.
mnist_blackbox_args = {
'train_start': 0,
'train_end': 5000,
'test_start': 0,
'test_end': 2000,
'data_aug': 1,
'holdout': 1000,
'nb_epochs': 2,
'nb_epochs_s': 6
}
report = mnist_blackbox.mnist_blackbox(**mnist_blackbox_args)
# Check accuracy values contained in the AccuracyReport object
self.assertTrue(report['bbox'] > 0.7, report['bbox'])
self.assertTrue(report['sub'] > 0.7, report['sub'])
self.assertTrue(report['bbox_on_sub_adv_ex'] < 0.2,
report['bbox_on_sub_adv_ex'])