def run_mnist_adv(num_epochs=NUM_EPOCHS, batch_size=BATCH_SIZE, testing=False, learning_rate=LEARNING_RATE): # Object used to keep track of (and return) key accuracies report = AccuracyReport() # set random seed tf.set_random_seed(42) # can use gpu config = tf.ConfigProto( device_count = {'GPU': 1 , 'CPU': 1} ) # Create TF session and set Keras backend session as TF sess = tf.Session(config=config) keras.backend.set_session(sess) # Get MNIST test data mnist = MNIST() x_train, y_train = mnist.get_set("train") x_test, y_test = mnist.get_set("test") # Obtain image params n_rows, n_cols, n_channels = x_train.shape[1:4] n_classes = y_train.shape[1] # define TF model graph model = ConvNet((n_rows, n_cols, n_channels), n_classes) model(model.input) wrap = KerasModelWrapper(model) fgsm = FastGradientMethod(wrap, sess=sess) fgsm_params = { 'eps': 0.3, 'clip_min': 0., 'clip_max': 1. } adv_acc_metric = get_adversarial_acc_metric(model, fgsm, fgsm_params) model.compile( optimizer=keras.optimizers.Adam(learning_rate), loss='categorical_crossentropy', metrics=['accuracy', adv_acc_metric] ) # Train an MNIST model model.fit(x_train, y_train, batch_size=batch_size, epochs=num_epochs, validation_data=(x_test, y_test), verbose=1) # Evaluate the accuracy on legitimate and adversarial test examples _, acc, adv_acc = model.evaluate(x_test, y_test, batch_size=batch_size, verbose=0) report.clean_train_clean_eval = acc report.clean_train_adv_eval = adv_acc print('Test accuracy on legitimate examples: %0.4f' % acc) print('Test accuracy on adversarial examples: %0.4f\n' % adv_acc) # Calculate training error if testing: _, train_acc, train_adv_acc = model.evaluate(x_train, y_train, batch_size=batch_size, verbose=0) report.train_clean_train_clean_eval = train_acc report.train_clean_train_adv_eval = train_adv_acc print("Repeating the process, using adversarial training") # Redefine Keras model model_2 = ConvNet((n_rows, n_cols, n_channels), n_classes) model_2(model_2.input) wrap_2 = KerasModelWrapper(model_2) fgsm_2 = FastGradientMethod(wrap_2, sess=sess) # Use a loss function based on legitimate and adversarial examples adv_loss_2 = get_adversarial_loss(model_2, fgsm_2, fgsm_params) adv_acc_metric_2 = get_adversarial_acc_metric(model_2, fgsm_2, fgsm_params) model_2.compile( optimizer=keras.optimizers.Adam(learning_rate), loss=adv_loss_2, metrics=['accuracy', adv_acc_metric_2] ) # Train an MNIST model model_2.fit(x_train, y_train, batch_size=batch_size, epochs=num_epochs, validation_data=(x_test, y_test), verbose=1) # Evaluate the accuracy on legitimate and adversarial test examples _, acc, adv_acc = model_2.evaluate(x_test, y_test, batch_size=batch_size, verbose=0) report.adv_train_clean_eval = acc report.adv_train_adv_eval = adv_acc print('Test accuracy on legitimate examples: %0.4f' % acc) print('Test accuracy on adversarial examples: %0.4f\n' % adv_acc) # Calculate training error if testing: _, train_acc, train_adv_acc = model_2.evaluate(x_train, y_train, batch_size=batch_size, verbose=0) report.train_adv_train_clean_eval = train_acc report.train_adv_train_adv_eval = train_adv_acc return report
def train_zero_knowledge_gandef_model(train_start=0, train_end=60000, test_start=0, test_end=10000, smoke_test=True, save=False, testing=False, backprop_through_attack=False, num_threads=None): """ MNIST cleverhans tutorial :param train_start: index of first training set example :param train_end: index of last training set example :param test_start: index of first test set example :param test_end: index of last test set example :param nb_epochs: number of epochs to train model :param train_batch_size: size of training batches :param test_batch_size: size of testing batches :param learning_rate: learning rate for training :param save: if true, the final model will be saved :param testing: if true, complete an AccuracyReport for unit tests to verify that performance is adequate :param backprop_through_attack: If True, backprop through adversarial example construction process during adversarial training. :return: an AccuracyReport object """ # Object used to keep track of (and return) key accuracies report = AccuracyReport() # Set TF random seed to improve reproducibility tf.set_random_seed(1234) # Set logging level to see debug information set_log_level(logging.DEBUG) # Create TF session if num_threads: config_args = dict(intra_op_parallelism_threads=1) else: config_args = {} sess = tf.Session(config=tf.ConfigProto(**config_args)) # Get MNIST test data X_train, Y_train, X_test, Y_test = data_fashion_mnist() if smoke_test: X_train, Y_train, X_test, Y_test = X_train[: 256], Y_train[: 256], X_test[: 256], Y_test[: 256] # Define input TF placeholder x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1)) y_soft = tf.placeholder(tf.float32, shape=(None, 10)) # Prepare optimizer learning_rate = 1e-4 clf_opt = tf.train.AdamOptimizer(learning_rate) dic_opt = tf.train.AdamOptimizer(learning_rate * 10) # Train an MNIST model train_params = { 'nb_epochs': 80, 'batch_size': 128, 'trade_off': 2, 'inner_epochs': 1 } rng = np.random.RandomState([2017, 8, 30]) # Adversarial training print("Start adversarial training") zero_knowledge_gandef_model = make_zero_knowledge_gandef_model( name="model_zero_knowledge_gandef") aug_x = gaussian_augment(x, std=1) preds_clean = zero_knowledge_gandef_model(x) preds_aug = zero_knowledge_gandef_model(aug_x) def cross_entropy(truth, preds, mean=True): # Get the logits operator op = preds.op if op.type == "Softmax": logits, = op.inputs else: logits = preds # Calculate cross entropy loss out = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=truth) # Take average loss and return if mean: out = tf.reduce_mean(out) return out def sigmoid_entropy(truth, preds, mean=True): # Get the logits operator op = preds.op if op.type == "Softmax": logits, = op.inputs else: logits = preds # Calculate cross entropy loss out = tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=truth) # Take average loss and return if mean: out = tf.reduce_mean(out) return out # Perform and evaluate adversarial training gan_train_v2(sess, x, y_soft, preds_clean, X_train, Y_train, loss_func=[cross_entropy, sigmoid_entropy], optimizer=[clf_opt, dic_opt], predictions_adv=preds_aug, evaluate=None, args=train_params, rng=rng, var_list=zero_knowledge_gandef_model.get_gan_params()) # Evaluate the accuracy of the MNIST model on Clean examples preds_clean = zero_knowledge_gandef_model(x) eval_params = { 'batch_size': 128, 'use_dic': False, 'is_clean': True, 'reject_threshold': 0.5 } clean_acc = confident_model_eval(sess, x, y_soft, preds_clean, X_test, Y_test, args=eval_params) print('Test accuracy on Clean test examples: %0.4f\n' % clean_acc) report.adv_train_clean_eval = clean_acc # Evaluate the accuracy of the MNIST model on FGSM examples fgsm_params = {'eps': 0.6, 'clip_min': -1., 'clip_max': 1.} fgsm_att = FastGradientMethod(zero_knowledge_gandef_model, sess=sess) fgsm_adv = fgsm_att.generate(x, **fgsm_params) preds_fgsm_adv = zero_knowledge_gandef_model(fgsm_adv) eval_params = { 'batch_size': 128, 'use_dic': False, 'is_clean': False, 'reject_threshold': 0.5 } fgsm_acc = confident_model_eval(sess, x, y_soft, preds_fgsm_adv, X_test, Y_test, args=eval_params) print('Test accuracy on FGSM test examples: %0.4f\n' % fgsm_acc) report.adv_train_adv_eval = fgsm_acc # Evaluate the accuracy of the MNIST model on BIM examples bim_params = {'eps': 0.6, 'eps_iter': 0.1, 'clip_min': -1., 'clip_max': 1.} bim_att = BasicIterativeMethod(zero_knowledge_gandef_model, sess=sess) bim_adv = bim_att.generate(x, **bim_params) preds_bim_adv = zero_knowledge_gandef_model(bim_adv) eval_params = { 'batch_size': 128, 'use_dic': False, 'is_clean': False, 'reject_threshold': 0.5 } bim_acc = confident_model_eval(sess, x, y_soft, preds_bim_adv, X_test, Y_test, args=eval_params) print('Test accuracy on BIM test examples: %0.4f\n' % bim_acc) report.adv_train_adv_eval = bim_acc # Evaluate the accuracy of the MNIST model on PGD examples pgd_params = { 'eps': 0.6, 'eps_iter': 0.02, 'nb_iter': 40, 'clip_min': -1., 'clip_max': 1., 'rand_init': True } pgd_att = MadryEtAl(zero_knowledge_gandef_model, sess=sess) pgd_adv = pgd_att.generate(x, **bim_params) preds_pgd_adv = zero_knowledge_gandef_model(pgd_adv) eval_params = { 'batch_size': 128, 'use_dic': False, 'is_clean': False, 'reject_threshold': 0.5 } pgd_acc = confident_model_eval(sess, x, y_soft, preds_pgd_adv, X_test, Y_test, args=eval_params) print('Test accuracy on PGD test examples: %0.4f\n' % pgd_acc) report.adv_train_adv_eval = pgd_acc # Save model if save: model_path = "models/zero_knowledge_gandef" vars_to_save = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='model_zero_knowledge_gandef*') assert len(vars_to_save) > 0 saver = tf.train.Saver(var_list=vars_to_save) saver.save(sess, model_path) print('Model saved\n') else: print('Model not saved\n')
def test_run_single_gpu_fgsm(self): """ Test the basic single GPU performance by comparing to the FGSM tutorial. """ from cleverhans_tutorials import mnist_tutorial_tf # Run the MNIST tutorial on a dataset of reduced size flags = { 'train_start': 0, 'train_end': 5000, 'test_start': 0, 'test_end': 333, 'nb_epochs': 5, 'testing': True } report = mnist_tutorial_tf.mnist_tutorial(**flags) # Run the multi-gpu trainer for clean training flags.update({ 'batch_size': 128, 'adam_lrn': 0.001, 'dataset': 'mnist', 'only_adv_train': False, 'eval_iters': 1, 'ngpu': 1, 'fast_tests': False, 'attack_type_train': '', 'save_dir': None, 'save_steps': 10000, 'attack_nb_iter_train': None, 'save': False, 'model_type': 'basic', 'attack_type_test': 'FGSM' }) flags.update({'adv_train': False}) HParams = namedtuple('HParams', flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.compat.v1.set_random_seed(42) with tf.compat.v1.variable_scope(None, 'runner'): report_dict = run_trainer(hparams) report_2 = AccuracyReport() report_2.train_clean_train_clean_eval = report_dict['train'] report_2.clean_train_clean_eval = report_dict['test'] report_2.clean_train_adv_eval = report_dict['FGSM'] # Run the multi-gpu trainer for adversarial training flags.update({'adv_train': True, 'attack_type_train': 'FGSM'}) HParams = namedtuple('HParams', flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.compat.v1.set_random_seed(42) with tf.compat.v1.variable_scope(None, 'runner'): report_dict = run_trainer(hparams) report_2.train_adv_train_clean_eval = report_dict['train'] report_2.adv_train_clean_eval = report_dict['test'] report_2.adv_train_adv_eval = report_dict['FGSM'] self.assertClose(report.train_clean_train_clean_eval, report_2.train_clean_train_clean_eval, atol=5e-2) self.assertClose(report.clean_train_clean_eval, report_2.clean_train_clean_eval, atol=2e-2) self.assertClose(report.clean_train_adv_eval, report_2.clean_train_adv_eval, atol=5e-2) self.assertClose(report.train_adv_train_clean_eval, report_2.train_adv_train_clean_eval, atol=1e-1) self.assertClose(report.adv_train_clean_eval, report_2.adv_train_clean_eval, atol=2e-2) self.assertClose(report.adv_train_adv_eval, report_2.adv_train_adv_eval, atol=1e-1)
def mnist_tutorial(train_start=0, train_end=60000, test_start=0, test_end=10000, nb_epochs=NB_EPOCHS, batch_size=BATCH_SIZE, learning_rate=LEARNING_RATE, testing=False, label_smoothing=0.1): """ MNIST CleverHans tutorial :param train_start: index of first training set example :param train_end: index of last training set example :param test_start: index of first test set example :param test_end: index of last test set example :param nb_epochs: number of epochs to train model :param batch_size: size of training batches :param learning_rate: learning rate for training :param testing: if true, training error is calculated :param label_smoothing: float, amount of label smoothing for cross entropy :return: an AccuracyReport object """ # Object used to keep track of (and return) key accuracies report = AccuracyReport() # Set TF random seed to improve reproducibility tf.set_random_seed(1234) # Force TensorFlow to use single thread to improve reproducibility config = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1) if keras.backend.image_data_format() != 'channels_last': raise NotImplementedError("this tutorial requires keras to be configured to channels_last format") # Create TF session and set as Keras backend session sess = tf.Session(config=config) keras.backend.set_session(sess) # Get MNIST test data mnist = MNIST(train_start=train_start, train_end=train_end, test_start=test_start, test_end=test_end) x_train, y_train = mnist.get_set('train') x_test, y_test = mnist.get_set('test') # Obtain Image Parameters img_rows, img_cols, nchannels = x_train.shape[1:4] nb_classes = y_train.shape[1] # Label smoothing y_train -= label_smoothing * (y_train - 1. / nb_classes) # Define Keras model model = cnn_model(img_rows=img_rows, img_cols=img_cols, channels=nchannels, nb_filters=64, nb_classes=nb_classes) print("Defined Keras model.") # To be able to call the model in the custom loss, we need to call it once # before, see https://github.com/tensorflow/tensorflow/issues/23769 model(model.input) # Initialize the Fast Gradient Sign Method (FGSM) attack object wrap = KerasModelWrapper(model) fgsm = FastGradientMethod(wrap, sess=sess) fgsm_params = {'eps': 0.3, 'clip_min': 0., 'clip_max': 1.} adv_acc_metric = get_adversarial_acc_metric(model, fgsm, fgsm_params) model.compile( optimizer=keras.optimizers.Adam(learning_rate), loss='categorical_crossentropy', metrics=['accuracy', adv_acc_metric] ) # Train an MNIST model model.fit(x_train, y_train, batch_size=batch_size, epochs=nb_epochs, validation_data=(x_test, y_test), verbose=2) # Evaluate the accuracy on legitimate and adversarial test examples _, acc, adv_acc = model.evaluate(x_test, y_test, batch_size=batch_size, verbose=0) report.clean_train_clean_eval = acc report.clean_train_adv_eval = adv_acc print('Test accuracy on legitimate examples: %0.4f' % acc) print('Test accuracy on adversarial examples: %0.4f\n' % adv_acc) # Calculate training error if testing: _, train_acc, train_adv_acc = model.evaluate(x_train, y_train, batch_size=batch_size, verbose=0) report.train_clean_train_clean_eval = train_acc report.train_clean_train_adv_eval = train_adv_acc print("Repeating the process, using adversarial training") # Redefine Keras model model_2 = cnn_model(img_rows=img_rows, img_cols=img_cols, channels=nchannels, nb_filters=64, nb_classes=nb_classes) model_2(model_2.input) wrap_2 = KerasModelWrapper(model_2) fgsm_2 = FastGradientMethod(wrap_2, sess=sess) # Use a loss function based on legitimate and adversarial examples adv_loss_2 = get_adversarial_loss(model_2, fgsm_2, fgsm_params) adv_acc_metric_2 = get_adversarial_acc_metric(model_2, fgsm_2, fgsm_params) model_2.compile( optimizer=keras.optimizers.Adam(learning_rate), loss=adv_loss_2, metrics=['accuracy', adv_acc_metric_2] ) # Train an MNIST model model_2.fit(x_train, y_train, batch_size=batch_size, epochs=nb_epochs, validation_data=(x_test, y_test), verbose=2) # Evaluate the accuracy on legitimate and adversarial test examples _, acc, adv_acc = model_2.evaluate(x_test, y_test, batch_size=batch_size, verbose=0) report.adv_train_clean_eval = acc report.adv_train_adv_eval = adv_acc print('Test accuracy on legitimate examples: %0.4f' % acc) print('Test accuracy on adversarial examples: %0.4f\n' % adv_acc) # Calculate training error if testing: _, train_acc, train_adv_acc = model_2.evaluate(x_train, y_train, batch_size=batch_size, verbose=0) report.train_adv_train_clean_eval = train_acc report.train_adv_train_adv_eval = train_adv_acc return report
def helper_run_multi_gpu_madryetal(self, extra_flags=None): """ Compare the single GPU performance to multiGPU performance. """ # Run the trainers on a dataset of reduced size flags = { 'train_start': 0, 'train_end': 5000, 'test_start': 0, 'test_end': 333, 'nb_epochs': 5, 'testing': True } # Run the multi-gpu trainer for adversarial training flags.update({ 'batch_size': 128, 'adam_lrn': 0.001, 'dataset': 'mnist', 'only_adv_train': False, 'eval_iters': 1, 'fast_tests': True, 'save_dir': None, 'save_steps': 10000, 'attack_nb_iter_train': 10, 'sync_step': None, 'adv_train': True, 'save': False, 'model_type': 'basic', 'attack_type_test': 'MadryEtAl_y' }) if extra_flags is not None: flags.update(extra_flags) # Run the multi-gpu trainer for adversarial training using 2 gpus # trainer_multigpu by default sets `allow_soft_placement=True` flags.update({ 'ngpu': 2, 'attack_type_train': 'MadryEtAl_y_multigpu', 'sync_step': 1 }) HParams = namedtuple('HParams', flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.compat.v1.set_random_seed(42) with tf.compat.v1.variable_scope(None, 'runner'): report_dict = run_trainer(hparams) report_m = AccuracyReport() report_m.train_adv_train_clean_eval = report_dict['train'] report_m.adv_train_clean_eval = report_dict['test'] report_m.adv_train_adv_eval = report_dict['MadryEtAl_y'] flags.update({'ngpu': 1, 'attack_type_train': 'MadryEtAl_y'}) hparams = HParams(**flags) np.random.seed(42) tf.compat.v1.set_random_seed(42) with tf.compat.v1.variable_scope(None, 'runner'): report_dict = run_trainer(hparams) report_s = AccuracyReport() report_s.train_adv_train_clean_eval = report_dict['train'] report_s.adv_train_clean_eval = report_dict['test'] report_s.adv_train_adv_eval = report_dict['MadryEtAl_y'] self.assertClose(report_s.train_adv_train_clean_eval, report_m.train_adv_train_clean_eval, atol=5e-2) self.assertClose(report_s.adv_train_clean_eval, report_m.adv_train_clean_eval, atol=2e-2) self.assertClose(report_s.adv_train_adv_eval, report_m.adv_train_adv_eval, atol=5e-2)
def test_run_single_gpu_fgsm(self): """ Test the basic single GPU performance by comparing to the FGSM tutorial. """ from cleverhans_tutorials import mnist_tutorial_tf # Run the MNIST tutorial on a dataset of reduced size flags = { "train_start": 0, "train_end": 5000, "test_start": 0, "test_end": 333, "nb_epochs": 5, "testing": True, } report = mnist_tutorial_tf.mnist_tutorial(**flags) # Run the multi-gpu trainer for clean training flags.update( { "batch_size": 128, "adam_lrn": 0.001, "dataset": "mnist", "only_adv_train": False, "eval_iters": 1, "ngpu": 1, "fast_tests": False, "attack_type_train": "", "save_dir": None, "save_steps": 10000, "attack_nb_iter_train": None, "save": False, "model_type": "basic", "attack_type_test": "FGSM", } ) flags.update({"adv_train": False}) HParams = namedtuple("HParams", flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.set_random_seed(42) with tf.variable_scope(None, "runner"): report_dict = run_trainer(hparams) report_2 = AccuracyReport() report_2.train_clean_train_clean_eval = report_dict["train"] report_2.clean_train_clean_eval = report_dict["test"] report_2.clean_train_adv_eval = report_dict["FGSM"] # Run the multi-gpu trainer for adversarial training flags.update({"adv_train": True, "attack_type_train": "FGSM"}) HParams = namedtuple("HParams", flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.set_random_seed(42) with tf.variable_scope(None, "runner"): report_dict = run_trainer(hparams) report_2.train_adv_train_clean_eval = report_dict["train"] report_2.adv_train_clean_eval = report_dict["test"] report_2.adv_train_adv_eval = report_dict["FGSM"] self.assertClose( report.train_clean_train_clean_eval, report_2.train_clean_train_clean_eval, atol=5e-2, ) self.assertClose( report.clean_train_clean_eval, report_2.clean_train_clean_eval, atol=2e-2 ) self.assertClose( report.clean_train_adv_eval, report_2.clean_train_adv_eval, atol=5e-2 ) self.assertClose( report.train_adv_train_clean_eval, report_2.train_adv_train_clean_eval, atol=1e-1, ) self.assertClose( report.adv_train_clean_eval, report_2.adv_train_clean_eval, atol=2e-2 ) self.assertClose( report.adv_train_adv_eval, report_2.adv_train_adv_eval, atol=1e-1 )
def helper_run_multi_gpu_madryetal(self, extra_flags=None): """ Compare the single GPU performance to multiGPU performance. """ # Run the trainers on a dataset of reduced size flags = { "train_start": 0, "train_end": 5000, "test_start": 0, "test_end": 333, "nb_epochs": 5, "testing": True, } # Run the multi-gpu trainer for adversarial training flags.update( { "batch_size": 128, "adam_lrn": 0.001, "dataset": "mnist", "only_adv_train": False, "eval_iters": 1, "fast_tests": True, "save_dir": None, "save_steps": 10000, "attack_nb_iter_train": 10, "sync_step": None, "adv_train": True, "save": False, "model_type": "basic", "attack_type_test": "MadryEtAl_y", } ) if extra_flags is not None: flags.update(extra_flags) # Run the multi-gpu trainer for adversarial training using 2 gpus # trainer_multigpu by default sets `allow_soft_placement=True` flags.update( {"ngpu": 2, "attack_type_train": "MadryEtAl_y_multigpu", "sync_step": 1} ) HParams = namedtuple("HParams", flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.set_random_seed(42) with tf.variable_scope(None, "runner"): report_dict = run_trainer(hparams) report_m = AccuracyReport() report_m.train_adv_train_clean_eval = report_dict["train"] report_m.adv_train_clean_eval = report_dict["test"] report_m.adv_train_adv_eval = report_dict["MadryEtAl_y"] flags.update({"ngpu": 1, "attack_type_train": "MadryEtAl_y"}) hparams = HParams(**flags) np.random.seed(42) tf.set_random_seed(42) with tf.variable_scope(None, "runner"): report_dict = run_trainer(hparams) report_s = AccuracyReport() report_s.train_adv_train_clean_eval = report_dict["train"] report_s.adv_train_clean_eval = report_dict["test"] report_s.adv_train_adv_eval = report_dict["MadryEtAl_y"] self.assertClose( report_s.train_adv_train_clean_eval, report_m.train_adv_train_clean_eval, atol=5e-2, ) self.assertClose( report_s.adv_train_clean_eval, report_m.adv_train_clean_eval, atol=2e-2 ) self.assertClose( report_s.adv_train_adv_eval, report_m.adv_train_adv_eval, atol=5e-2 )
def test_run_single_gpu_fgsm(self): """ Test the basic single GPU performance by comparing to the FGSM tutorial. """ from cleverhans_tutorials import mnist_tutorial_tf # Run the MNIST tutorial on a dataset of reduced size flags = {'train_start': 0, 'train_end': 5000, 'test_start': 0, 'test_end': 333, 'nb_epochs': 5, 'testing': True} report = mnist_tutorial_tf.mnist_tutorial(**flags) # Run the multi-gpu trainer for clean training flags.update({'batch_size': 128, 'adam_lrn': 0.001, 'dataset': 'mnist', 'only_adv_train': False, 'eval_iters': 1, 'ngpu': 1, 'fast_tests': False, 'attack_type_train': '', 'save_dir': None, 'save_steps': 10000, 'attack_nb_iter_train': None, 'save': False, 'model_type': 'basic', 'attack_type_test': 'FGSM'}) flags.update({'adv_train': False}) HParams = namedtuple('HParams', flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.set_random_seed(42) with tf.variable_scope(None, 'runner'): report_dict = run_trainer(hparams) report_2 = AccuracyReport() report_2.train_clean_train_clean_eval = report_dict['train'] report_2.clean_train_clean_eval = report_dict['test'] report_2.clean_train_adv_eval = report_dict['FGSM'] # Run the multi-gpu trainer for adversarial training flags.update({'adv_train': True, 'attack_type_train': 'FGSM', }) HParams = namedtuple('HParams', flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.set_random_seed(42) with tf.variable_scope(None, 'runner'): report_dict = run_trainer(hparams) report_2.train_adv_train_clean_eval = report_dict['train'] report_2.adv_train_clean_eval = report_dict['test'] report_2.adv_train_adv_eval = report_dict['FGSM'] self.assertClose(report.train_clean_train_clean_eval, report_2.train_clean_train_clean_eval, atol=5e-2) self.assertClose(report.clean_train_clean_eval, report_2.clean_train_clean_eval, atol=2e-2) self.assertClose(report.clean_train_adv_eval, report_2.clean_train_adv_eval, atol=5e-2) self.assertClose(report.train_adv_train_clean_eval, report_2.train_adv_train_clean_eval, atol=1e-1) self.assertClose(report.adv_train_clean_eval, report_2.adv_train_clean_eval, atol=2e-2) self.assertClose(report.adv_train_adv_eval, report_2.adv_train_adv_eval, atol=1e-1)
def helper_run_multi_gpu_madryetal(self, extra_flags=None): """ Compare the single GPU performance to multiGPU performance. """ # Run the trainers on a dataset of reduced size flags = {'train_start': 0, 'train_end': 5000, 'test_start': 0, 'test_end': 333, 'nb_epochs': 5, 'testing': True} # Run the multi-gpu trainer for adversarial training flags.update({'batch_size': 128, 'adam_lrn': 0.001, 'dataset': 'mnist', 'only_adv_train': False, 'eval_iters': 1, 'fast_tests': True, 'save_dir': None, 'save_steps': 10000, 'attack_nb_iter_train': 10, 'sync_step': None, 'adv_train': True, 'save': False, 'model_type': 'basic', 'attack_type_test': 'MadryEtAl_y', }) if extra_flags is not None: flags.update(extra_flags) # Run the multi-gpu trainer for adversarial training using 2 gpus # trainer_multigpu by default sets `allow_soft_placement=True` flags.update({'ngpu': 2, 'attack_type_train': 'MadryEtAl_y_multigpu', 'sync_step': 1}) HParams = namedtuple('HParams', flags.keys()) hparams = HParams(**flags) np.random.seed(42) tf.set_random_seed(42) with tf.variable_scope(None, 'runner'): report_dict = run_trainer(hparams) report_m = AccuracyReport() report_m.train_adv_train_clean_eval = report_dict['train'] report_m.adv_train_clean_eval = report_dict['test'] report_m.adv_train_adv_eval = report_dict['MadryEtAl_y'] flags.update({'ngpu': 1, 'attack_type_train': 'MadryEtAl_y', }) hparams = HParams(**flags) np.random.seed(42) tf.set_random_seed(42) with tf.variable_scope(None, 'runner'): report_dict = run_trainer(hparams) report_s = AccuracyReport() report_s.train_adv_train_clean_eval = report_dict['train'] report_s.adv_train_clean_eval = report_dict['test'] report_s.adv_train_adv_eval = report_dict['MadryEtAl_y'] self.assertClose(report_s.train_adv_train_clean_eval, report_m.train_adv_train_clean_eval, atol=5e-2) self.assertClose(report_s.adv_train_clean_eval, report_m.adv_train_clean_eval, atol=2e-2) self.assertClose(report_s.adv_train_adv_eval, report_m.adv_train_adv_eval, atol=5e-2)