def convert(graph, img, img_p, layers): net = load_caffe(img_p, layers) param_provider = CaffeParamProvider(net) with tf.device('/cpu:0'): images = tf.placeholder("float32", [None, 224, 224, 3], name="images") m = resnet.Model(param_provider) m.build(images, layers) sess = tf.Session() sess.run(tf.initialize_all_variables()) i = [ graph.get_tensor_by_name("conv1/relu:0"), graph.get_tensor_by_name("pool1:0"), graph.get_tensor_by_name("res2a/relu:0"), graph.get_tensor_by_name("res2b/relu:0"), graph.get_tensor_by_name("res2c/relu:0"), graph.get_tensor_by_name("res3a/relu:0"), graph.get_tensor_by_name("res5c/relu:0"), graph.get_tensor_by_name("pool5:0"), graph.get_tensor_by_name("prob:0"), ] o = sess.run(i, {images: img[np.newaxis, :]}) assert_almost_equal(net.blobs['conv1'].data, o[0]) assert_almost_equal(net.blobs['pool1'].data, o[1]) assert_almost_equal(net.blobs['res2a'].data, o[2]) assert_almost_equal(net.blobs['res2b'].data, o[3]) assert_almost_equal(net.blobs['res2c'].data, o[4]) assert_almost_equal(net.blobs['res3a'].data, o[5]) assert_almost_equal(net.blobs['res5c'].data, o[6]) assert_almost_equal(net.blobs['pool5'].data, o[7]) utils.print_prob(o[8][0]) prob_dist = np.linalg.norm(net.blobs['prob'].data - o[8]) print 'prob_dist ', prob_dist assert prob_dist < 0.2 # XXX can this be tightened? save_graph("resnet-%d.tfmodel" % layers)
def train(config): # seeding randomness tf.set_random_seed(config.training.tf_random_seed) np.random.seed(config.training.np_random_seed) # Setting up training parameters max_num_training_steps = config.training.max_num_training_steps step_size_schedule = config.training.step_size_schedule weight_decay = config.training.weight_decay momentum = config.training.momentum batch_size = config.training.batch_size adversarial_training = config.training.adversarial_training eval_during_training = config.training.eval_during_training if eval_during_training: num_eval_steps = config.training.num_eval_steps # Setting up output parameters num_output_steps = config.training.num_output_steps num_summary_steps = config.training.num_summary_steps num_checkpoint_steps = config.training.num_checkpoint_steps # Setting up the data and the model data_path = config.data.data_path raw_cifar = cifar10_input.CIFAR10Data(data_path) global_step = tf.contrib.framework.get_or_create_global_step() model = resnet.Model(config.model) # uncomment to get a list of trainable variables # model_vars = tf.trainable_variables() # slim.model_analyzer.analyze_vars(model_vars, print_info=True) # Setting up the optimizer boundaries = [int(sss[0]) for sss in step_size_schedule] boundaries = boundaries[1:] values = [sss[1] for sss in step_size_schedule] learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32), boundaries, values) total_loss = model.mean_xent + weight_decay * model.weight_decay_loss optimizer = tf.train.MomentumOptimizer(learning_rate, momentum) train_step = optimizer.minimize(total_loss, global_step=global_step) # Set up adversary attack = SpatialAttack(model, config.attack) # Setting up the Tensorboard and checkpoint outputs model_dir = config.model.output_dir if eval_during_training: eval_dir = os.path.join(model_dir, 'eval') if not os.path.exists(eval_dir): os.makedirs(eval_dir) # We add accuracy and xent twice so we can easily make three types of # comparisons in Tensorboard: # - train vs eval (for a single run) # - train of different runs # - eval of different runs saver = tf.train.Saver(max_to_keep=3) tf.summary.scalar('accuracy_adv_train', model.accuracy, collections=['adv']) tf.summary.scalar('accuracy_adv', model.accuracy, collections=['adv']) tf.summary.scalar('xent_adv_train', model.xent / batch_size, collections=['adv']) tf.summary.scalar('xent_adv', model.xent / batch_size, collections=['adv']) tf.summary.image('images_adv_train', model.x_image, collections=['adv']) adv_summaries = tf.summary.merge_all('adv') tf.summary.scalar('accuracy_nat_train', model.accuracy, collections=['nat']) tf.summary.scalar('accuracy_nat', model.accuracy, collections=['nat']) tf.summary.scalar('xent_nat_train', model.xent / batch_size, collections=['nat']) tf.summary.scalar('xent_nat', model.xent / batch_size, collections=['nat']) tf.summary.image('images_nat_train', model.x_image, collections=['nat']) tf.summary.scalar('learning_rate', learning_rate, collections=['nat']) nat_summaries = tf.summary.merge_all('nat') os.environ["CUDA_VISIBLE_DEVICES"] = "1" gpu_options = tf.GPUOptions(allow_growth=True) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: # initialize data augmentation if config.training.data_augmentation: cifar = cifar10_input.AugmentedCIFAR10Data(raw_cifar, sess) else: cifar = raw_cifar # Initialize the summary writer, global variables, and our time counter. summary_writer = tf.summary.FileWriter(model_dir, sess.graph) if eval_during_training: eval_summary_writer = tf.summary.FileWriter(eval_dir) sess.run(tf.global_variables_initializer()) training_time = 0.0 # Main training loop for ii in range(max_num_training_steps + 1): x_batch, y_batch = cifar.train_data.get_next_batch( batch_size, multiple_passes=True) noop_trans = np.zeros([len(x_batch), 3]) # Compute Adversarial Perturbations if adversarial_training: start = timer() x_batch_adv, adv_trans = attack.perturb(x_batch, y_batch, sess) end = timer() training_time += end - start else: x_batch_adv, adv_trans = x_batch, noop_trans nat_dict = { model.x_input: x_batch, model.y_input: y_batch, model.transform: noop_trans, model.is_training: False } adv_dict = { model.x_input: x_batch_adv, model.y_input: y_batch, model.transform: adv_trans, model.is_training: False } # Output to stdout if ii % num_output_steps == 0: nat_acc = sess.run(model.accuracy, feed_dict=nat_dict) adv_acc = sess.run(model.accuracy, feed_dict=adv_dict) print('Step {}: ({})'.format(ii, datetime.now())) print(' training nat accuracy {:.4}%'.format(nat_acc * 100)) print(' training adv accuracy {:.4}%'.format(adv_acc * 100)) if ii != 0: print(' {} examples per second'.format( num_output_steps * batch_size / training_time)) training_time = 0.0 # Tensorboard summaries if ii % num_summary_steps == 0: summary = sess.run(adv_summaries, feed_dict=adv_dict) summary_writer.add_summary(summary, global_step.eval(sess)) summary = sess.run(nat_summaries, feed_dict=nat_dict) summary_writer.add_summary(summary, global_step.eval(sess)) # Write a checkpoint if ii % num_checkpoint_steps == 0: saver.save(sess, os.path.join(model_dir, 'checkpoint'), global_step=global_step) if eval_during_training and ii % num_eval_steps == 0: evaluate(model, attack, sess, config, eval_summary_writer) # Actual training step start = timer() if adversarial_training: adv_dict[model.is_training] = True sess.run(train_step, feed_dict=adv_dict) else: nat_dict[model.is_training] = True sess.run(train_step, feed_dict=nat_dict) end = timer() training_time += end - start
parser.add_argument('-c', '--config', type=str, help='path to config file', default="config.json", required=False) parser.add_argument('--loop', help='continuously monitor model_dir' 'evaluating new ckpt', action="store_true") args = parser.parse_args() config_dict = utilities.get_config(args.config) config = utilities.config_to_namedtuple(config_dict) model = resnet.Model(config.model) model_dir = config.model.output_dir global_step = tf.contrib.framework.get_or_create_global_step() attack = SpatialAttack(model, config.attack) if args.loop: eval_dir = os.path.join(model_dir, 'eval') if not os.path.exists(eval_dir): os.makedirs(eval_dir) summary_writer = tf.summary.FileWriter(eval_dir) loop(model, attack, config, summary_writer) else: saver = tf.train.Saver()
config_dict_copy = copy.deepcopy(config_dict) out_dict = {} out_dict['hyperparameters'] = config_dict_copy config = utilities.config_to_namedtuple(config_dict) reg_model_path = args.reg_model_path # num_ids in model does not matter for eval num_ids = 64 model_family = config.model.model_family if model_family == "resnet": if config.attack.use_spatial and config.attack.spatial_method == 'fo': diffable = True else: diffable = False model = resnet.Model(config.model, num_ids, diffable) elif model_family == "vgg": if config.attack.use_spatial and config.attack.spatial_method == 'fo': diffable = True else: diffable = False model = vgg.Model(config.model, num_ids, diffable) global_step = tf.train.get_or_create_global_step() if args.linf_attack: attack_eval = SpatialAttack(model, config.attack, 'fo', 1, config.attack.spatial_limits, config.attack.epsilon, config.attack.step_size, config.attack.num_steps) else:
def train(config='configs/fannyconfig.json', save_root_path='/cluster/work/math/fanyang-broglil/CoreRepo', experiment_json_fname='experiments.json', local_json_dir_name='local_json_files', worstofk=None, attack_style=None, attack_limits=None, fo_epsilon=None, fo_step_size=None, fo_num_steps=None, lambda_core=None, num_ids=None, group_size=None, use_core=None, seed=None, save_in_local_json=True, this_repo=None): # reset default graph (needed for running locally with run_jobs_ray.py) tf.reset_default_graph() # get configs config_dict = utilities.get_config(config) config_dict_copy = copy.deepcopy(config_dict) config = utilities.config_to_namedtuple(config_dict) # seeding randomness if seed == None: seed = config.training.tf_random_seed else: config_dict_copy['training']['tf_random_seed'] = seed tf.set_random_seed(seed) np.random.seed(seed) # Setting up training parameters max_num_training_steps = config.training.max_num_training_steps step_size_schedule = config.training.step_size_schedule weight_decay = config.training.weight_decay momentum = config.training.momentum if group_size == None: group_size = config.training.group_size else: config_dict_copy['training']['group_size'] = int(group_size) if num_ids == None: num_ids = config.training.num_ids else: config_dict_copy['training']['num_ids'] = int(num_ids) if lambda_core == None: lambda_core = config.training.lambda_ else: config_dict_copy['training']['lambda_'] = float(lambda_core) if use_core == None: use_core = config.training.use_core else: config_dict_copy['training']['use_core'] = use_core batch_size = config.training.batch_size # number of groups with group size > 1 num_grouped_ids = batch_size - num_ids # number of unique ids needs to be larger than half the desired batch size # so that full batch can be filled up assert num_ids >= batch_size / group_size # currently, code is designed for groups of size 2 # assert batch_size % group_size == 0 adversarial_training = config.training.adversarial_training eval_during_training = config.training.eval_during_training if eval_during_training: num_eval_steps = config.training.num_eval_steps # Setting up output parameters num_output_steps = config.training.num_output_steps num_summary_steps = config.training.num_summary_steps num_checkpoint_steps = config.training.num_checkpoint_steps num_easyeval_steps = config.training.num_easyeval_steps # Setting up the data and the model data_path = config.data.data_path if config.data.dataset_name == "cifar-10": raw_iterator = cifar10_input.CIFAR10Data(data_path) elif config.data.dataset_name == "cifar-100": raw_iterator = cifar100_input.CIFAR100Data(data_path) elif config.data.dataset_name == "svhn": raw_iterator = svhn_input.SVHNData(data_path) else: raise ValueError("Unknown dataset name.") global_step = tf.train.get_or_create_global_step() model_family = config.model.model_family if model_family == "resnet": if config.attack.use_spatial and config.attack.spatial_method == 'fo': diffable = True else: diffable = False model = resnet.Model(config.model, num_ids, diffable) elif model_family == "vgg": if config.attack.use_spatial and config.attack.spatial_method == 'fo': # TODO: add differentiable transformer to vgg.py raise NotImplementedError model = vgg.Model(config.model, num_ids) # uncomment to get a list of trainable variables # model_vars = tf.trainable_variables() # Setting up the optimizer boundaries = [int(sss[0]) for sss in step_size_schedule] boundaries = boundaries[1:] values = [sss[1] for sss in step_size_schedule] learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32), boundaries, values) if use_core and lambda_core > 0: total_loss = (model.mean_xent + weight_decay * model.weight_decay_loss + lambda_core * model.core_loss) else: total_loss = model.mean_xent + weight_decay * model.weight_decay_loss optimizer = tf.train.MomentumOptimizer(learning_rate, momentum) train_step = optimizer.minimize(total_loss, global_step=global_step) # Set up adversary if worstofk == None: worstofk = config.attack.random_tries else: config_dict_copy['attack']['random_tries'] = worstofk if fo_epsilon == None: fo_epsilon = config.attack.epsilon else: config_dict_copy['attack']['epsilon'] = fo_epsilon if fo_step_size == None: fo_step_size = config.attack.step_size else: config_dict_copy['attack']['step_size'] = fo_step_size if fo_num_steps == None: fo_num_steps = config.attack.num_steps else: config_dict_copy['attack']['num_steps'] = fo_num_steps # @ Luzius: incorporate being able to choose multiple transformations if attack_style == None: attack_style = 'rotate' # Training attack # L-inf attack if use_spatial is False and use_linf is True # spatial attack if use_spatial is True and use_linf is False # spatial random attack if spatial_method is 'random' # spatial PGD attack if spatial_method is 'fo' attack = SpatialAttack(model, config.attack, config.attack.spatial_method, worstofk, attack_limits, fo_epsilon, fo_step_size, fo_num_steps) # Different eval attacks # Random attack # L-inf attack if use_spatial is False and use_linf is True # random (worst-of-1) spatial attack if use_spatial is True # and use_linf is False attack_eval_random = SpatialAttack(model, config.attack, 'random', 1, attack_limits, fo_epsilon, fo_step_size, fo_num_steps) # First order attack # L-inf attack if use_spatial is False and use_linf is True # first-order spatial attack if use_spatial is True and use_linf is False attack_eval_fo = SpatialAttack(model, config.attack, 'fo', 1, attack_limits, fo_epsilon, fo_step_size, fo_num_steps) # Grid attack # spatial attack if use_spatial is True and use_linf is False # not executed for L-inf attacks attack_eval_grid = SpatialAttack(model, config.attack, 'grid', None, attack_limits) # TODO(christina): add L-inf attack with random restarts # ------------------START EXPERIMENT ------------------------- # Initialize the Repo print("==> Creating repo..") # Create repo object if it wasn't passed, comment out if repo has issues if this_repo == None: this_repo = exprepo.ExperimentRepo( save_in_local_json=save_in_local_json, json_filename=experiment_json_fname, local_dir_name=local_json_dir_name, root_dir=save_root_path) # Create new experiment if this_repo != None: exp_id = this_repo.create_new_experiment(config.data.dataset_name, model_family, worstofk, attack_style, attack_limits, lambda_core, num_grouped_ids, group_size, config_dict_copy) # Setting up the Tensorboard and checkpoint outputs model_dir = '%s/logdir/%s' % (save_root_path, exp_id) # We add accuracy and xent twice so we can easily make three types of # comparisons in Tensorboard: # - train vs eval (for a single run) # - train of different runs # - eval of different runs saver = tf.train.Saver(max_to_keep=3) tf.summary.scalar('accuracy_nat_train', model.accuracy, collections=['nat']) tf.summary.scalar('accuracy_nat', model.accuracy, collections=['nat']) tf.summary.scalar('xent_nat_train', model.xent / batch_size, collections=['nat']) tf.summary.scalar('xent_nat', model.xent / batch_size, collections=['nat']) tf.summary.image('images_nat_train', model.x_image, collections=['nat']) tf.summary.scalar('learning_rate', learning_rate, collections=['nat']) nat_summaries = tf.summary.merge_all('nat') # data augmentation used if config.training.data_augmentation_core is True x_input_placeholder = tf.placeholder(tf.float32, shape=[None, 32, 32, 3]) flipped = tf.map_fn(lambda img: tf.image.random_flip_left_right(img), x_input_placeholder) with tf.Session() as sess: # initialize standard data augmentation if config.training.data_augmentation: if config.data.dataset_name == "cifar-10": data_iterator = cifar10_input.AugmentedCIFAR10Data( raw_iterator, sess) elif config.data.dataset_name == "cifar-100": data_iterator = cifar100_input.AugmentedCIFAR100Data( raw_iterator, sess) elif config.data.dataset_name == "svhn": data_iterator = svhn_input.AugmentedSVHNData( raw_iterator, sess) else: raise ValueError("Unknown dataset name.") else: data_iterator = raw_iterator eval_dict = { model.x_input: data_iterator.eval_data.xs, model.y_input: data_iterator.eval_data.ys, model.group: np.arange(0, batch_size, 1, dtype="int32"), model.transform: np.zeros([data_iterator.eval_data.n, 3]), model.is_training: False } # Initialize the summary writer, global variables, and our time counter. summary_writer = tf.summary.FileWriter(model_dir, sess.graph) # if eval_during_training: eval_dir = os.path.join(model_dir, 'eval') os.makedirs(eval_dir, exist_ok=True) eval_summary_writer = tf.summary.FileWriter(eval_dir) sess.run(tf.global_variables_initializer()) training_time = 0.0 run_time_without_eval = 0.0 run_time_adv_ex_creation = 0.0 run_time_train_step = 0.0 #################################### # Main training loop #################################### start_time = time.time() no_epochs_done = 0 # the same as epoch_count, need to merge start_epoch = timer() it_count = 0 epoch_count = 0 acc_sum = 0 for ii in range(max_num_training_steps + 1): # original batch x_batch, y_batch, epoch_done = data_iterator.train_data.get_next_batch( num_ids, multiple_passes=True) no_epochs_done += epoch_done # noop trans noop_trans = np.zeros([len(x_batch), 3]) # id_batch starts with IDs of original examples id_batch = np.arange(0, num_ids, 1, dtype="int32") if use_core: # first num_id examples of batch are natural x_batch_inp = x_batch y_batch_inp = y_batch trans_inp = noop_trans id_batch_inp = id_batch start = timer() for _ in range(group_size - 1): if config.training.data_augmentation_core: raise NotImplementedError # create rotated examples x_batch_adv_i, trans_adv_i = attack.perturb( x_batch, y_batch, sess) # construct new batches including rotated examples x_batch_inp = np.concatenate((x_batch_inp, x_batch_adv_i), axis=0) y_batch_inp = np.concatenate((y_batch_inp, y_batch), axis=0) trans_inp = np.concatenate((trans_inp, trans_adv_i), axis=0) id_batch_inp = np.concatenate((id_batch_inp, id_batch), axis=0) end = timer() training_time += end - start run_time_without_eval += end - start run_time_adv_ex_creation += end - start trans_adv = trans_inp[num_ids:, ...] id_batch_adv = id_batch_inp[num_ids:] y_batch_adv = y_batch_inp[num_ids:] x_batch_adv = x_batch_inp[num_ids:, ...] else: if adversarial_training: start = timer() x_batch_inp, trans_inp = attack.perturb( x_batch, y_batch, sess) end = timer() training_time += end - start run_time_without_eval += end - start run_time_adv_ex_creation += end - start else: x_batch_inp, trans_inp = x_batch, noop_trans # for adversarial training and plain training, the following # variables coincide y_batch_inp = y_batch y_batch_adv = y_batch trans_adv = trans_inp x_batch_adv = x_batch_inp id_batch_inp = id_batch id_batch_adv = id_batch # feed_dict for training step inp_dict = { model.x_input: x_batch_inp, model.y_input: y_batch_inp, model.group: id_batch_inp, model.transform: trans_inp, model.is_training: False } # separate natural and adversarially transformed examples for eval nat_dict = { model.x_input: x_batch, model.y_input: y_batch, model.group: id_batch, model.transform: noop_trans, model.is_training: False } adv_dict = { model.x_input: x_batch_adv, model.y_input: y_batch_adv, model.group: id_batch_adv, model.transform: trans_adv, model.is_training: False } ########### Outputting/saving weights and evaluations ########### acc_grid_te = -1.0 avg_xent_grid_te = -1.0 acc_fo_te = -1.0 avg_xent_fo_te = -1.0 saved_weights = 0 # Compute training accuracy on this minibatch nat_acc_tr = 100 * sess.run(model.accuracy, feed_dict=nat_dict) # Output to stdout if epoch_done: epoch_time = timer() - start_epoch # Average av_acc = acc_sum / it_count # ToDo: Log this to file as well # Training accuracy over epoch print('Epoch {}: ({})'.format(epoch_count, datetime.now())) print(' training natural accuracy {:.4}%'.format(av_acc)) print(' {:.4} seconds per epoch'.format(epoch_time)) # Accuracy on entire test set nat_acc_te = 100 * sess.run(model.accuracy, feed_dict=eval_dict) print( ' test set natural accuracy {:.4}%'.format(nat_acc_te)) # Set loss sum, it count back to zero acc_sum = nat_acc_tr epoch_done = 0 epoch_count += 1 start_epoch = timer() it_count = 1 else: it_count += 1 acc_sum += nat_acc_tr # Output to stdout if ii % num_output_steps == 0: # nat_acc_tr = 100 * sess.run(model.accuracy, feed_dict=nat_dict) adv_acc_tr = 100 * sess.run(model.accuracy, feed_dict=adv_dict) inp_acc_tr = 100 * sess.run(model.accuracy, feed_dict=inp_dict) # print('Step {}: ({})'.format(ii, datetime.now())) # print(' training nat accuracy {:.4}%'.format(nat_acc_tr)) # print(' training adv accuracy {:.4}%'.format(adv_acc_tr)) # print(' training inp accuracy {:.4}%'.format(inp_acc_tr)) if ii != 0: # print(' {} examples per second'.format( # num_output_steps * batch_size / training_time)) training_time = 0.0 # Tensorboard summaries and heavy checkpoints if ii % num_summary_steps == 0: summary = sess.run(nat_summaries, feed_dict=nat_dict) summary_writer.add_summary(summary, global_step.eval(sess)) # Write a checkpoint and eval if it's time if ii % num_checkpoint_steps == 0 or ii == max_num_training_steps: # Save checkpoint data (weights) saver.save(sess, os.path.join(model_dir, 'checkpoint'), global_step=global_step) saved_weights = 1 # Write evaluation meta data for checkpoint if ii % num_easyeval_steps == 0 or ii == max_num_training_steps: # Get training accuracies nat_acc_tr = 100 * sess.run(model.accuracy, feed_dict=nat_dict) adv_acc_tr = 100 * sess.run(model.accuracy, feed_dict=adv_dict) inp_acc_tr = 100 * sess.run(model.accuracy, feed_dict=inp_dict) # Evaluation on random and natural [ acc_nat_te, acc_rand_adv_te, avg_xent_nat_te, avg_xent_adv_te ] = evaluate(model, attack_eval_random, sess, config, 'random', data_path, None) # Evaluation on grid (only for spatial attacks) if ((eval_during_training and ii % num_eval_steps == 0 and ii > 0 and config.attack.use_spatial) or (eval_during_training and ii == max_num_training_steps and config.attack.use_spatial)): if config.attack.use_spatial and config.attack.spatial_method == 'fo': # Evaluation on first-order PDG attack (too expensive to # evaluate more frequently on whole dataset) [_, acc_fo_te, _, avg_xent_fo_te] = evaluate(model, attack_eval_fo, sess, config, 'fo', data_path, None) # Evaluation on grid [_, acc_grid_te, _, avg_xent_grid_te ] = evaluate(model, attack_eval_grid, sess, config, "grid", data_path, eval_summary_writer) chkpt_id = this_repo.create_training_checkpoint( exp_id, training_step=ii, epoch=no_epochs_done, train_acc_nat=nat_acc_tr, train_acc_adv=adv_acc_tr, train_acc_inp=inp_acc_tr, test_acc_nat=acc_nat_te, test_acc_adv=acc_rand_adv_te, test_acc_fo=acc_fo_te, test_acc_grid=acc_grid_te, test_loss_nat=avg_xent_nat_te, test_loss_adv=avg_xent_adv_te, test_loss_fo=avg_xent_fo_te, test_loss_grid=avg_xent_grid_te) if saved_weights == 0: # Save checkpoint data (weights) saver.save( sess, os.path.join(model_dir, '{}_checkpoint'.format(chkpt_id))) # Actual training step start = timer() inp_dict[model.is_training] = True sess.run(train_step, feed_dict=inp_dict) end = timer() training_time += end - start run_time_without_eval += end - start run_time_train_step += end - start runtime = time.time() - start_time this_repo.mark_experiment_as_completed( exp_id, train_acc_nat=nat_acc_tr, train_acc_adv=adv_acc_tr, train_acc_inp=inp_acc_tr, test_acc_nat=acc_nat_te, test_acc_adv=acc_rand_adv_te, test_acc_fo=acc_fo_te, test_acc_grid=acc_grid_te, runtime=runtime, runtime_wo_eval=run_time_without_eval, runtime_train_step=run_time_train_step, runtime_adv_ex_creation=run_time_adv_ex_creation) return 0
def train(config): # seeding randomness tf.set_random_seed(config.training.tf_random_seed) np.random.seed(config.training.np_random_seed) # Setting up training parameters max_num_training_steps = config.training.max_num_training_steps step_size_schedule = config.training.step_size_schedule weight_decay = config.training.weight_decay momentum = config.training.momentum batch_size = config.training.batch_size eval_during_training = config.training.eval_during_training num_clean_examples = config.training.num_examples if eval_during_training: num_eval_steps = config.training.num_eval_steps # Setting up output parameters num_output_steps = config.training.num_output_steps num_summary_steps = config.training.num_summary_steps num_checkpoint_steps = config.training.num_checkpoint_steps # Setting up the data and the model dataset = dataset_input.CIFAR10Data(config, seed=config.training.np_random_seed) print('Num Poisoned Left: {}'.format(dataset.num_poisoned_left)) print('Poison Position: {}'.format(config.data.position)) print('Poison Color: {}'.format(config.data.color)) num_training_examples = len(dataset.train_data.xs) global_step = tf.contrib.framework.get_or_create_global_step() model = resnet.Model(config.model) # uncomment to get a list of trainable variables model_vars = tf.trainable_variables() slim.model_analyzer.analyze_vars(model_vars, print_info=True) # Setting up the optimizer boundaries = [int(sss[0]) for sss in step_size_schedule] boundaries = boundaries[1:] values = [sss[1] for sss in step_size_schedule] learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32), boundaries, values) total_loss = model.mean_xent + weight_decay * model.weight_decay_loss optimizer = tf.train.MomentumOptimizer(learning_rate, momentum) train_step = optimizer.minimize(total_loss, global_step=global_step) # Setting up the Tensorboard and checkpoint outputs model_dir = config.model.output_dir if eval_during_training: eval_dir = os.path.join(model_dir, 'eval') if not os.path.exists(eval_dir): os.makedirs(eval_dir) # We add accuracy and xent twice so we can easily make three types of # comparisons in Tensorboard: # - train vs eval (for a single run) # - train of different runs # - eval of different runs saver = tf.train.Saver(max_to_keep=3) tf.summary.scalar('accuracy_nat_train', model.accuracy, collections=['nat']) tf.summary.scalar('accuracy_nat', model.accuracy, collections=['nat']) tf.summary.scalar('xent_nat_train', model.xent / batch_size, collections=['nat']) tf.summary.scalar('xent_nat', model.xent / batch_size, collections=['nat']) tf.summary.image('images_nat_train', model.train_xs, collections=['nat']) tf.summary.scalar('learning_rate', learning_rate, collections=['nat']) nat_summaries = tf.summary.merge_all('nat') with tf.Session() as sess: print('Dataset Size: ', len(dataset.train_data.xs)) # Initialize the summary writer, global variables, and our time counter. summary_writer = tf.summary.FileWriter(model_dir, sess.graph) if eval_during_training: eval_summary_writer = tf.summary.FileWriter(eval_dir) sess.run(tf.global_variables_initializer()) training_time = 0.0 # Main training loop for ii in range(max_num_training_steps + 1): x_batch, y_batch = dataset.train_data.get_next_batch( batch_size, multiple_passes=True) nat_dict = { model.x_input: x_batch, model.y_input: y_batch, model.is_training: False } # Output to stdout if ii % num_output_steps == 0: nat_acc = sess.run(model.accuracy, feed_dict=nat_dict) print('Step {}: ({})'.format(ii, datetime.now())) print(' training nat accuracy {:.4}%'.format(nat_acc * 100)) if ii != 0: print(' {} examples per second'.format( num_output_steps * batch_size / training_time)) training_time = 0.0 # Tensorboard summaries if ii % num_summary_steps == 0: summary = sess.run(nat_summaries, feed_dict=nat_dict) summary_writer.add_summary(summary, global_step.eval(sess)) # Write a checkpoint if ii % num_checkpoint_steps == 0: saver.save(sess, os.path.join(model_dir, 'checkpoint'), global_step=global_step) if eval_during_training and ii % num_eval_steps == 0: evaluate(model, sess, config, eval_summary_writer) # Actual training step start = timer() nat_dict[model.is_training] = True sess.run(train_step, feed_dict=nat_dict) end = timer() training_time += end - start
def train(config): # seeding randomness tf.set_random_seed(config.training.tf_random_seed) np.random.seed(config.training.np_random_seed) # Setting up training parameters max_num_training_steps = config.training.max_num_training_steps step_size_schedule = config.training.step_size_schedule weight_decay = config.training.weight_decay momentum = config.training.momentum batch_size = config.training.batch_size group_size = config.training.group_size adversarial_training = config.training.adversarial_training eval_during_training = config.training.eval_during_training if eval_during_training: num_eval_steps = config.training.num_eval_steps # Setting up output parameters num_output_steps = config.training.num_output_steps num_summary_steps = config.training.num_summary_steps num_checkpoint_steps = config.training.num_checkpoint_steps #adapting batch size batch_size_group = batch_size * config.training.group_size # Setting up the data and the model data_path = config.data.data_path raw_cifar = cifar10_input.CIFAR10Data(data_path) global_step = tf.train.get_or_create_global_step() model = resnet.Model(config.model) # uncomment to get a list of trainable variables # model_vars = tf.trainable_variables() # slim.model_analyzer.analyze_vars(model_vars, print_info=True) # Setting up the optimizer boundaries = [int(sss[0]) for sss in step_size_schedule] boundaries = boundaries[1:] values = [sss[1] for sss in step_size_schedule] learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32), boundaries, values) total_loss = model.mean_xent + weight_decay * model.weight_decay_loss + model.core_loss optimizer = tf.train.MomentumOptimizer(learning_rate, momentum) train_step = optimizer.minimize(total_loss, global_step=global_step) # Set up adversary attack = SpatialAttack(model, config.attack) attack_eval_random = SpatialAttack(model, config.eval_attack_random) attack_eval_grid = SpatialAttack(model, config.eval_attack_grid) # Setting up the Tensorboard and checkpoint outputs model_dir = config.model.output_dir if eval_during_training: eval_dir = os.path.join(model_dir, 'eval') if not os.path.exists(eval_dir): os.makedirs(eval_dir) # We add accuracy and xent twice so we can easily make three types of # comparisons in Tensorboard: # - train vs eval (for a single run) # - train of different runs # - eval of different runs saver = tf.train.Saver(max_to_keep=3) tf.summary.scalar('accuracy_adv_train', model.accuracy, collections=['adv']) tf.summary.scalar('accuracy_adv', model.accuracy, collections=['adv']) tf.summary.scalar('xent_adv_train', model.xent / batch_size_group, collections=['adv']) tf.summary.scalar('xent_adv', model.xent / batch_size_group, collections=['adv']) tf.summary.image('images_adv_train', model.x_image, collections=['adv']) adv_summaries = tf.summary.merge_all('adv') tf.summary.scalar('accuracy_nat_train', model.accuracy, collections=['nat']) tf.summary.scalar('accuracy_nat', model.accuracy, collections=['nat']) tf.summary.scalar('xent_nat_train', model.xent / batch_size_group, collections=['nat']) tf.summary.scalar('xent_nat', model.xent / batch_size_group, collections=['nat']) tf.summary.image('images_nat_train', model.x_image, collections=['nat']) tf.summary.scalar('learning_rate', learning_rate, collections=['nat']) nat_summaries = tf.summary.merge_all('nat') #dataAugmentation x_input_placeholder = tf.placeholder(tf.float32, shape=[None, 32, 32, 3]) flipped = tf.map_fn(lambda img: tf.image.random_flip_left_right(img), x_input_placeholder) with tf.Session() as sess: # initialize data augmentation if config.training.data_augmentation: cifar = cifar10_input.AugmentedCIFAR10Data(raw_cifar, sess) else: cifar = raw_cifar # Initialize the summary writer, global variables, and our time counter. summary_writer = tf.summary.FileWriter(model_dir, sess.graph) if eval_during_training: eval_summary_writer = tf.summary.FileWriter(eval_dir) sess.run(tf.global_variables_initializer()) training_time = 0.0 training_time_total = 0.0 adv_time = 0.0 eval_time = 0.0 core_time = 0.0 # Main training loop for ii in range(max_num_training_steps + 1): x_batch, y_batch = cifar.train_data.get_next_batch( batch_size, multiple_passes=True) noop_trans = np.zeros([len(x_batch), 3]) # Compute Adversarial Perturbations if adversarial_training: start = timer() x_batch_adv, adv_trans = attack.perturb(x_batch, y_batch, sess) end = timer() adv_time += end - start else: x_batch_adv, adv_trans = x_batch, noop_trans #Create rotatated examples start = timer() x_batch_nat = x_batch y_batch_nat = y_batch id_batch = np.arange(0, batch_size, 1, dtype="int32") ids = np.arange(0, batch_size, 1, dtype="int32") for i in range(config.training.group_size): if config.training.data_augmentation_core: x_batch_core = sess.run( flipped, feed_dict={x_input_placeholder: x_batch}) else: x_batch_core = x_batch x_batch_group, trans_group = attack.perturb( x_batch_core, y_batch, sess) #construct new batches including rotateted examples x_batch_adv = np.concatenate((x_batch_adv, x_batch_group), axis=0) x_batch_nat = np.concatenate((x_batch_nat, x_batch_group), axis=0) y_batch_nat = np.concatenate((y_batch_nat, y_batch), axis=0) adv_trans = np.concatenate((adv_trans, trans_group), axis=0) noop_trans = np.concatenate((noop_trans, trans_group), axis=0) id_batch = np.concatenate((id_batch, ids), axis=0) end = timer() core_time += end - start nat_dict = { model.x_input: x_batch_nat, model.y_input: y_batch_nat, model.group: id_batch, model.transform: noop_trans, model.is_training: False } adv_dict = { model.x_input: x_batch_adv, model.y_input: y_batch_nat, model.group: id_batch, model.transform: adv_trans, model.is_training: False } # Output to stdout if ii % num_output_steps == 0: nat_acc = sess.run(model.accuracy, feed_dict=nat_dict) adv_acc = sess.run(model.accuracy, feed_dict=adv_dict) print('Step {}: ({})'.format(ii, datetime.now())) print(' training nat accuracy {:.4}%'.format(nat_acc * 100)) print(' training adv accuracy {:.4}%'.format(adv_acc * 100)) if ii != 0: print(' {} examples per second'.format( num_output_steps * batch_size_group / training_time)) training_time = 0.0 # Tensorboard summaries if ii % num_summary_steps == 0: summary = sess.run(adv_summaries, feed_dict=adv_dict) summary_writer.add_summary(summary, global_step.eval(sess)) summary = sess.run(nat_summaries, feed_dict=nat_dict) summary_writer.add_summary(summary, global_step.eval(sess)) # Write a checkpoint if ii % num_checkpoint_steps == 0: saver.save(sess, os.path.join(model_dir, 'checkpoint'), global_step=global_step) if eval_during_training and ii % num_eval_steps == 0: start = timer() evaluate(model, attack_eval_random, sess, config, "random", eval_summary_writer) evaluate(model, attack_eval_grid, sess, config, "grid", eval_summary_writer) end = timer() eval_time += end - start print(' {}seconds total training time'.format( training_time_total)) print(' {}seconds total adv. example time'.format(adv_time)) print( ' {}seconds total core example time'.format(core_time)) print(' {}seconds total evalutation time'.format(eval_time)) # Actual training step start = timer() if adversarial_training: adv_dict[model.is_training] = True sess.run(train_step, feed_dict=adv_dict) else: nat_dict[model.is_training] = True sess.run(train_step, feed_dict=nat_dict) end = timer() training_time += end - start training_time_total += end - start
# ap = argparse.ArgumentParser() # ap.add_argument("-train", "--train", type=bool, default = True, # help="Training set") # ap.add_argument("-test", "--test", type=bool, default=False, # help="Testing set") # args = vars(ap.parse_args()) # # grab the number of GPUs and store it in a conveience variable # train = args["train"] # test = args["test"] train = args.train test = args.test pred = args.predict input_img = Input((im_height, im_width, 1), name='img') model = import resnet.Model(input_img) # save_path=cwd + '/weights/weight.h5' # loss = "binary_crossentropy" # Train if train: train_model(model, save_path=save_path, loss=loss) # Test if test: weight_path = save_path threshold = 0.5 test_model(weight_path, threshold) if pred:
def __init__(self, sess, config, filepath, batch_size, regu, learning_rate = 0.1, binary_search_steps = 1, max_iterations = 101, initial_const = 1): image_size, num_channels, num_labels = 32, 3, 10 self.sess = sess self.LEARNING_RATE = learning_rate self.MAX_ITERATIONS = max_iterations self.BINARY_SEARCH_STEPS = binary_search_steps #self.ABORT_EARLY = abort_early self.initial_const = initial_const self.batch_size = batch_size shape = (batch_size,image_size,image_size,num_channels) shape_pert = (batch_size,image_size,image_size,num_channels) # these are variables to be more efficient in sending data to tf self.timg = tf.Variable(np.zeros(shape), dtype=tf.float32) self.tlab = tf.Variable(np.zeros((batch_size,num_labels)), dtype=tf.float32) self.const = tf.Variable(np.zeros(batch_size), dtype=tf.float32) self.modifier = tf.Variable(np.zeros(shape_pert,dtype=np.float32)) self.det = tf.Variable(np.ones(shape_pert,dtype=np.float32), constraint = lambda x:tf.clip_by_value(x, 0, 255)) self.newimg = self.modifier*self.det + self.timg*(1-self.modifier) self.assign_tlab = tf.placeholder(tf.float32, (batch_size,num_labels), name="tlab") self.assign_timg = tf.placeholder(tf.float32, shape, name="timag") self.assign_const = tf.placeholder(tf.float32, [batch_size], name="tconst") # the variable we're going to optimize over global_step = tf.contrib.framework.get_or_create_global_step() #with tf.variable_scope(reuse=tf.AUTO_REUSE): model = resnet.Model(config.model, self.newimg) # Setting up the Tensorboard and checkpoint outputs model_dir = filepath #saver = tf.train.Saver() var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) var_list = var_list[5:] #saver = tf.train.Saver(var_list=[v for v in all_variables if not in v.name]) saver = tf.train.Saver(max_to_keep=3, var_list=var_list) latest_checkpoint = tf.train.latest_checkpoint(model_dir) if latest_checkpoint is not None: saver.restore(sess, latest_checkpoint) print('Restoring last saved checkpoint: ', latest_checkpoint) else: print('Check model directory') exit() self.output=model.pre_softmax # compute the probability of the label class versus the maximum other real = tf.reduce_sum((self.tlab)*self.output,1) other = tf.reduce_max((1-self.tlab)*self.output - (self.tlab*10000),1) loss1 = tf.maximum(-15.0, other - real) self.loss1 = tf.reduce_sum(loss1) if regu == "l2": self.loss2 = tf.reduce_sum(tf.square(self.modifier)) else: self.loss2 = tf.reduce_sum(tf.abs(self.modifier)) self.loss = self.loss1 self.modifier_ST = tf.clip_by_value(tf.sign(self.modifier)*tf.maximum(tf.abs(self.modifier) - self.LEARNING_RATE/self.initial_const, 0), clip_value_min=0, clip_value_max=1) self.assign_ST = tf.assign(self.modifier, self.modifier_ST) # Setup the adam optimizer and keep track of variables we're creating start_vars = set(x.name for x in tf.global_variables()) optimizer = tf.train.AdamOptimizer(self.LEARNING_RATE) self.train1= optimizer.minimize(self.loss, var_list=[self.modifier]) self.train2 = optimizer.minimize(self.loss, var_list=[self.det]) end_vars = tf.global_variables() new_vars = [x for x in end_vars if x.name not in start_vars] # these are the variables to initialize when we run self.setup = [] self.setup.append(self.timg.assign(self.assign_timg)) self.setup.append(self.tlab.assign(self.assign_tlab)) self.setup.append(self.const.assign(self.assign_const)) #self.init = tf.variables_initializer(var_list=[self.modifier]+new_vars) self.init = tf.variables_initializer(var_list=[self.modifier]+[self.det]+new_vars)
def compute_corr(config): # seeding randomness tf.set_random_seed(config.training.tf_random_seed) np.random.seed(config.training.np_random_seed) # Setting up the data and the model poison_eps = config.data.poison_eps clean_label = config.data.clean_label target_label = config.data.target_label dataset = dataset_input.CIFAR10Data(config, seed=config.training.np_random_seed) num_poisoned_left = dataset.num_poisoned_left print('Num poisoned left: ', num_poisoned_left) num_training_examples = len(dataset.train_data.xs) global_step = tf.contrib.framework.get_or_create_global_step() model = resnet.Model(config.model) # Setting up the Tensorboard and checkpoint outputs model_dir = config.model.output_dir saver = tf.train.Saver(max_to_keep=3) with tf.Session() as sess: # initialize data augmentation print('Dataset Size: ', len(dataset.train_data.xs)) sess.run(tf.global_variables_initializer()) latest_checkpoint = tf.train.latest_checkpoint(model_dir) if latest_checkpoint is not None: saver.restore(sess, latest_checkpoint) print('Restoring last saved checkpoint: ', latest_checkpoint) else: print('Check model directory') exit() lbl = target_label cur_indices = np.where(dataset.train_data.ys==lbl)[0] cur_examples = len(cur_indices) print('Label, num ex: ', lbl, cur_examples) cur_op = model.representation for iex in trange(cur_examples): cur_im = cur_indices[iex] x_batch = dataset.train_data.xs[cur_im:cur_im+1,:] y_batch = dataset.train_data.ys[cur_im:cur_im+1] dict_nat = {model.x_input: x_batch, model.y_input: y_batch, model.is_training: False} batch_grads = sess.run(cur_op, feed_dict=dict_nat) if iex==0: clean_cov = np.zeros(shape=(cur_examples-num_poisoned_left, len(batch_grads))) full_cov = np.zeros(shape=(cur_examples, len(batch_grads))) if iex < (cur_examples-num_poisoned_left): clean_cov[iex]=batch_grads full_cov[iex] = batch_grads #np.save(corr_dir+str(lbl)+'_full_cov.npy', full_cov) total_p = config.data.percentile clean_mean = np.mean(clean_cov, axis=0, keepdims=True) full_mean = np.mean(full_cov, axis=0, keepdims=True) print('Norm of Difference in Mean: ', np.linalg.norm(clean_mean-full_mean)) clean_centered_cov = clean_cov - clean_mean s_clean = np.linalg.svd(clean_centered_cov, full_matrices=False, compute_uv=False) print('Top 7 Clean SVs: ', s_clean[0:7]) centered_cov = full_cov - full_mean u,s,v = np.linalg.svd(centered_cov, full_matrices=False) print('Top 7 Singular Values: ', s[0:7]) eigs = v[0:1] p = total_p corrs = np.matmul(eigs, np.transpose(full_cov)) #shape num_top, num_active_indices scores = np.linalg.norm(corrs, axis=0) #shape num_active_indices np.save(os.path.join(model_dir, 'scores.npy'), scores) print('Length Scores: ', len(scores)) p_score = np.percentile(scores, p) top_scores = np.where(scores>p_score)[0] print(top_scores) num_bad_removed = np.count_nonzero(top_scores>=(len(scores)-num_poisoned_left)) print('Num Bad Removed: ', num_bad_removed) print('Num Good Rmoved: ', len(top_scores)-num_bad_removed) num_poisoned_after = num_poisoned_left - num_bad_removed removed_inds = np.copy(top_scores) removed_inds_file = os.path.join(model_dir, 'removed_inds.npy') np.save(removed_inds_file, cur_indices[removed_inds]) print('Num Poisoned Left: ', num_poisoned_after) if os.path.exists('job_result.json'): with open('job_result.json') as result_file: result = json.load(result_file) result['num_poisoned_left'] = '{}'.format(num_poisoned_after) else: result = {'num_poisoned_left': '{}'.format(num_poisoned_after)} with open('job_result.json', 'w') as result_file: json.dump(result, result_file, sort_keys=True, indent=4)
def train(config='configs/cifar10_config_stn.json', save_root_path='/cluster/work/math/fanyang-broglil/CoreRepo', worstofk=None, attack_style=None, attack_limits=None, lambda_core=None, num_grouped_ids=None, num_ids = None, group_size=None, use_core=None, seed=None, this_repo=None): config_dict = utilities.get_config(config) config_dict_copy = copy.deepcopy(config_dict) # model_dir = config_dict['model']['output_dir'] # if not os.path.exists(model_dir): # os.makedirs(model_dir) # # keep the configuration file with the model for reproducibility # with open(os.path.join(model_dir, 'config.json'), 'w') as f: # json.dump(config_dict, f, sort_keys=True, indent=4) config = utilities.config_to_namedtuple(config_dict) # seeding randomness if seed == None: seed = config.training.tf_random_seed else: config_dict_copy['training']['tf_random_seed'] = seed tf.set_random_seed(seed) np.random.seed(seed) # Setting up training parameters max_num_epochs = config.training.max_num_epochs step_size_schedule = config.training.step_size_schedule weight_decay = config.training.weight_decay momentum = config.training.momentum num_ids = config.training.num_ids # number of IDs per minibatch if group_size == None: group_size = config.training.group_size else: config_dict_copy['training']['group_size'] = group_size if num_grouped_ids == None: num_grouped_ids = config.training.num_grouped_ids else: config_dict_copy['training']['num_grouped_ids'] = num_grouped_ids if num_ids == None: num_ids = config.training.num_ids else: config_dict_copy['training']['num_ids'] = num_ids if lambda_core == None: lambda_core = config.training.lambda_ else: config_dict_copy['training']['lambda_'] = lambda_core if use_core == None: use_core = config.training.use_core else: config_dict_copy['training']['use_core'] = use_core adversarial_training = config.training.adversarial_training eval_during_training = config.training.eval_during_training if eval_during_training: num_eval_steps = config.training.num_eval_steps # Setting up output parameters num_summary_steps = config.training.num_summary_steps num_checkpoint_steps = config.training.num_checkpoint_steps num_easyeval_steps = config.training.num_easyeval_steps # mini batch size per iteration # ToDo: need to make this support variable number of num_grouped_ids batch_size = num_ids + num_grouped_ids # Setting up model and loss model_family = config.model.model_family with_transformer = config.model.transformer translation_model =config.model.translation_model if model_family == "resnet": if with_transformer == True: if translation_model == "fc": model = stn_resnet.Model(config.model) print("Using stn_resnet") if translation_model == "conv": model = stn_resnet_conv.Model(config.model) print("Using stn_resnet_conv") else: model = resnet.Model(config.model) else: print("Model family does not exist") exit() if use_core: total_loss = model.mean_xent + weight_decay * model.weight_decay_loss + lambda_core * model.core_loss2 else: total_loss = model.mean_xent + weight_decay * model.weight_decay_loss # Setting up the data and the model data_path = config.data.data_path if config.data.dataset_name == "cifar-10": raw_cifar = cifar10_input.CIFAR10Data(data_path) elif config.data.dataset_name == "cifar-100": raw_cifar = cifar100_input.CIFAR100Data(data_path) else: raise ValueError("Unknown dataset name.") # uncomment to get a list of trainable variables # model_vars = tf.trainable_variables() # slim.model_analyzer.analyze_vars(model_vars, print_info=True) # Setting up the optimizer boundaries = [int(sss[0]) for sss in step_size_schedule] boundaries = boundaries[1:] values = [sss[1] for sss in step_size_schedule] global_step = tf.train.get_or_create_global_step() learning_rate = tf.train.piecewise_constant( tf.cast(global_step, tf.int32), boundaries, values) optimizer = tf.train.MomentumOptimizer(learning_rate, momentum) #optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, # name="Adam") train_step = optimizer.minimize( total_loss, global_step=global_step) # Set up adversary if worstofk == None: worstofk = config.attack.random_tries else: config_dict_copy['attack']['random_tries'] = worstofk # @ Luzius: incorporate being able to choose multiple transformations if attack_style == None: attack_style = 'rotate' # Training attack attack = SpatialAttack(model, config.attack, 'random', worstofk, attack_limits) # Different eval attacks # Same attack as worstofk # @ Luzius: currently the names are not clear/consistent since I wasn't sure if we actually want random or not since you originally had your attack like that but I feel like it should rather be worstofk? # attack_eval_adv = SpatialAttack(model, config.attack, 'random', worstofk, attack_limits) attack_eval_random = SpatialAttack(model, config.attack, 'random', 1, attack_limits) # Grid attack attack_eval_grid = SpatialAttack(model, config.attack, 'grid', None, attack_limits) # ------------------START EXPERIMENT ------------------------- # Initialize the Repo print("==> Creating repo..") # Create repo object if it wasn't passed, comment out if repo has issues if this_repo == None: this_repo = exprepo.ExperimentRepo(root_dir=save_root_path) # Create new experiment if this_repo != None: exp_id = this_repo.create_new_experiment('cifar-10', model_family, worstofk, attack_style, attack_limits, lambda_core, num_grouped_ids, group_size, config_dict_copy) # Setting up the Tensorboard and checkpoint outputs model_dir = '%s/logdir/%s' % (save_root_path, exp_id) os.makedirs(model_dir, exist_ok=True) # We add accuracy and xent twice so we can easily make three types of # comparisons in Tensorboard: # - train vs eval (for a single run) # - train of different runs # - eval of different runs saver = tf.train.Saver(max_to_keep=3) tf.summary.scalar('accuracy_nat_train', model.accuracy, collections=['nat']) tf.summary.scalar('accuracy_nat', model.accuracy, collections = ['nat']) tf.summary.scalar('xent_nat_train', model.xent / batch_size, collections=['nat']) tf.summary.scalar('xent_nat', model.xent / batch_size, collections=['nat']) tf.summary.image('images_nat_train', model.x_image, collections=['nat']) tf.summary.scalar('learning_rate', learning_rate, collections=['nat']) tf.summary.scalar('regression_loss', model.reg_loss, collections=['nat']) nat_summaries = tf.summary.merge_all('nat') #dataAugmentation x_input_placeholder = tf.placeholder(tf.float32, shape=[None, 32, 32, 3]) flipped = tf.map_fn(lambda img: tf.image.random_flip_left_right(img), x_input_placeholder) tot_samp = raw_cifar.train_data.n max_num_iterations = int(np.floor((tot_samp/num_ids)*max_num_epochs)) print("Total # of samples is: %d; This exp. will run %d iterations" % (tot_samp, max_num_iterations)) # Compute the (epoch) gaps between summary, worstof1eval, checkpoints should happen summary_gap = int(np.floor(max_num_epochs/num_summary_steps)) easyeval_gap = int(np.floor(max_num_epochs/num_easyeval_steps)) checkpoint_gap = int(np.floor(max_num_epochs/num_checkpoint_steps)) with tf.Session() as sess: # initialize data augmentation if config.training.data_augmentation: if config.data.dataset_name == "cifar-10": cifar = cifar10_input.AugmentedCIFAR10Data(raw_cifar, sess) elif config.data.dataset_name == "cifar-100": cifar = cifar100_input.AugmentedCIFAR100Data(raw_cifar, sess) else: raise ValueError("Unknown dataset name.") else: cifar = raw_cifar cifar_eval_dict = {model.x_input: cifar.eval_data.xs, model.y_input: cifar.eval_data.ys, model.group: np.arange(0, batch_size, 1, dtype="int32"), model.transform: np.zeros([cifar.eval_data.n, 3]), model.is_training: False} # Initialize the summary writer, global variables, and our time counter. summary_writer = tf.summary.FileWriter(model_dir, sess.graph) #if eval_during_training: eval_dir = os.path.join(model_dir, 'eval') os.makedirs(eval_dir, exist_ok=True) eval_summary_writer = tf.summary.FileWriter(eval_dir) sess.run(tf.global_variables_initializer()) training_time = 0.0 #################################### # Main training loop #################################### # Initialize cache variables start_time = time.time() start_epoch = timer() it_count = 0 epoch_count = 0 acc_sum = 0 it_summary = 0 it_easyeval = 0 it_ckpt = 0 adv_time = 0 train_time = 0 for ii in range(max_num_iterations+1): x_batch, y_batch, epoch_done = cifar.train_data.get_next_batch(num_ids, multiple_passes=True) noop_trans = np.zeros([len(x_batch), 3]) x_batch_nat = x_batch y_batch_nat = y_batch id_batch = np.arange(0, num_ids, 1, dtype="int32") if use_core: # Create rotated examples start = timer() ids = np.arange(0,num_grouped_ids,1,dtype="int32") for i in range(config.training.group_size): if config.training.data_augmentation_core: x_batch_core = sess.run(flipped,feed_dict={x_input_placeholder: x_batch[0:num_grouped_ids,:,:,:]}) else: x_batch_core = x_batch[0:num_grouped_ids,:,:,:] x_batch_group, trans_group = attack.perturb(x_batch_core, y_batch[0:num_grouped_ids], sess) #construct new batches including rotated examples x_batch_nat = np.concatenate((x_batch_nat, x_batch_group), axis=0) y_batch_nat = np.concatenate((y_batch_nat, y_batch), axis=0) noop_trans = np.concatenate((noop_trans, trans_group), axis=0) id_batch = np.concatenate((id_batch, ids), axis=0) end = timer() training_time += end - start adv_time += end - start else: if adversarial_training: start = timer() x_batch_nat, noop_trans = attack.perturb(x_batch, y_batch, sess) end = timer() adv_time += end - start else: x_batch_nat, noop_trans = x_batch, noop_trans nat_dict = {model.x_input: x_batch_nat, model.y_input: y_batch_nat, model.group: id_batch, model.transform: noop_trans, model.is_training: False} ################# Outputting/saving weights and evaluations ############### nat_acc = -1.0 acc_grid = -1.0 avg_xent_grid = -1.0 saved_weights = 0 # Compute training accuracy on this minibatch train_nat_acc = sess.run(model.accuracy, feed_dict=nat_dict) # Output to stdout if epoch_done: epoch_time = timer() - start_epoch # Average av_acc = acc_sum/it_count # ToDo: Log this to file as well # Training accuracy over epoch print('Epoch {}: ({})'.format(epoch_count, datetime.now())) print(' training natural accuracy {:.4}%'.format(av_acc * 100)) print(' {:.4} seconds per epoch'.format(epoch_time)) # Accuracy on entire test set test_nat_acc = sess.run(model.accuracy, feed_dict=cifar_eval_dict) print(' test set natural accuracy {:.4}%'.format(test_nat_acc * 100)) # print(' {:.4} seconds for test evaluation'.format(test_time)) print("example TIME") print(adv_time) print("train TIME") print(train_time) ########### Things to do every xxx epochs ############# # Check if worstof1 eval should be run if it_summary == summary_gap - 1 or epoch_count == max_num_epochs - 1: summary = sess.run(nat_summaries, feed_dict=nat_dict) summary_writer.add_summary(summary, global_step.eval(sess)) it_summary = 0 else: it_summary += 1 if it_easyeval == easyeval_gap - 1 or epoch_count == max_num_epochs - 1: # Evaluation on adv and natural [acc_nat, acc_adv, avg_xent_nat, avg_xent_adv] = evaluate(model, attack_eval_random, sess, config, "random", data_path, None) # Save in checkpoint chkpt_id = this_repo.create_training_checkpoint( exp_id, training_step=ii, epoch=epoch_count, train_acc_nat=nat_acc, test_acc_adv=acc_adv, test_acc_nat=acc_nat, test_loss_adv=avg_xent_adv, test_loss_nat=avg_xent_nat) it_easyeval = 0 else: it_easyeval += 1 startt = timer() if it_ckpt == checkpoint_gap - 1 or epoch_count == max_num_epochs - 1: # Create checkpoint id if non-existent if not chkpt_id : chkpt_id = this_repo.create_training_checkpoint( exp_id, training_step=ii, epoch=epoch_count, train_acc_nat=train_nat_acc, test_acc_nat=test_nat_acc) # Save checkpoint data (weights) saver.save(sess, os.path.join(model_dir, '{}_checkpoint'.format(chkpt_id))) print(' chkpt saving took {:.4}s '.format(timer()-startt)) it_ckpt = 0 else: it_ckpt += 1 # Set loss sum, it count back to zero acc_sum = train_nat_acc epoch_done = 0 epoch_count += 1 start_epoch = timer() it_count = 1 else: it_count += 1 acc_sum += train_nat_acc # Actual training step start = timer() nat_dict[model.is_training] = True sess.run(train_step, feed_dict=nat_dict) training_time += timer() - start train_time += timer() - start runtime = time.time() - start_time # Do all evaluations in the last step - on grid [_, acc_grid, _, avg_xent_grid] = evaluate(model, attack_eval_grid, sess, config, "grid", data_path, eval_summary_writer) this_repo.mark_experiment_as_completed( exp_id, train_acc_nat=nat_acc, test_acc_adv=acc_adv, test_acc_nat=acc_nat, test_acc_grid=acc_grid, runtime=runtime) return 0