def Eval(mnist_data_file, network_parameters, num_testing_images, randomize, load_path, save_mistakes=False): """Evaluate MNIST for a number of steps. Args: mnist_data_file: Path of a file containing the MNIST images to process. network_parameters: parameters for defining and training the network. num_testing_images: the number of images we will evaluate on. randomize: if false, randomize; otherwise, read the testing images sequentially. load_path: path where to load trained parameters from. save_mistakes: save the mistakes if True. Returns: The evaluation accuracy as a float. """ batch_size = 100 # Like for training, we need a session for executing the TensorFlow graph. with tf.Graph().as_default(), tf.Session() as sess: # Create the basic Mnist model. images, labels = MnistInput(mnist_data_file, batch_size, randomize) logits, _, _ = utils.BuildNetwork(images, network_parameters) softmax = tf.nn.softmax(logits) # Load the variables. ckpt_state = tf.train.get_checkpoint_state(load_path) if not (ckpt_state and ckpt_state.model_checkpoint_path): raise ValueError("No model checkpoint to eval at %s\n" % load_path) saver = tf.train.Saver() saver.restore(sess, ckpt_state.model_checkpoint_path) coord = tf.train.Coordinator() _ = tf.train.start_queue_runners(sess=sess, coord=coord) total_examples = 0 correct_predictions = 0 image_index = 0 mistakes = [] for _ in xrange((num_testing_images + batch_size - 1) // batch_size): predictions, label_values = sess.run([softmax, labels]) # Count how many were predicted correctly. for prediction, label_value in zip(predictions, label_values): total_examples += 1 if np.argmax(prediction) == label_value: correct_predictions += 1 elif save_mistakes: mistakes.append({ "index": image_index, "label": label_value, "pred": np.argmax(prediction) }) image_index += 1 return (correct_predictions / total_examples, mistakes if save_mistakes else None)
def build_model(network_parameters): images = tf.placeholder(tf.float32, [None, 784], name='images') labels = tf.placeholder(tf.int64, [None], name='labels') #images, labels = MnistInput(mnist_train_file, batch_size, FLAGS.randomize) logits, projection, training_params = utils.BuildNetwork( images, network_parameters) return training_params
def CreateModel(sess, mnist_train_file, FLAGS): """ This function creates a simple feed-forward neural network. """ batch_size = FLAGS.batch_size network_parameters = FLAGS.network_parameters # Create the basic Mnist model. images, labels = MnistInput(mnist_train_file, FLAGS.batch_size, FLAGS.randomize, FLAGS=FLAGS) logits, projection, training_params = utils.BuildNetwork(images, network_parameters) cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=tf.one_hot(labels, 10)) # The actual cost is the average across the examples. cost = tf.reduce_sum(cost, [0]) / batch_size return cost, logits, training_params
def Train(mnist_train_file, mnist_test_file, network_parameters, num_steps, save_path, eval_steps=0): """Train MNIST for a number of steps. Args: mnist_train_file: path of MNIST train_bkp data file. mnist_test_file: path of MNIST test data file. network_parameters: parameters for defining and training the network. num_steps: number of steps to run. Here steps = lots save_path: path where to save trained parameters. eval_steps: evaluate the model every eval_steps. Returns: the result after the final training step. Raises: ValueError: if the accountant_type is not supported. """ batch_size = FLAGS.batch_size params = { "accountant_type": FLAGS.accountant_type, "task_id": 0, "batch_size": FLAGS.batch_size, "projection_dimensions": FLAGS.projection_dimensions, "default_gradient_l2norm_bound": network_parameters.default_gradient_l2norm_bound, "num_hidden_layers": FLAGS.num_hidden_layers, "hidden_layer_num_units": FLAGS.hidden_layer_num_units, "num_examples": NUM_TRAINING_IMAGES, "learning_rate": FLAGS.lr, "end_learning_rate": FLAGS.end_lr, "learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs } # Log different privacy parameters dependent on the accountant type. if FLAGS.accountant_type == "Amortized": params.update({ "flag_eps": FLAGS.eps, "flag_delta": FLAGS.delta, "flag_pca_eps": FLAGS.pca_eps, "flag_pca_delta": FLAGS.pca_delta, }) elif FLAGS.accountant_type == "Moments": params.update({ "sigma": FLAGS.sigma, "pca_sigma": FLAGS.pca_sigma, }) with tf.Graph().as_default(), tf.Session() as sess, tf.device('/cpu:0'): # Create the basic Mnist model. images, labels = MnistInput(mnist_train_file, batch_size, FLAGS.randomize) logits, projection, training_params = utils.BuildNetwork( images, network_parameters) cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=tf.one_hot( labels, 10)) # The actual cost is the average across the examples. cost = tf.reduce_sum(cost, [0]) / batch_size if FLAGS.accountant_type == "Amortized": priv_accountant = accountant.AmortizedAccountant( NUM_TRAINING_IMAGES) sigma = None pca_sigma = None with_privacy = FLAGS.eps > 0 elif FLAGS.accountant_type == "Moments": priv_accountant = accountant.GaussianMomentsAccountant( NUM_TRAINING_IMAGES) sigma = FLAGS.sigma pca_sigma = FLAGS.pca_sigma with_privacy = FLAGS.sigma > 0 else: raise ValueError("Undefined accountant type, needs to be " "Amortized or Moments, but got %s" % FLAGS.accountant) # Note: Here and below, we scale down the l2norm_bound by # batch_size. This is because per_example_gradients computes the # gradient of the minibatch loss with respect to each individual # example, and the minibatch loss (for our model) is the *average* # loss over examples in the minibatch. Hence, the scale of the # per-example gradients goes like 1 / batch_size. gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer( priv_accountant, [ network_parameters.default_gradient_l2norm_bound / batch_size, True ]) for var in training_params: if "gradient_l2norm_bound" in training_params[var]: l2bound = training_params[var][ "gradient_l2norm_bound"] / batch_size gaussian_sanitizer.set_option( var, sanitizer.ClipOption(l2bound, True)) lr = tf.placeholder(tf.float32) eps = tf.placeholder(tf.float32) delta = tf.placeholder(tf.float32) init_ops = [] if network_parameters.projection_type == "PCA": with tf.variable_scope("pca"): # Compute differentially private PCA. all_data, _ = MnistInput(mnist_train_file, NUM_TRAINING_IMAGES, False) pca_projection = dp_pca.ComputeDPPrincipalProjection( all_data, network_parameters.projection_dimensions, gaussian_sanitizer, [FLAGS.pca_eps, FLAGS.pca_delta], pca_sigma) assign_pca_proj = tf.assign(projection, pca_projection) init_ops.append(assign_pca_proj) # Add global_step global_step = tf.Variable(0, dtype=tf.int32, trainable=False, name="global_step") if with_privacy: gd_op = dp_optimizer.DPGradientDescentOptimizer( lr, [eps, delta], gaussian_sanitizer, sigma=sigma, batches_per_lot=FLAGS.batches_per_lot).minimize( cost, global_step=global_step) else: gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost) saver = tf.train.Saver() coord = tf.train.Coordinator() _ = tf.train.start_queue_runners(sess=sess, coord=coord) # We need to maintain the intialization sequence. for v in tf.trainable_variables(): sess.run(tf.variables_initializer([v])) sess.run(tf.global_variables_initializer()) sess.run(init_ops) results = [] start_time = time.time() prev_time = start_time filename = "results-0.json" log_path = os.path.join(save_path, filename) target_eps = [float(s) for s in FLAGS.target_eps.split(",")] if FLAGS.accountant_type == "Amortized": # Only matters if --terminate_based_on_privacy is true. target_eps = [max(target_eps)] max_target_eps = max(target_eps) lot_size = FLAGS.batches_per_lot * FLAGS.batch_size lots_per_epoch = NUM_TRAINING_IMAGES / lot_size for step in xrange(num_steps): epoch = step / lots_per_epoch curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr, FLAGS.lr_saturate_epochs, epoch) curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps, FLAGS.eps_saturate_epochs, epoch) for _ in xrange(FLAGS.batches_per_lot): _ = sess.run([gd_op], feed_dict={ lr: curr_lr, eps: curr_eps, delta: FLAGS.delta }) sys.stderr.write("step: %d\n" % step) # See if we should stop training due to exceeded privacy budget: should_terminate = False terminate_spent_eps_delta = None if with_privacy and FLAGS.terminate_based_on_privacy: terminate_spent_eps_delta = priv_accountant.get_privacy_spent( sess, target_eps=[max_target_eps])[0] # For the Moments accountant, we should always have # spent_eps == max_target_eps. if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta or terminate_spent_eps_delta.spent_eps > max_target_eps): should_terminate = True if (eval_steps > 0 and (step + 1) % eval_steps == 0) or should_terminate: if with_privacy: spent_eps_deltas = priv_accountant.get_privacy_spent( sess, target_eps=target_eps) else: spent_eps_deltas = [accountant.EpsDelta(0, 0)] for spent_eps, spent_delta in spent_eps_deltas: sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" % (spent_eps, spent_delta)) saver.save(sess, save_path=save_path + "/ckpt") train_accuracy, _ = Eval(mnist_train_file, network_parameters, num_testing_images=NUM_TESTING_IMAGES, randomize=True, load_path=save_path) sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy) test_accuracy, mistakes = Eval( mnist_test_file, network_parameters, num_testing_images=NUM_TESTING_IMAGES, randomize=False, load_path=save_path, save_mistakes=FLAGS.save_mistakes) sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy) curr_time = time.time() elapsed_time = curr_time - prev_time prev_time = curr_time results.append({ "step": step + 1, # Number of lots trained so far. "elapsed_secs": elapsed_time, "spent_eps_deltas": spent_eps_deltas, "train_accuracy": train_accuracy, "test_accuracy": test_accuracy, "mistakes": mistakes }) loginfo = { "elapsed_secs": curr_time - start_time, "spent_eps_deltas": spent_eps_deltas, "train_accuracy": train_accuracy, "test_accuracy": test_accuracy, "num_training_steps": step + 1, # Steps so far. "mistakes": mistakes, "result_series": results } loginfo.update(params) if log_path: with tf.gfile.Open(log_path, "w") as f: json.dump(loginfo, f, indent=2) f.write("\n") f.close() if should_terminate: break
def Train(cifar_train_file, cifar_test_file, network_parameters, num_steps, save_path, eval_steps=0): """Train CIFAR-10 for a number of steps. Args: cifar_train_file: path of CIFAR-10 train data file. cifar_test_file: path of CIFAR-10 test data file. network_parameters: parameters for defining and training the network. num_steps: number of steps to run. Here steps = lots save_path: path where to save trained parameters. eval_steps: evaluate the model every eval_steps. Returns: the result after the final training step. Raises: ValueError: if the accountant_type is not supported. """ model_dir = FLAGS.train_dir self.model_dir = model_dir batch_size = FLAGS.batch_size params = { "accountant_type": FLAGS.accountant_type, "task_id": 0, "batch_size": FLAGS.batch_size, "default_gradient_l2norm_bound": network_parameters.default_gradient_l2norm_bound, "num_hidden_layers": FLAGS.num_hidden_layers, "hidden_layer_num_units": FLAGS.hidden_layer_num_units, "num_examples": self.num_training_images, "learning_rate": FLAGS.lr, "end_learning_rate": FLAGS.end_lr, "learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs } params.update({"sigma": FLAGS.sigma}) with tf.Graph().as_default(), tf.Session() as sess, tf.device( '/cpu:0'): # Create the basic Cifar model. # TODO: GET INPUT FOR CLIENT #images, labels = CifarInput(cifar_train_file, batch_size, FLAGS.randomize) images, labels = cifar10_input.distorted_inputs( "data/clients/cifar", self.index, TFFLAGS.batch_size) logits, projection, training_params = utils.BuildNetwork( images, network_parameters) cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=tf.one_hot( labels, 10)) # The actual cost is the average across the examples. cost = tf.reduce_sum(cost, [0]) / batch_size priv_accountant = accountant.GaussianMomentsAccountant( self.num_training_images) sigma = FLAGS.sigma with_privacy = FLAGS.sigma > 0 # Note: Here and below, we scale down the l2norm_bound by # batch_size. This is because per_example_gradients computes the # gradient of the minibatch loss with respect to each individual # example, and the minibatch loss (for our model) is the *average* # loss over examples in the minibatch. Hence, the scale of the # per-example gradients goes like 1 / batch_size. gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer( priv_accountant, [ network_parameters.default_gradient_l2norm_bound / batch_size, True ]) for var in training_params: if "gradient_l2norm_bound" in training_params[var]: l2bound = training_params[var][ "gradient_l2norm_bound"] / batch_size gaussian_sanitizer.set_option( var, sanitizer.ClipOption(l2bound, True)) lr = tf.placeholder(tf.float32) eps = tf.placeholder(tf.float32) delta = tf.placeholder(tf.float32) init_ops = [] # Add global_step global_step = tf.Variable(0, dtype=tf.int32, trainable=False, name="global_step") if with_privacy: gd_op = dp_optimizer.DPGradientDescentOptimizer( lr, [eps, delta], gaussian_sanitizer, sigma=sigma, batches_per_lot=FLAGS.batches_per_lot).minimize( cost, global_step=global_step) else: gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost) saver = tf.train.Saver() coord = tf.train.Coordinator() _ = tf.train.start_queue_runners(sess=sess, coord=coord) # We need to maintain the intialization sequence. for v in tf.trainable_variables(): sess.run(tf.variables_initializer([v])) sess.run(tf.global_variables_initializer()) sess.run(init_ops) results = [] start_time = time.time() prev_time = start_time filename = "results-0.json" log_path = os.path.join(save_path, "client-" + self.index + "/", filename) target_eps = [float(s) for s in FLAGS.target_eps.split(",")] max_target_eps = max(target_eps) lot_size = FLAGS.batches_per_lot * FLAGS.batch_size lots_per_epoch = self.num_training_images / lot_size for step in range(num_steps): epoch = step / lots_per_epoch curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr, FLAGS.lr_saturate_epochs, epoch) curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps, FLAGS.eps_saturate_epochs, epoch) for _ in range(FLAGS.batches_per_lot): _ = sess.run([gd_op], feed_dict={ lr: curr_lr, eps: curr_eps, delta: FLAGS.delta }) sys.stderr.write("step: %d\n" % step) # See if we should stop training due to exceeded privacy budget: should_terminate = False terminate_spent_eps_delta = None if with_privacy and FLAGS.terminate_based_on_privacy: terminate_spent_eps_delta = priv_accountant.get_privacy_spent( sess, target_eps=[max_target_eps])[0] # For the Moments accountant, we should always have # spent_eps == max_target_eps. if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta or terminate_spent_eps_delta.spent_eps > max_target_eps): should_terminate = True if (eval_steps > 0 and (step + 1) % eval_steps == 0) or should_terminate: if with_privacy: spent_eps_deltas = priv_accountant.get_privacy_spent( sess, target_eps=target_eps) else: spent_eps_deltas = [accountant.EpsDelta(0, 0)] for spent_eps, spent_delta in spent_eps_deltas: sys.stderr.write( "spent privacy: eps %.4f delta %.5g\n" % (spent_eps, spent_delta)) saver.save(sess, save_path=save_path + "/ckpt") train_accuracy, _ = Eval( cifar_train_file, network_parameters, num_testing_images=self.num_testing_images, randomize=True, load_path=TFFLAGS.load_path) sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy) test_accuracy, mistakes = Eval( cifar_test_file, network_parameters, num_testing_images=self.num_testing_images, randomize=False, load_path=TFFLAGS.load_path, save_mistakes=FLAGS.save_mistakes) sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy) curr_time = time.time() elapsed_time = curr_time - prev_time prev_time = curr_time results.append({ "step": step + 1, # Number of lots trained so far. "elapsed_secs": elapsed_time, "spent_eps_deltas": spent_eps_deltas, "train_accuracy": train_accuracy, "test_accuracy": test_accuracy, "mistakes": mistakes }) loginfo = { "elapsed_secs": curr_time - start_time, "spent_eps_deltas": spent_eps_deltas, "train_accuracy": train_accuracy, "test_accuracy": test_accuracy, "num_training_steps": step + 1, # Steps so far. "mistakes": mistakes, "result_series": results } loginfo.update(params) if log_path: with tf.gfile.Open(log_path, "w") as f: json.dump(loginfo, f, indent=2) f.write("\n") f.close() if should_terminate: for t in tf.trainable_variables(): weights.append(t.eval(session=mon_sess)) print("\nTERMINATING.\n") break
def Eval(mnist_data_file, network_parameters, num_testing_images, randomize, load_path, save_mistakes=False): """Evaluate MNIST for a number of steps. Args: mnist_data_file: Path of a file containing the MNIST images to process. network_parameters: parameters for defining and training the network. num_testing_images: the number of images we will evaluate on. randomize: if false, randomize; otherwise, read the testing images sequentially. load_path: path where to load trained parameters from. save_mistakes: save the mistakes if True. Returns: The evaluation accuracy as a float. """ batch_size = 100 # Like for training, we need a session for executing the TensorFlow graph. with tf.Graph().as_default(), tf.Session() as sess: # Create the basic Mnist model. images, labels = MnistInput(mnist_data_file, batch_size, randomize) logits, _, _ = utils.BuildNetwork(images, network_parameters) softmax = tf.nn.softmax(logits) # Load the variables. ckpt_state = tf.train.get_checkpoint_state(load_path) if not (ckpt_state and ckpt_state.model_checkpoint_path): raise ValueError("No model checkpoint to eval at %s\n" % load_path) saver = tf.train.Saver() saver.restore(sess, ckpt_state.model_checkpoint_path) coord = tf.train.Coordinator() _ = tf.train.start_queue_runners(sess=sess, coord=coord) total_examples = 0 correct_predictions = 0 image_index = 0 mistakes = [] for _ in range((num_testing_images + batch_size - 1) // batch_size): options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() predictions, label_values = sess.run([softmax, labels], options=options, run_metadata=run_metadata) cg = CompGraph('differential_privacy_sgd', run_metadata, tf.get_default_graph()) cg_tensor_dict = cg.get_tensors() cg_sorted_keys = sorted(cg_tensor_dict.keys()) cg_sorted_items = [] for cg_key in cg_sorted_keys: cg_sorted_items.append(tf.shape(cg_tensor_dict[cg_key])) cg_sorted_shape = sess.run(cg_sorted_items) cg.op_analysis(dict(zip(cg_sorted_keys, cg_sorted_shape)), 'differential_privacy_sgd.pickle') exit(0) # Count how many were predicted correctly. for prediction, label_value in zip(predictions, label_values): total_examples += 1 if np.argmax(prediction) == label_value: correct_predictions += 1 elif save_mistakes: mistakes.append({ "index": image_index, "label": label_value, "pred": np.argmax(prediction) }) image_index += 1 return (correct_predictions / total_examples, mistakes if save_mistakes else None)
def Train(train_file, test_file, validation_file, network_parameters, num_steps, save_path, total_rho, eval_steps=0): """Train MNIST for a number of steps. Args: mnist_train_file: path of MNIST train data file. mnist_test_file: path of MNIST test data file. network_parameters: parameters for defining and training the network. num_steps: number of steps to run. Here steps = lots save_path: path where to save trained parameters. eval_steps: evaluate the model every eval_steps. Returns: the result after the final training step. Raises: ValueError: if the accountant_type is not supported. """ batch_size = NUM_TRAINING_IMAGES params = { "accountant_type": FLAGS.accountant_type, "task_id": 0, "batch_size": FLAGS.batch_size, "default_gradient_l2norm_bound": network_parameters.default_gradient_l2norm_bound, "num_examples": NUM_TRAINING_IMAGES, "learning_rate": FLAGS.lr, "end_learning_rate": FLAGS.end_lr, "learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs } # Log different privacy parameters dependent on the accountant type. if FLAGS.accountant_type == "Amortized": params.update({ "flag_eps": FLAGS.eps, "flag_delta": FLAGS.delta, }) elif FLAGS.accountant_type == "Moments": params.update({ "sigma": FLAGS.sigma, }) with tf.device('/gpu:0'), tf.Graph().as_default(), tf.Session() as sess: #print_csv_tfrecords.print_tfrecords(train_file) features, labels = DataInput(train_file, batch_size, False) print("network_parameters.input_size", network_parameters.input_size) logits, projection, training_params = utils.BuildNetwork( features, network_parameters) cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=tf.one_hot( labels, LABEL_SIZE)) # The actual cost is the average across the examples. cost = tf.reduce_sum(cost, [0]) / batch_size if FLAGS.accountant_type == "Amortized": priv_accountant = accountant.AmortizedAccountant( NUM_TRAINING_IMAGES) sigma = None elif FLAGS.accountant_type == "Moments": priv_accountant = accountant.GaussianMomentsAccountant( NUM_TRAINING_IMAGES) sigma = FLAGS.sigma elif FLAGS.accountant_type == "ZDCP": priv_accountant = accountant.DumpzCDPAccountant() else: raise ValueError("Undefined accountant type, needs to be " "Amortized or Moments, but got %s" % FLAGS.accountant) # Note: Here and below, we scale down the l2norm_bound by # batch_size. This is because per_example_gradients computes the # gradient of the minibatch loss with respect to each individual # example, and the minibatch loss (for our model) is the *average* # loss over examples in the minibatch. Hence, the scale of the # per-example gradients goes like 1 / batch_size. gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer( priv_accountant, [ network_parameters.default_gradient_l2norm_bound / batch_size, True ]) for var in training_params: if "gradient_l2norm_bound" in training_params[var]: l2bound = training_params[var][ "gradient_l2norm_bound"] / batch_size gaussian_sanitizer.set_option( var, sanitizer.ClipOption(l2bound, True)) lr = tf.placeholder(tf.float32) eps = tf.placeholder(tf.float32) delta = tf.placeholder(tf.float32) varsigma = tf.placeholder(tf.float32, shape=[]) init_ops = [] # Add global_step global_step = tf.Variable(0, dtype=tf.int32, trainable=False, name="global_step") with_privacy = True if with_privacy: gd_op = dp_optimizer.DPGradientDescentOptimizer( lr, [eps, delta], gaussian_sanitizer, varsigma, batches_per_lot=FLAGS.batches_per_lot).minimize( cost, global_step=global_step) else: print("No privacy") gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost) saver = tf.train.Saver() coord = tf.train.Coordinator() _ = tf.train.start_queue_runners(sess=sess, coord=coord) # We need to maintain the intialization sequence. for v in tf.trainable_variables(): sess.run(tf.variables_initializer([v])) sess.run(tf.global_variables_initializer()) sess.run(init_ops) results = [] start_time = time.time() prev_time = start_time filename = "results-0.json" log_path = os.path.join(save_path, filename) target_eps = [float(s) for s in FLAGS.target_eps.split(",")] if FLAGS.accountant_type == "Amortized": # Only matters if --terminate_based_on_privacy is true. target_eps = [max(target_eps)] max_target_eps = max(target_eps) lot_size = FLAGS.batches_per_lot * FLAGS.batch_size lots_per_epoch = NUM_TRAINING_IMAGES / lot_size previous_epoch = -1 rho_tracking = [0] validation_accuracy_list = [] previous_validaccuracy = 0 tracking_sigma = [] curr_sigma = 35 for step in range(num_steps): epoch = step // lots_per_epoch curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr, FLAGS.lr_saturate_epochs, epoch) curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps, FLAGS.eps_saturate_epochs, epoch) if with_privacy: old_sigma = curr_sigma #total budget rhototal = total_rho # validation based decay # period=10, threshold=0.01, decay_factor=0.9 period = 20 decay_factor = 0.99 threshold = 0.01 m = 1 if epoch - previous_epoch == 1 and ( epoch + 1) % period == 0: # checking epoch current_validaccuracy = sum( validation_accuracy_list[-m:]) / m if current_validaccuracy - previous_validaccuracy < threshold: curr_sigma = decay_factor * curr_sigma previous_validaccuracy = current_validaccuracy if old_sigma != curr_sigma: print(curr_sigma) # for tracking by epoch if epoch - previous_epoch == 1: tracking_sigma.append(curr_sigma) rho_tracking.append(rho_tracking[-1] + 1 / (2.0 * curr_sigma**2)) previous_epoch = epoch if with_privacy == True and rho_tracking[-1] > rhototal: print("stop at epoch%d" % epoch) break print(rho_tracking) print(tracking_sigma) if step % 100 == 0: print(curr_sigma) print(rho_tracking[-1]) for _ in range(FLAGS.batches_per_lot): _ = sess.run( [gd_op], feed_dict={ lr: curr_lr, eps: curr_eps, delta: FLAGS.delta, varsigma: curr_sigma }) sys.stderr.write("step: %d\n" % step) # See if we should stop training due to exceeded privacy budget: should_terminate = False if (eval_steps > 0 and (step + 1) % eval_steps == 0) or should_terminate: saver.save(sess, save_path=save_path + "/ckpt") train_accuracy, _ = Eval( train_file, network_parameters, num_testing_images=NUM_TRAINING_IMAGES, randomize=False, load_path=save_path) sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy) test_accuracy, mistakes = Eval( test_file, network_parameters, num_testing_images=NUM_TESTING_IMAGES, randomize=False, load_path=save_path, save_mistakes=FLAGS.save_mistakes) sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy) validation_accuracy, mistakes = Eval( validation_file, network_parameters, num_testing_images=NUM_VALIDATION_IMAGES, randomize=False, load_path=save_path, save_mistakes=FLAGS.save_mistakes) sys.stderr.write("validation_accuracy: %.2f\n" % validation_accuracy) validation_accuracy_list.append(validation_accuracy) curr_time = time.time() elapsed_time = curr_time - prev_time prev_time = curr_time results.append({ "step": step + 1, # Number of lots trained so far. "elapsed_secs": elapsed_time, "train_accuracy": train_accuracy, "test_accuracy": test_accuracy, "mistakes": mistakes }) loginfo = { "elapsed_secs": curr_time - start_time, "train_accuracy": train_accuracy, "test_accuracy": test_accuracy, "num_training_steps": step + 1, # Steps so far. "mistakes": mistakes, "result_series": results } loginfo.update(params) if log_path: with tf.gfile.Open(log_path, "w") as f: json.dump(loginfo, f, indent=2) f.write("\n") f.close() if should_terminate: break print(rho_tracking[:-1]) saver.save(sess, save_path=save_path + "/ckpt") train_accuracy, _ = Eval(train_file, network_parameters, num_testing_images=NUM_TRAINING_IMAGES, randomize=False, load_path=save_path) sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy) test_accuracy, mistakes = Eval(test_file, network_parameters, num_testing_images=NUM_TESTING_IMAGES, randomize=False, load_path=save_path, save_mistakes=FLAGS.save_mistakes) sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy) curr_time = time.time() elapsed_time = curr_time - prev_time prev_time = curr_time results.append({ "step": step + 1, # Number of lots trained so far. "elapsed_secs": elapsed_time, "train_accuracy": train_accuracy, "test_accuracy": test_accuracy, "mistakes": mistakes }) loginfo = { "elapsed_secs": curr_time - start_time, "train_accuracy": train_accuracy, "test_accuracy": test_accuracy, "num_training_steps": step, # Steps so far. "mistakes": mistakes, "result_series": results } loginfo.update(params) if log_path: with tf.gfile.Open(log_path, "w") as f: json.dump(loginfo, f, indent=2) f.write("\n") f.close()