def Eval(mnist_data_file,
network_parameters,
num_testing_images,
randomize,
load_path,
save_mistakes=False):
"""Evaluate MNIST for a number of steps.
Args:
mnist_data_file: Path of a file containing the MNIST images to process.
network_parameters: parameters for defining and training the network.
num_testing_images: the number of images we will evaluate on.
randomize: if false, randomize; otherwise, read the testing images
sequentially.
load_path: path where to load trained parameters from.
save_mistakes: save the mistakes if True.
Returns:
The evaluation accuracy as a float.
"""
batch_size = 100
# Like for training, we need a session for executing the TensorFlow graph.
with tf.Graph().as_default(), tf.Session() as sess:
# Create the basic Mnist model.
images, labels = MnistInput(mnist_data_file, batch_size, randomize)
logits, _, _ = utils.BuildNetwork(images, network_parameters)
softmax = tf.nn.softmax(logits)
# Load the variables.
ckpt_state = tf.train.get_checkpoint_state(load_path)
if not (ckpt_state and ckpt_state.model_checkpoint_path):
raise ValueError("No model checkpoint to eval at %s\n" % load_path)
saver = tf.train.Saver()
saver.restore(sess, ckpt_state.model_checkpoint_path)
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
total_examples = 0
correct_predictions = 0
image_index = 0
mistakes = []
for _ in xrange((num_testing_images + batch_size - 1) // batch_size):
predictions, label_values = sess.run([softmax, labels])
# Count how many were predicted correctly.
for prediction, label_value in zip(predictions, label_values):
total_examples += 1
if np.argmax(prediction) == label_value:
correct_predictions += 1
elif save_mistakes:
mistakes.append({
"index": image_index,
"label": label_value,
"pred": np.argmax(prediction)
})
image_index += 1
return (correct_predictions / total_examples,
mistakes if save_mistakes else None)
def build_model(network_parameters):
images = tf.placeholder(tf.float32, [None, 784], name='images')
labels = tf.placeholder(tf.int64, [None], name='labels')
#images, labels = MnistInput(mnist_train_file, batch_size, FLAGS.randomize)
logits, projection, training_params = utils.BuildNetwork(
images, network_parameters)
return training_params
def CreateModel(sess, mnist_train_file, FLAGS):
"""
This function creates a simple feed-forward neural network.
"""
batch_size = FLAGS.batch_size
network_parameters = FLAGS.network_parameters
# Create the basic Mnist model.
images, labels = MnistInput(mnist_train_file, FLAGS.batch_size, FLAGS.randomize, FLAGS=FLAGS)
logits, projection, training_params = utils.BuildNetwork(images, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=tf.one_hot(labels, 10))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
return cost, logits, training_params
def Train(mnist_train_file,
mnist_test_file,
network_parameters,
num_steps,
save_path,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train_bkp data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"projection_dimensions": FLAGS.projection_dimensions,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
"flag_pca_eps": FLAGS.pca_eps,
"flag_pca_delta": FLAGS.pca_delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
"pca_sigma": FLAGS.pca_sigma,
})
with tf.Graph().as_default(), tf.Session() as sess, tf.device('/cpu:0'):
# Create the basic Mnist model.
images, labels = MnistInput(mnist_train_file, batch_size,
FLAGS.randomize)
logits, projection, training_params = utils.BuildNetwork(
images, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 10))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(
NUM_TRAINING_IMAGES)
sigma = None
pca_sigma = None
with_privacy = FLAGS.eps > 0
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
pca_sigma = FLAGS.pca_sigma
with_privacy = FLAGS.sigma > 0
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" %
FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
init_ops = []
if network_parameters.projection_type == "PCA":
with tf.variable_scope("pca"):
# Compute differentially private PCA.
all_data, _ = MnistInput(mnist_train_file, NUM_TRAINING_IMAGES,
False)
pca_projection = dp_pca.ComputeDPPrincipalProjection(
all_data, network_parameters.projection_dimensions,
gaussian_sanitizer, [FLAGS.pca_eps, FLAGS.pca_delta],
pca_sigma)
assign_pca_proj = tf.assign(projection, pca_projection)
init_ops.append(assign_pca_proj)
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
sigma=sigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
for step in xrange(num_steps):
epoch = step / lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
for _ in xrange(FLAGS.batches_per_lot):
_ = sess.run([gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or
terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(mnist_train_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=True,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
mnist_test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
def Train(cifar_train_file,
cifar_test_file,
network_parameters,
num_steps,
save_path,
eval_steps=0):
"""Train CIFAR-10 for a number of steps.
Args:
cifar_train_file: path of CIFAR-10 train data file.
cifar_test_file: path of CIFAR-10 test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
model_dir = FLAGS.train_dir
self.model_dir = model_dir
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": self.num_training_images,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
params.update({"sigma": FLAGS.sigma})
with tf.Graph().as_default(), tf.Session() as sess, tf.device(
'/cpu:0'):
# Create the basic Cifar model.
# TODO: GET INPUT FOR CLIENT
#images, labels = CifarInput(cifar_train_file, batch_size, FLAGS.randomize)
images, labels = cifar10_input.distorted_inputs(
"data/clients/cifar", self.index, TFFLAGS.batch_size)
logits, projection, training_params = utils.BuildNetwork(
images, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 10))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
priv_accountant = accountant.GaussianMomentsAccountant(
self.num_training_images)
sigma = FLAGS.sigma
with_privacy = FLAGS.sigma > 0
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound /
batch_size, True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
init_ops = []
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
sigma=sigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, "client-" + self.index + "/",
filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = self.num_training_images / lot_size
for step in range(num_steps):
epoch = step / lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
for _ in range(FLAGS.batches_per_lot):
_ = sess.run([gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or
terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write(
"spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(
cifar_train_file,
network_parameters,
num_testing_images=self.num_testing_images,
randomize=True,
load_path=TFFLAGS.load_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
cifar_test_file,
network_parameters,
num_testing_images=self.num_testing_images,
randomize=False,
load_path=TFFLAGS.load_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
for t in tf.trainable_variables():
weights.append(t.eval(session=mon_sess))
print("\nTERMINATING.\n")
break
def Eval(mnist_data_file,
network_parameters,
num_testing_images,
randomize,
load_path,
save_mistakes=False):
"""Evaluate MNIST for a number of steps.
Args:
mnist_data_file: Path of a file containing the MNIST images to process.
network_parameters: parameters for defining and training the network.
num_testing_images: the number of images we will evaluate on.
randomize: if false, randomize; otherwise, read the testing images
sequentially.
load_path: path where to load trained parameters from.
save_mistakes: save the mistakes if True.
Returns:
The evaluation accuracy as a float.
"""
batch_size = 100
# Like for training, we need a session for executing the TensorFlow graph.
with tf.Graph().as_default(), tf.Session() as sess:
# Create the basic Mnist model.
images, labels = MnistInput(mnist_data_file, batch_size, randomize)
logits, _, _ = utils.BuildNetwork(images, network_parameters)
softmax = tf.nn.softmax(logits)
# Load the variables.
ckpt_state = tf.train.get_checkpoint_state(load_path)
if not (ckpt_state and ckpt_state.model_checkpoint_path):
raise ValueError("No model checkpoint to eval at %s\n" % load_path)
saver = tf.train.Saver()
saver.restore(sess, ckpt_state.model_checkpoint_path)
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
total_examples = 0
correct_predictions = 0
image_index = 0
mistakes = []
for _ in range((num_testing_images + batch_size - 1) // batch_size):
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
predictions, label_values = sess.run([softmax, labels],
options=options,
run_metadata=run_metadata)
cg = CompGraph('differential_privacy_sgd', run_metadata,
tf.get_default_graph())
cg_tensor_dict = cg.get_tensors()
cg_sorted_keys = sorted(cg_tensor_dict.keys())
cg_sorted_items = []
for cg_key in cg_sorted_keys:
cg_sorted_items.append(tf.shape(cg_tensor_dict[cg_key]))
cg_sorted_shape = sess.run(cg_sorted_items)
cg.op_analysis(dict(zip(cg_sorted_keys, cg_sorted_shape)),
'differential_privacy_sgd.pickle')
exit(0)
# Count how many were predicted correctly.
for prediction, label_value in zip(predictions, label_values):
total_examples += 1
if np.argmax(prediction) == label_value:
correct_predictions += 1
elif save_mistakes:
mistakes.append({
"index": image_index,
"label": label_value,
"pred": np.argmax(prediction)
})
image_index += 1
return (correct_predictions / total_examples,
mistakes if save_mistakes else None)
def Train(train_file,
test_file,
validation_file,
network_parameters,
num_steps,
save_path,
total_rho,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = NUM_TRAINING_IMAGES
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
})
with tf.device('/gpu:0'), tf.Graph().as_default(), tf.Session() as sess:
#print_csv_tfrecords.print_tfrecords(train_file)
features, labels = DataInput(train_file, batch_size, False)
print("network_parameters.input_size", network_parameters.input_size)
logits, projection, training_params = utils.BuildNetwork(
features, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, LABEL_SIZE))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(
NUM_TRAINING_IMAGES)
sigma = None
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
elif FLAGS.accountant_type == "ZDCP":
priv_accountant = accountant.DumpzCDPAccountant()
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" %
FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
varsigma = tf.placeholder(tf.float32, shape=[])
init_ops = []
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
with_privacy = True
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
varsigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
print("No privacy")
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
previous_epoch = -1
rho_tracking = [0]
validation_accuracy_list = []
previous_validaccuracy = 0
tracking_sigma = []
curr_sigma = 35
for step in range(num_steps):
epoch = step // lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
if with_privacy:
old_sigma = curr_sigma
#total budget
rhototal = total_rho
# validation based decay
# period=10, threshold=0.01, decay_factor=0.9
period = 20
decay_factor = 0.99
threshold = 0.01
m = 1
if epoch - previous_epoch == 1 and (
epoch + 1) % period == 0: # checking epoch
current_validaccuracy = sum(
validation_accuracy_list[-m:]) / m
if current_validaccuracy - previous_validaccuracy < threshold:
curr_sigma = decay_factor * curr_sigma
previous_validaccuracy = current_validaccuracy
if old_sigma != curr_sigma:
print(curr_sigma)
# for tracking by epoch
if epoch - previous_epoch == 1:
tracking_sigma.append(curr_sigma)
rho_tracking.append(rho_tracking[-1] + 1 /
(2.0 * curr_sigma**2))
previous_epoch = epoch
if with_privacy == True and rho_tracking[-1] > rhototal:
print("stop at epoch%d" % epoch)
break
print(rho_tracking)
print(tracking_sigma)
if step % 100 == 0:
print(curr_sigma)
print(rho_tracking[-1])
for _ in range(FLAGS.batches_per_lot):
_ = sess.run(
[gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta,
varsigma: curr_sigma
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(
train_file,
network_parameters,
num_testing_images=NUM_TRAINING_IMAGES,
randomize=False,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
validation_accuracy, mistakes = Eval(
validation_file,
network_parameters,
num_testing_images=NUM_VALIDATION_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("validation_accuracy: %.2f\n" %
validation_accuracy)
validation_accuracy_list.append(validation_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
print(rho_tracking[:-1])
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(train_file,
network_parameters,
num_testing_images=NUM_TRAINING_IMAGES,
randomize=False,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
