def BoundGradients(training_params, priv_accountant, network_parameters, batch_size):
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant,
[network_parameters.default_gradient_l2norm_bound / batch_size, True]
)
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var]["gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(var, sanitizer.ClipOption(l2bound, True))
return gaussian_sanitizer
def ComputeDPPrincipalProjection(data, projection_dims, sanitizer, eps_delta,
sigma):
"""Compute differentially private projection.
Args:
data: the input data, each row is a data vector.
projection_dims: the projection dimension.
sanitizer: the sanitizer used for acheiving privacy.
eps_delta: (eps, delta) pair.
sigma: if not None, use noise sigma; otherwise compute it using
eps_delta pair.
Returns:
A projection matrix with projection_dims columns.
"""
eps, delta = eps_delta
# Normalize each row.
normalized_data = tf.nn.l2_normalize(data, 1)
covar = tf.matmul(tf.transpose(normalized_data), normalized_data)
saved_shape = tf.shape(covar)
num_examples = tf.slice(tf.shape(data), [0], [1])
if eps > 0:
# Since the data is already normalized, there is no need to clip
# the covariance matrix.
assert delta > 0
saned_covar = sanitizer.sanitize(tf.reshape(covar, [1, -1]),
eps_delta,
sigma=sigma,
option=san.ClipOption(1.0, False),
num_examples=num_examples)
saned_covar = tf.reshape(saned_covar, saved_shape)
# Symmetrize saned_covar. This also reduces the noise variance.
saned_covar = 0.5 * (saned_covar + tf.transpose(saned_covar))
else:
saned_covar = covar
# Compute the eigen decomposition of the covariance matrix, and
# return the top projection_dims eigen vectors, represented as columns of
# the projection matrix.
eigvals, eigvecs = tf.self_adjoint_eig(saned_covar)
_, topk_indices = tf.nn.top_k(eigvals, projection_dims)
topk_indices = tf.reshape(topk_indices, [projection_dims])
# Gather and return the corresponding eigenvectors.
return tf.transpose(tf.gather(tf.transpose(eigvecs), topk_indices))
def Train(sess,
train_images,
train_labels,
mnist_test_file,
network_parameters,
num_steps,
save_path,
training_params,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"projection_dimensions": FLAGS.projection_dimensions,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
"flag_pca_eps": FLAGS.pca_eps,
"flag_pca_delta": FLAGS.pca_delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
"pca_sigma": FLAGS.pca_sigma,
})
# Create the basic Mnist model.
images = tf.get_default_graph().get_tensor_by_name("images:0")
labels = tf.get_default_graph().get_tensor_by_name("labels:0")
logits = tf.get_default_graph().get_tensor_by_name("logits:0")
projection = tf.get_default_graph().get_tensor_by_name("projection:0")
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 10))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(NUM_TRAINING_IMAGES)
sigma = None
pca_sigma = None
with_privacy = FLAGS.eps > 0
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
pca_sigma = FLAGS.pca_sigma
with_privacy = FLAGS.sigma > 0
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" % FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant,
[network_parameters.default_gradient_l2norm_bound / batch_size, True])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var]["gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(var,
sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
init_ops = []
if network_parameters.projection_type == "PCA":
with tf.variable_scope("pca"):
# Compute differentially private PCA.
all_data = tf.constant(train_images, dtype=tf.float32)
pca_projection = dp_pca.ComputeDPPrincipalProjection(
all_data, network_parameters.projection_dimensions,
gaussian_sanitizer, [FLAGS.pca_eps, FLAGS.pca_delta],
pca_sigma)
assign_pca_proj = tf.assign(projection, pca_projection)
init_ops.append(assign_pca_proj)
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
sigma=sigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = np.array([float(s) for s in FLAGS.target_eps.split(",")])
target_eps = -np.sort(-target_eps)
index_eps_to_save = 0
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
for step in xrange(num_steps):
epoch = step / lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
lot = np.random.choice(np.arange(NUM_TRAINING_IMAGES),
lot_size,
replace=False)
#print lot.shape
images_lot = train_images[lot, :]
labels_lot = train_labels[lot]
for i in xrange(FLAGS.batches_per_lot):
_ = sess.run(
[gd_op],
feed_dict={
lr:
curr_lr,
eps:
curr_eps,
delta:
FLAGS.delta,
images:
images_lot[i * FLAGS.batch_size:(i + 1) *
FLAGS.batch_size, :],
labels:
labels_lot[i * FLAGS.batch_size:(i + 1) * FLAGS.batch_size]
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
while index_eps_to_save < len(
spent_eps_deltas
) and spent_eps_deltas[index_eps_to_save][1] < FLAGS.delta:
saver.save(
sess,
save_path=save_path + "/eps" +
str(spent_eps_deltas[index_eps_to_save][0]) +
"_delta" +
'%.2g' % spent_eps_deltas[index_eps_to_save][1] +
"_pcasigma" + str(FLAGS.pca_sigma) + "_sigma" +
str(FLAGS.sigma) + "/ckpt")
index_eps_to_save += 1
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
pred_train = np.argmax(predict(sess, train_images), axis=1)
train_accuracy = np.mean(pred_train == train_labels)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
# test_accuracy, mistakes = Eval(mnist_test_file, network_parameters,
# num_testing_images=NUM_TESTING_IMAGES,
# randomize=False, load_path=save_path,
# save_mistakes=FLAGS.save_mistakes)
# sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
# "test_accuracy": test_accuracy,
# "mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
# "test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
# "mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
def Train(cifar_train_file,
mnist_test_file,
network_parameters,
num_steps,
save_path,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
cifar_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
params.update({"sigma": FLAGS.sigma})
with tf.Graph().as_default(), tf.Session() as sess, tf.device('/cpu:0'):
# Create the basic Cifar model.
images, labels = CifarInput(cifar_train_file, batch_size,
FLAGS.randomize)
logits, projection, training_params = utils.BuildNetwork(
images, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 100))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
with_privacy = FLAGS.sigma > 0
with_privacy = False
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
init_ops = []
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
sigma=sigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
for step in range(num_steps):
epoch = step / lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
for _ in range(FLAGS.batches_per_lot):
_ = sess.run([gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or
terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(cifar_train_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=True,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
mnist_test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
print("\nTERMINATING.\n")
break
def Train(mnist_train_file,
mnist_test_file,
network_parameters,
num_steps,
save_path,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"projection_dimensions": FLAGS.projection_dimensions,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
"flag_pca_eps": FLAGS.pca_eps,
"flag_pca_delta": FLAGS.pca_delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
"pca_sigma": FLAGS.pca_sigma,
})
with tf.Graph().as_default(), tf.Session() as sess, tf.device('/cpu:0'):
# Create the basic Mnist model.
images, labels = MnistInput(mnist_train_file, batch_size,
FLAGS.randomize)
logits, projection, training_params = utils.BuildNetwork(
images, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 10))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(
NUM_TRAINING_IMAGES)
sigma = None
pca_sigma = None
with_privacy = FLAGS.eps > 0
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
pca_sigma = FLAGS.pca_sigma
with_privacy = FLAGS.sigma > 0
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" %
FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
init_ops = []
if network_parameters.projection_type == "PCA":
with tf.variable_scope("pca"):
# Compute differentially private PCA.
all_data, _ = MnistInput(mnist_train_file, NUM_TRAINING_IMAGES,
False)
pca_projection = dp_pca.ComputeDPPrincipalProjection(
all_data, network_parameters.projection_dimensions,
gaussian_sanitizer, [FLAGS.pca_eps, FLAGS.pca_delta],
pca_sigma)
assign_pca_proj = tf.assign(projection, pca_projection)
init_ops.append(assign_pca_proj)
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
sigma=sigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
for step in xrange(num_steps):
epoch = step / lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
for _ in xrange(FLAGS.batches_per_lot):
_ = sess.run([gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or
terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(mnist_train_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=True,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
mnist_test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
network_parameters = utils.NetworkParameters()
# If the ASCII proto isn't specified, then construct a config protobuf based
# on 3 flags.
network_parameters.input_size = IMAGE_SIZE**2
network_parameters.default_gradient_l2norm_bound = (
FLAGS.default_gradient_l2norm_bound)
if FLAGS.projection_dimensions > 0 and FLAGS.num_conv_layers > 0:
raise ValueError("Currently you can't do PCA and have convolutions"
"at the same time. Pick one")
# could add support for PCA after convolutions.
# Currently BuildNetwork can build the network with conv followed by
# projection, but the PCA training works on data, rather than data run
# through a few layers. Will need to init the convs before running the
# PCA, and need to change the PCA subroutine to take a network and perhaps
# allow for batched inputs, to handle larger datasets.
if FLAGS.num_conv_layers > 0:
conv = utils.ConvParameters()
conv.name = "conv1"
conv.in_channels = 1
conv.out_channels = 128
conv.num_outputs = 128 * 14 * 14
network_parameters.conv_parameters.append(conv)
# defaults for the rest: 5x5,stride 1, relu, maxpool 2x2,stride 2.
# insize 28x28, bias, stddev 0.1, non-trainable.
if FLAGS.num_conv_layers > 1:
conv = network_parameters.ConvParameters()
conv.name = "conv2"
conv.in_channels = 128
conv.out_channels = 128
conv.num_outputs = 128 * 7 * 7
conv.in_size = 14
# defaults for the rest: 5x5,stride 1, relu, maxpool 2x2,stride 2.
# bias, stddev 0.1, non-trainable.
network_parameters.conv_parameters.append(conv)
if FLAGS.num_conv_layers > 2:
raise ValueError(
"Currently --num_conv_layers must be 0,1 or 2."
"Manually create a network_parameters proto for more.")
if FLAGS.projection_dimensions > 0:
network_parameters.projection_type = "PCA"
network_parameters.projection_dimensions = FLAGS.projection_dimensions
for i in xrange(FLAGS.num_hidden_layers):
hidden = utils.LayerParameters()
hidden.name = "hidden%d" % i
hidden.num_units = FLAGS.hidden_layer_num_units
hidden.relu = True
hidden.with_bias = False
hidden.trainable = not FLAGS.freeze_bottom_layers
network_parameters.layer_parameters.append(hidden)
logits = utils.LayerParameters()
logits.name = "logits"
logits.num_units = 10
logits.relu = False
logits.with_bias = False
network_parameters.layer_parameters.append(logits)
inputs = tf.placeholder(tf.float32, [None, 784], name='inputs')
outputs, _, _ = utils.BuildNetwork(inputs, network_parameters)
def Train(mnist_train_file,
mnist_test_file,
mnist_validation_file,
network_parameters,
num_steps,
save_path,
total_rho,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"projection_dimensions": FLAGS.projection_dimensions,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
"flag_pca_eps": FLAGS.pca_eps,
"flag_pca_delta": FLAGS.pca_delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
"pca_sigma": FLAGS.pca_sigma,
})
elif FLAGS.accountant_type == "zCDP":
params.update()
with tf.device('/gpu:0'), tf.Graph().as_default(), tf.Session() as sess:
# Create the basic Mnist model.
images, labels = MnistInput(mnist_train_file, batch_size,
FLAGS.randomize)
logits, projection, training_params = utils.BuildNetwork(
images, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 10))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(
NUM_TRAINING_IMAGES)
sigma = None
pca_sigma = None
with_privacy = FLAGS.eps > 0
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
pca_sigma = FLAGS.pca_sigma
with_privacy = FLAGS.sigma > 0
elif FLAGS.accountant_type == "ZCDP":
priv_accountant = accountant.DumpzCDPAccountant()
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" %
FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
varsigma = tf.placeholder(tf.float32, shape=[])
init_ops = []
if network_parameters.projection_type == "PCA":
with tf.variable_scope("pca"):
# Compute differentially private PCA.
all_data, _ = MnistInput(mnist_train_file, NUM_TRAINING_IMAGES,
False)
pca_projection = dp_pca.ComputeDPPrincipalProjection(
all_data, network_parameters.projection_dimensions,
gaussian_sanitizer, [FLAGS.pca_eps, FLAGS.pca_delta],
pca_sigma)
assign_pca_proj = tf.assign(projection, pca_projection)
init_ops.append(assign_pca_proj)
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
with_privacy = True
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
varsigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
print("No privacy")
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results" + datetime.datetime.now().strftime(
'%Y-%m-%d-%H-%M-%S') + ".json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
#
previous_epoch = -1
rho_tracking = [0]
validation_accuracy_list = []
previous_validaccuracy = 0
tracking_sigma = []
curr_sigma = 10
# total budget
rhototal = total_rho
for step in range(num_steps):
epoch = step // np.ceil(lots_per_epoch)
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
old_sigma = curr_sigma
#validation based decay
#period=10, threshold=0.01, decay_factor=0.9
period = 10
decay_factor = 0.8
threshold = 0.01
m = 5
if epoch - previous_epoch == 1 and (
epoch + 1) % period == 0: #checking epoch
current_validaccuracy = sum(validation_accuracy_list[-m:]) / m
if current_validaccuracy - previous_validaccuracy < threshold:
curr_sigma = decay_factor * curr_sigma
previous_validaccuracy = current_validaccuracy
if old_sigma != curr_sigma:
print(curr_sigma)
#for tracking by epoch
if epoch - previous_epoch == 1:
tracking_sigma.append(curr_sigma)
rho_tracking.append(rho_tracking[-1] + 1 /
(2.0 * curr_sigma**2))
previous_epoch = epoch
if with_privacy == True and rho_tracking[-1] > rhototal:
print("stop at epoch%d" % epoch)
break
print(rho_tracking)
print(rho_tracking)
print(tracking_sigma)
for _ in range(FLAGS.batches_per_lot):
_ = sess.run(
[gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta,
varsigma: curr_sigma
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or
terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
print(spent_eps_deltas)
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(mnist_train_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=True,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
mnist_test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
validation_accuracy, mistakes = Eval(
mnist_validation_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("validation_accuracy: %.2f\n" %
validation_accuracy)
validation_accuracy_list.append(validation_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
def Train(train_file,
test_file,
network_parameters,
num_steps,
save_path,
total_rho,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = NUM_TRAINING_IMAGES
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
})
with tf.device('/gpu:0'), tf.Graph().as_default(), tf.Session() as sess:
#print_csv_tfrecords.print_tfrecords(train_file)
features, labels = DataInput(train_file, batch_size, False)
print("network_parameters.input_size", network_parameters.input_size)
logits, projection, training_params = utils.BuildNetwork(
features, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, LABEL_SIZE))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(
NUM_TRAINING_IMAGES)
sigma = None
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
elif FLAGS.accountant_type == "ZDCP":
priv_accountant = accountant.DumpzCDPAccountant()
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" %
FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
varsigma = tf.placeholder(tf.float32, shape=[])
init_ops = []
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
with_privacy = True
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
varsigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
print("No privacy")
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
curr_sigma = 0
previous_epoch = -1
rho_tracking = [0]
for step in range(num_steps):
epoch = step // lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
if with_privacy:
old_sigma = curr_sigma
#total budget
rhototal = total_rho
curr_sigma = get_current_sigma(epoch)
if step % 100 == 0:
print(curr_sigma)
print(rho_tracking[-1])
if epoch - previous_epoch == 1:
rho_tracking.append(rho_tracking[-1] + 1.0 /
(2.0 * curr_sigma**2))
previous_epoch = epoch
if with_privacy == True and rho_tracking[-1] > rhototal:
print("stop at epoch%d" % epoch)
print(rho_tracking[:-1])
break
for _ in range(FLAGS.batches_per_lot):
_ = sess.run(
[gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta,
varsigma: curr_sigma
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(
train_file,
network_parameters,
num_testing_images=NUM_TRAINING_IMAGES,
randomize=False,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
print(rho_tracking[:-1])
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(train_file,
network_parameters,
num_testing_images=NUM_TRAINING_IMAGES,
randomize=False,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
