def main(argv):
del argv # argv is not used.
assert FLAGS.N is not None, 'Flag N is missing.'
assert FLAGS.batch_size is not None, 'Flag batch_size is missing.'
assert FLAGS.noise_multiplier is not None, 'Flag noise_multiplier is missing.'
assert FLAGS.epochs is not None, 'Flag epochs is missing.'
compute_dp_sgd_privacy(FLAGS.N, FLAGS.batch_size, FLAGS.noise_multiplier,
FLAGS.epochs, FLAGS.delta)
def test_compute_dpsgd_noise_multiplier(self, num_train, epsilon, delta,
epochs, batch_size, tolerance):
noise_multiplier = multinomial_logistic.compute_dpsgd_noise_multiplier(
num_train, epsilon, delta, epochs, batch_size, tolerance)
epsilon_lower_bound = compute_dp_sgd_privacy_lib.compute_dp_sgd_privacy(
num_train, batch_size, noise_multiplier + tolerance, epochs,
delta)[0]
epsilon_upper_bound = compute_dp_sgd_privacy_lib.compute_dp_sgd_privacy(
num_train, batch_size, noise_multiplier - tolerance, epochs,
delta)[0]
self.assertLess(epsilon_lower_bound, epsilon)
self.assertLess(epsilon, epsilon_upper_bound)
def test_compute_dp_sgd_privacy(self, n, batch_size, noise_multiplier,
epochs, delta, expected_eps,
expected_order):
eps, order = compute_dp_sgd_privacy_lib.compute_dp_sgd_privacy(
n, batch_size, noise_multiplier, epochs, delta)
self.assertAlmostEqual(eps, expected_eps)
self.assertEqual(order, expected_order)
# We perform an additional sanity check on the hard-coded test values.
# We do a back-of-the-envelope calculation to obtain a lower bound.
# Specifically, we make the approximation that subsampling a q-fraction is
# equivalent to multiplying noise scale by 1/q.
# This is only an approximation, but can be justified by the central limit
# theorem in the Gaussian Differential Privacy framework; see
# https://arxiv.org/1911.11607
# The approximation error is one-sided and provides a lower bound, which is
# the basis of this sanity check. This is confirmed in the above paper.
q = batch_size / n
steps = epochs * n / batch_size
sigma = noise_multiplier * math.sqrt(steps) / q
# We compute the optimal guarantee for Gaussian
# using https://arxiv.org/abs/1805.06530 Theorem 8 (in v2).
low_delta = .5 * math.erfc((eps * sigma - .5 / sigma) / math.sqrt(2))
if eps < 100: # Skip this if it causes overflow; error is minor.
low_delta -= math.exp(eps) * .5 * math.erfc(
(eps * sigma + .5 / sigma) / math.sqrt(2))
self.assertLessEqual(low_delta, delta)
def test_compute_dp_sgd_privacy(self, n, batch_size, noise_multiplier,
epochs, delta, expected_eps,
expected_order):
eps, order = compute_dp_sgd_privacy_lib.compute_dp_sgd_privacy(
n, batch_size, noise_multiplier, epochs, delta)
self.assertAlmostEqual(eps, expected_eps)
self.assertAlmostEqual(order, expected_order)
def main(unused_argv):
logging.set_verbosity(logging.INFO)
if FLAGS.dpsgd and FLAGS.batch_size % FLAGS.microbatches != 0:
raise ValueError(
'Number of microbatches should divide evenly batch_size')
# Instantiate the tf.Estimator.
run_config = tf.estimator.tpu.RunConfig(master=FLAGS.master)
mnist_classifier = tf.estimator.tpu.TPUEstimator(
train_batch_size=FLAGS.batch_size,
eval_batch_size=FLAGS.batch_size,
model_fn=cnn_model_fn,
model_dir=FLAGS.model_dir,
config=run_config)
# Training loop.
steps_per_epoch = 60000 // FLAGS.batch_size
eval_steps_per_epoch = 10000 // FLAGS.batch_size
for epoch in range(1, FLAGS.epochs + 1):
start_time = time.time()
# Train the model for one epoch.
mnist_classifier.train(input_fn=common.make_input_fn('train',
FLAGS.batch_size /
FLAGS.cores,
tpu=True),
steps=steps_per_epoch)
end_time = time.time()
logging.info('Epoch %d time in seconds: %.2f', epoch,
end_time - start_time)
# Evaluate the model and print results
eval_results = mnist_classifier.evaluate(input_fn=common.make_input_fn(
'test', FLAGS.batch_size / FLAGS.cores, 1, tpu=True),
steps=eval_steps_per_epoch)
test_accuracy = eval_results['accuracy']
print('Test accuracy after %d epochs is: %.3f' %
(epoch, test_accuracy))
# Compute the privacy budget expended.
if FLAGS.dpsgd:
if FLAGS.noise_multiplier > 0.0:
# Due to the nature of Gaussian noise, the actual noise applied is
# equal to FLAGS.noise_multiplier * sqrt(number of cores).
eps, _ = compute_dp_sgd_privacy_lib.compute_dp_sgd_privacy(
60000, FLAGS.batch_size,
FLAGS.noise_multiplier * math.sqrt(FLAGS.cores), epoch,
1e-5)
print('For delta=1e-5, the current epsilon is: %.2f' % eps)
else:
print('Trained with DP-SGD but with zero noise.')
else:
print('Trained with vanilla non-private SGD optimizer')
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
# Load training and test data.
train_data, train_labels, test_data, test_labels = load_mnist()
# Instantiate the tf.Estimator.
mnist_classifier = tf.estimator.Estimator(model_fn=cnn_model_fn)
# Create tf.Estimator input functions for the training and test data.
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={'x': train_data},
y=train_labels,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.epochs,
shuffle=True)
eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={'x': test_data},
y=test_labels,
num_epochs=1,
shuffle=False)
# Training loop.
steps_per_epoch = 60000 // FLAGS.batch_size
for epoch in range(1, FLAGS.epochs + 1):
start_time = time.time()
# Train the model for one epoch.
mnist_classifier.train(input_fn=train_input_fn, steps=steps_per_epoch)
end_time = time.time()
print('Epoch %d time in seconds: %.2f', epoch, end_time - start_time)
# Evaluate the model and print results
eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
test_accuracy = eval_results['accuracy']
print('Test accuracy after %d epochs is: %.3f' %
(epoch, test_accuracy))
# Compute the privacy budget expended.
if FLAGS.dpsgd:
if FLAGS.noise_multiplier > 0.0:
eps, _ = compute_dp_sgd_privacy_lib.compute_dp_sgd_privacy(
60000, FLAGS.batch_size, FLAGS.noise_multiplier, epoch,
1e-5)
print('For delta=1e-5, the current epsilon is: %.2f' % eps)
else:
print('Trained with DP-SGD but with zero noise.')
else:
print('Trained with vanilla non-private SGD optimizer')
def main(unused_argv):
logging.set_verbosity(logging.INFO)
if FLAGS.dpsgd and FLAGS.batch_size % FLAGS.microbatches != 0:
raise ValueError(
'Number of microbatches should divide evenly batch_size')
# Instantiate the tf.Estimator.
mnist_classifier = tf.estimator.Estimator(model_fn=cnn_model_fn,
model_dir=FLAGS.model_dir)
# Training loop.
steps_per_epoch = 60000 // FLAGS.batch_size
for epoch in range(1, FLAGS.epochs + 1):
start_time = time.time()
# Train the model for one epoch.
mnist_classifier.train(input_fn=common.make_input_fn(
'train', FLAGS.batch_size),
steps=steps_per_epoch)
end_time = time.time()
logging.info('Epoch %d time in seconds: %.2f', epoch,
end_time - start_time)
# Evaluate the model and print results
eval_results = mnist_classifier.evaluate(
input_fn=common.make_input_fn('test', FLAGS.batch_size, 1))
test_accuracy = eval_results['accuracy']
print('Test accuracy after %d epochs is: %.3f' %
(epoch, test_accuracy))
# Compute the privacy budget expended.
if FLAGS.dpsgd:
if FLAGS.noise_multiplier > 0.0:
eps, _ = compute_dp_sgd_privacy_lib.compute_dp_sgd_privacy(
60000, FLAGS.batch_size, FLAGS.noise_multiplier, epoch,
1e-5)
print('For delta=1e-5, the current epsilon is: %.2f' % eps)
else:
print('Trained with DP-SGD but with zero noise.')
else:
print('Trained with vanilla non-private SGD optimizer')
def _func(x):
result = compute_dp_sgd_privacy_lib.compute_dp_sgd_privacy(
num_train, batch_size, x, epochs, delta)
return result[0]
def train(self, feature_network_checkpoint_path=None, restore=False):
tf.global_variables_initializer().run()
if restore is True:
restore_global_id = self.load()
print("Loaded from global_id {}".format(restore_global_id))
else:
restore_global_id = -1
if feature_network_checkpoint_path is not None:
# feature
variables = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES,
scope=self.generator.scope_name + "/feature")
print(variables)
saver = tf.train.Saver(variables)
saver.restore(self.sess, feature_network_checkpoint_path)
# min max
variables = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES,
scope=self.generator.scope_name + "/attribute_addi")
print(variables)
if len(variables) > 0:
saver = tf.train.Saver(variables)
saver.restore(self.sess, feature_network_checkpoint_path)
self.summary_writer = tf.summary.FileWriter(
self.checkpoint_dir, self.sess.graph)
batch_num = self.data_feature.shape[0] // self.batch_size
global_id = 0
if (self.dp_noise_multiplier is not None and
self.dp_l2_norm_clip is not None):
if self.attr_discriminator:
# The effective noise multiplier for DP guarantee is
# 0.5*noise_multiplier because each batch of data is utilized
# twice for the two discriminators.
noise_multiplier = self.dp_noise_multiplier * 0.5
else:
noise_multiplier = self.dp_noise_multiplier
print("Using DP training")
print("The final DP parameters will be:")
compute_dp_sgd_privacy(
self.data_feature.shape[0],
self.batch_size,
noise_multiplier,
self.epoch,
self.dp_delta)
for epoch_id in tqdm(range(self.epoch)):
data_id = np.random.choice(
self.data_feature.shape[0],
size=(self.data_feature.shape[0], self.num_packing))
if global_id > restore_global_id:
if ((epoch_id + 1) % self.epoch_checkpoint_freq == 0 or
epoch_id == self.epoch - 1):
with open(self.time_path, "a") as f:
time = datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S.%f')
f.write("epoch {} starts: {}\n".format(epoch_id, time))
for batch_id in range(batch_num):
feed_dict = {}
for i in range(self.num_packing):
batch_data_id = data_id[batch_id * self.batch_size:
(batch_id + 1) * self.batch_size,
i]
batch_data_feature = self.data_feature[batch_data_id]
batch_data_attribute = self.data_attribute[batch_data_id]
batch_real_attribute_input_noise = \
self.gen_attribute_input_noise(self.batch_size)
batch_addi_attribute_input_noise = \
self.gen_attribute_input_noise(self.batch_size)
batch_feature_input_noise = \
self.gen_feature_input_noise(
self.batch_size, self.sample_time)
batch_feature_input_data = \
self.gen_feature_input_data_free(self.batch_size)
feed_dict[self.real_feature_pl_l[i]] = \
batch_data_feature
feed_dict[self.real_attribute_pl_l[i]] = \
batch_data_attribute
feed_dict[self.
g_real_attribute_input_noise_train_pl_l[i]] = \
batch_real_attribute_input_noise
feed_dict[self.
g_addi_attribute_input_noise_train_pl_l[i]] = \
batch_addi_attribute_input_noise
feed_dict[self.g_feature_input_noise_train_pl_l[i]] = \
batch_feature_input_noise
feed_dict[self.g_feature_input_data_train_pl_l[i]] = \
batch_feature_input_data
if global_id > restore_global_id:
for _ in range(self.d_rounds - 1):
self.sess.run(self.d_op, feed_dict=feed_dict)
if self.attr_discriminator is not None:
self.sess.run(self.attr_d_op, feed_dict=feed_dict)
summary_result, _ = self.sess.run(
[self.d_summary, self.d_op],
feed_dict=feed_dict)
self.summary_writer.add_summary(summary_result, global_id)
if self.attr_discriminator is not None:
summary_result, _ = self.sess.run(
[self.attr_d_summary, self.attr_d_op],
feed_dict=feed_dict)
self.summary_writer.add_summary(
summary_result, global_id)
for _ in range(self.g_rounds - 1):
self.sess.run(self.g_op, feed_dict=feed_dict)
summary_result, _ = self.sess.run(
[self.g_summary, self.g_op],
feed_dict=feed_dict)
self.summary_writer.add_summary(summary_result, global_id)
if (batch_id + 1) % self.vis_freq == 0:
self.visualize(epoch_id, batch_id, global_id)
global_id += 1
if global_id - 1 > restore_global_id:
if ((epoch_id + 1) % self.epoch_checkpoint_freq == 0 or
epoch_id == self.epoch - 1):
self.visualize(epoch_id, -1, global_id - 1)
self.save(global_id - 1)
with open(self.time_path, "a") as f:
time = datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S.%f')
f.write("epoch {} ends: {}\n".format(epoch_id, time))
if (epoch_id + 1) % self.extra_checkpoint_freq == 0:
saver = tf.train.Saver()
checkpoint_dir = os.path.join(
self.checkpoint_dir,
"epoch_id-{}".format(epoch_id))
self.save(global_id - 1, saver, checkpoint_dir)
dpsgd_subopt_params = {}
dpsgd_subopt_params["num_epochs"] = 20
dpsgd_subopt_params["clip_norm"] = 1e-3
dpsgd_subopt_params["learning_rate"] = 0.1
dpsgd_subopt_params["batch_size"] = 64
dpsgd_subopt_params["noise_multiplier"] = 1.150390625
# set remaining parameters
eps = np.log(3)
delta = 1e-5
m_range = np.linspace(1000, 2000, 5)
num_trials = 3
# verify dpsgd hyperparameters
compute_dp_sgd_privacy(len(synthetic_x), dpsgd_opt_params["batch_size"],
dpsgd_opt_params["noise_multiplier"],
dpsgd_opt_params["num_epochs"], delta)
compute_dp_sgd_privacy(len(synthetic_x), dpsgd_subopt_params["batch_size"],
dpsgd_subopt_params["noise_multiplier"],
dpsgd_subopt_params["num_epochs"], delta)
# run r2 experiments and store r2 quantiles and times
results = experiment.run_trials(synthetic_x, synthetic_y, eps, delta, m_range,
dpsgd_opt_params, dpsgd_subopt_params,
num_trials)
# plot r2 experiments results
experiment.plot_r2(results[0], results[1], results[2], m_range, num_trials,
"synthetic_r2")
experiment.plot_time(results[3], m_range, num_trials, "synthetic_time", True)