def GAN_solvers(D_loss, G_loss, learning_rate, batch_size, total_examples,
l2norm_bound, batches_per_lot, sigma, dp=False):
"""
Optimizers
"""
discriminator_vars = [v for v in tf.trainable_variables() if v.name.startswith('discriminator')]
generator_vars = [v for v in tf.trainable_variables() if v.name.startswith('generator')]
if dp:
print('Using differentially private SGD to train discriminator!')
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
priv_accountant = accountant.GaussianMomentsAccountant(total_examples)
clip = True
l2norm_bound = l2norm_bound/batch_size
batches_per_lot = 1
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant,
[l2norm_bound, clip])
# the trick is that we need to calculate the gradient with respect to
# each example in the batch, during the DP SGD step
D_solver = dp_optimizer.DPGradientDescentOptimizer(learning_rate,
[eps, delta],
sanitizer=gaussian_sanitizer,
sigma=sigma,
batches_per_lot=batches_per_lot).minimize(D_loss, var_list=discriminator_vars)
else:
D_loss_mean_over_batch = tf.reduce_mean(D_loss)
D_solver = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(D_loss_mean_over_batch, var_list=discriminator_vars)
priv_accountant = None
G_loss_mean_over_batch = tf.reduce_mean(G_loss)
G_solver = tf.train.AdamOptimizer().minimize(G_loss_mean_over_batch, var_list=generator_vars)
return D_solver, G_solver, priv_accountant
def __init__(
self,
sess,
epoch,
z_dim,
batch_size,
sigma,
clipping,
delta,
epsilon,
learning_rate,
dataset_name,
base_dir,
result_dir):
self.accountant = accountant.GaussianMomentsAccountant(60000)
self.sess = sess
self.dataset_name = dataset_name
self.base_dir = base_dir
self.result_dir = result_dir
self.epoch = epoch
self.batch_size = batch_size
self.sigma = sigma
self.clipping = clipping
self.delta = delta
self.epsilon = epsilon
self.learning_rate = learning_rate
if dataset_name == 'mnist' or dataset_name == 'fashion-mnist':
# parameters
self.input_height = 28
self.input_width = 28
self.output_height = 28
self.output_width = 28
self.z_dim = z_dim # dimension of noise-vector
self.y_dim = 10 # dimension of condition-vector (label)
self.c_dim = 1
# train
self.beta1 = 0.5
# test
self.sample_num = 64 # number of generated images to be saved
# load mnist
self.data_X, self.data_y = load_mnist(self.dataset_name, self.base_dir)
# get number of batches for a single epoch
self.num_batches = len(self.data_X) // self.batch_size
else:
raise NotImplementedError
def DPSGD(sigma, l2norm_bound, learning_rate, total_examples):
import tensorflow as tf
from differential_privacy.dp_sgd.dp_optimizer import dp_optimizer
from differential_privacy.dp_sgd.dp_optimizer import sanitizer
from differential_privacy.privacy_accountant.tf import accountant
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
priv_accountant = accountant.GaussianMomentsAccountant(total_examples)
clip = True
batches_per_lot = 1
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [l2norm_bound, clip])
return dp_optimizer.DPGradientDescentOptimizer(
learning_rate, [eps, delta],
sanitizer=gaussian_sanitizer,
sigma=sigma,
batches_per_lot=batches_per_lot)
def runTensorFlow(sigma, clippingValue, batchSize, epsilon, delta, iteration):
h_dim = 128
Z_dim = 100
# Initializations for a two-layer discriminator network
mnist = input_data.read_data_sets(
baseDir + "our_dp_conditional_gan_mnist/mnist_dataset", one_hot=True)
X_dim = mnist.train.images.shape[1]
y_dim = mnist.train.labels.shape[1]
X = tf.placeholder(tf.float32, shape=[None, X_dim])
y = tf.placeholder(tf.float32, shape=[None, y_dim])
D_W1 = tf.Variable(xavier_init([X_dim + y_dim, h_dim]))
D_b1 = tf.Variable(tf.zeros(shape=[h_dim]))
D_W2 = tf.Variable(xavier_init([h_dim, 1]))
D_b2 = tf.Variable(tf.zeros(shape=[1]))
theta_D = [D_W1, D_W2, D_b1, D_b2]
# Initializations for a two-layer genrator network
Z = tf.placeholder(tf.float32, shape=[None, Z_dim])
G_W1 = tf.Variable(xavier_init([Z_dim + y_dim, h_dim]))
G_b1 = tf.Variable(tf.zeros(shape=[h_dim]))
G_W2 = tf.Variable(xavier_init([h_dim, X_dim]))
G_b2 = tf.Variable(tf.zeros(shape=[X_dim]))
theta_G = [G_W1, G_W2, G_b1, G_b2]
# Delete all Flags
del_all_flags(tf.flags.FLAGS)
# Set training parameters
tf.flags.DEFINE_string('f', '', 'kernel')
tf.flags.DEFINE_float("lr", 0.1, "start learning rate")
tf.flags.DEFINE_float("end_lr", 0.052, "end learning rate")
tf.flags.DEFINE_float(
"lr_saturate_epochs", 10000,
"learning rate saturate epochs; set to 0 for a constant"
"learning rate of --lr.")
tf.flags.DEFINE_integer("batch_size", batchSize,
"The training batch size.")
tf.flags.DEFINE_integer("batches_per_lot", 1, "Number of batches per lot.")
tf.flags.DEFINE_integer(
"num_training_steps", 100000, "The number of training"
"steps. This counts number of lots.")
# Flags that control privacy spending during training
tf.flags.DEFINE_float("target_delta", delta, "Maximum delta for"
"--terminate_based_on_privacy.")
tf.flags.DEFINE_float(
"sigma", sigma, "Noise sigma, used only if accountant_type"
"is Moments")
tf.flags.DEFINE_string(
"target_eps", str(epsilon),
"Log the privacy loss for the target epsilon's. Only"
"used when accountant_type is Moments.")
tf.flags.DEFINE_float("default_gradient_l2norm_bound", clippingValue,
"norm clipping")
FLAGS = tf.flags.FLAGS
# Set accountant type to GaussianMomentsAccountant
NUM_TRAINING_IMAGES = 60000
priv_accountant = accountant.GaussianMomentsAccountant(NUM_TRAINING_IMAGES)
# Sanitizer
batch_size = FLAGS.batch_size
clipping_value = FLAGS.default_gradient_l2norm_bound
# clipping_value = tf.placeholder(tf.float32)
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [clipping_value / batch_size, True])
# Instantiate the Generator Network
G_sample = generator(Z, y, theta_G)
# Instantiate the Discriminator Network
D_real, D_logit_real = discriminator(X, y, theta_D)
D_fake, D_logit_fake = discriminator(G_sample, y, theta_D)
# Discriminator loss for real data
D_loss_real = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits( \
logits=D_logit_real, \
labels=tf.ones_like(D_logit_real)), \
[0])
# Discriminator loss for fake data
D_loss_fake = tf.reduce_mean( \
tf.nn.sigmoid_cross_entropy_with_logits( \
logits=D_logit_fake, \
labels=tf.zeros_like(D_logit_fake)), [0])
# Generator loss
G_loss = tf.reduce_mean( \
tf.nn.sigmoid_cross_entropy_with_logits( \
logits=D_logit_fake, labels=tf.ones_like(D_logit_fake)) \
, [0])
# ------------------------------------------------------------------------------
"""
minimize_ours :
Our method (Clipping the gradients of loss on real data and making
them noisy + Clipping the gradients of loss on fake data) is
implemented in this function .
It can be found in the following directory:
differential_privacy/dp_sgd/dp_optimizer/dp_optimizer.py'
"""
lr = tf.placeholder(tf.float32)
sigma = FLAGS.sigma
# Generator optimizer
G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G)
# Discriminator Optimizer
D_solver = dp_optimizer.DPGradientDescentOptimizer( \
lr, [None, None], \
gaussian_sanitizer, \
sigma=sigma, \
batches_per_lot= \
FLAGS.batches_per_lot). \
minimize_ours( \
D_loss_real, \
D_loss_fake, \
var_list=theta_D)
# ------------------------------------------------------------------------------
# Set output directory
resultDir = baseDir + "out/"
if not os.path.exists(resultDir):
os.makedirs(resultDir)
resultPath = resultDir + "/run_{}_bs_{}_s_{}_c_{}_d_{}_e_{}".format( \
iteration, \
batch_size, \
sigma, \
clipping_value, \
FLAGS.target_delta, FLAGS.target_eps)
if not os.path.exists(resultPath):
os.makedirs(resultPath)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
max_target_eps = max(target_eps)
gpu_options = tf.GPUOptions(visible_device_list="0, 1")
# Main Session
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)) as sess:
init = tf.initialize_all_variables()
sess.run(init)
step = 0
# Is true when the spent privacy budget exceeds the target budget
should_terminate = False
# Main loop
while (step < FLAGS.num_training_steps and should_terminate == False):
epoch = step
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr, \
FLAGS.lr_saturate_epochs, epoch)
eps = compute_epsilon(FLAGS, (step + 1), sigma * clipping_value)
# Save the generated images every 50 steps
if step % 50 == 0:
print("step : " + str(step) + " eps : " + str(eps))
n_sample = 10
Z_sample = sample_Z(n_sample, Z_dim)
y_sample = np.zeros(shape=[n_sample, 10])
y_sample[0, 0] = 1
y_sample[1, 1] = 1
y_sample[2, 2] = 1
y_sample[3, 3] = 1
y_sample[4, 4] = 1
y_sample[5, 5] = 1
y_sample[6, 6] = 1
y_sample[7, 7] = 1
y_sample[8, 8] = 1
y_sample[9, 9] = 1
samples = sess.run(G_sample,
feed_dict={
Z: Z_sample,
y: y_sample
})
fig = plot(samples)
plt.savefig(
(resultPath + "/step_{}.png").format(str(step).zfill(3)),
bbox_inches='tight')
plt.close(fig)
X_mb, y_mb = mnist.train.next_batch(batch_size, shuffle=True)
Z_sample = sample_Z(batch_size, Z_dim)
# Update the discriminator network
_, D_loss_real_curr, D_loss_fake_curr = sess.run([D_solver, D_loss_real, D_loss_fake], \
feed_dict={X: X_mb, \
Z: Z_sample, \
y: y_mb, \
lr: curr_lr})
# Update the generator network
_, G_loss_curr = sess.run([G_solver, G_loss],
feed_dict={
Z: Z_sample,
y: y_mb,
lr: curr_lr
})
if (eps > max_target_eps):
print("TERMINATE!!!!")
print("Termination Step : " + str(step))
should_terminate = True
for i in range(0, 10):
n_sample = 10
Z_sample = sample_Z(n_sample, Z_dim)
y_sample = np.zeros(shape=[n_sample, y_dim])
y_sample[0, 0] = 1
y_sample[1, 1] = 1
y_sample[2, 2] = 1
y_sample[3, 3] = 1
y_sample[4, 4] = 1
y_sample[5, 5] = 1
y_sample[6, 6] = 1
y_sample[7, 7] = 1
y_sample[8, 8] = 1
y_sample[9, 9] = 1
samples = sess.run(G_sample,
feed_dict={
Z: Z_sample,
y: y_sample
})
fig = plot(samples)
plt.savefig((resultPath + "/Final_step_{}.png").format(
str(i).zfill(3)),
bbox_inches='tight')
plt.close(fig)
n_class = np.zeros(10)
n_class[0] = 5923
n_class[1] = 6742
n_class[2] = 5958
n_class[3] = 6131
n_class[4] = 5842
n_class[5] = 5421
n_class[6] = 5918
n_class[7] = 6265
n_class[8] = 5851
n_class[9] = 5949
n_image = int(sum(n_class))
image_lables = np.zeros(shape=[n_image, len(n_class)])
image_cntr = 0
for class_cntr in np.arange(len(n_class)):
for cntr in np.arange(n_class[class_cntr]):
image_lables[image_cntr, class_cntr] = 1
image_cntr += 1
Z_sample = sample_Z(n_image, Z_dim)
images = sess.run(G_sample,
feed_dict={
Z: Z_sample,
y: image_lables
})
X_test, Y_test = loadlocal_mnist(
images_path=baseDir + "our_dp_conditional_gan_mnist/" +
'mnist_dataset/t10k-images.idx3-ubyte',
labels_path=baseDir + "our_dp_conditional_gan_mnist/" +
'mnist_dataset/t10k-labels.idx1-ubyte')
Y_test = [int(y) for y in Y_test]
resultFile = open(resultPath + "/" + "results.txt", "w")
print("Binarizing the labels ...")
classes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Y_test = label_binarize(Y_test, classes=classes)
print(
"\n################# Logistic Regression #######################"
)
print(" Classifying ...")
Y_score = classify(images,
image_lables,
X_test,
"lr",
random_state_value=30)
print(" Computing ROC ...")
false_positive_rate, true_positive_rate, roc_auc = compute_fpr_tpr_roc(
Y_test, Y_score)
print(" AUROC: " + str(roc_auc["micro"]))
resultFile.write("LR AUROC: " + str(roc_auc["micro"]) + "\n")
print(
"\n################# Multi-layer Perceptron #######################"
)
print(" Classifying ...")
Y_score = classify(images,
image_lables,
X_test,
"mlp",
random_state_value=30)
print(" Computing ROC ...")
false_positive_rate, true_positive_rate, roc_auc = compute_fpr_tpr_roc(
Y_test, Y_score)
print(" AUROC: " + str(roc_auc["micro"]))
resultFile.write("MLP AUROC: " + str(roc_auc["micro"]) + "\n")
step = FLAGS.num_training_steps
break
step = step + 1
def Train(sess,
train_images,
train_labels,
mnist_test_file,
network_parameters,
num_steps,
save_path,
training_params,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"projection_dimensions": FLAGS.projection_dimensions,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
"flag_pca_eps": FLAGS.pca_eps,
"flag_pca_delta": FLAGS.pca_delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
"pca_sigma": FLAGS.pca_sigma,
})
# Create the basic Mnist model.
images = tf.get_default_graph().get_tensor_by_name("images:0")
labels = tf.get_default_graph().get_tensor_by_name("labels:0")
logits = tf.get_default_graph().get_tensor_by_name("logits:0")
projection = tf.get_default_graph().get_tensor_by_name("projection:0")
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 10))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(NUM_TRAINING_IMAGES)
sigma = None
pca_sigma = None
with_privacy = FLAGS.eps > 0
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
pca_sigma = FLAGS.pca_sigma
with_privacy = FLAGS.sigma > 0
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" % FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant,
[network_parameters.default_gradient_l2norm_bound / batch_size, True])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var]["gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(var,
sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
init_ops = []
if network_parameters.projection_type == "PCA":
with tf.variable_scope("pca"):
# Compute differentially private PCA.
all_data = tf.constant(train_images, dtype=tf.float32)
pca_projection = dp_pca.ComputeDPPrincipalProjection(
all_data, network_parameters.projection_dimensions,
gaussian_sanitizer, [FLAGS.pca_eps, FLAGS.pca_delta],
pca_sigma)
assign_pca_proj = tf.assign(projection, pca_projection)
init_ops.append(assign_pca_proj)
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
sigma=sigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = np.array([float(s) for s in FLAGS.target_eps.split(",")])
target_eps = -np.sort(-target_eps)
index_eps_to_save = 0
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
for step in xrange(num_steps):
epoch = step / lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
lot = np.random.choice(np.arange(NUM_TRAINING_IMAGES),
lot_size,
replace=False)
#print lot.shape
images_lot = train_images[lot, :]
labels_lot = train_labels[lot]
for i in xrange(FLAGS.batches_per_lot):
_ = sess.run(
[gd_op],
feed_dict={
lr:
curr_lr,
eps:
curr_eps,
delta:
FLAGS.delta,
images:
images_lot[i * FLAGS.batch_size:(i + 1) *
FLAGS.batch_size, :],
labels:
labels_lot[i * FLAGS.batch_size:(i + 1) * FLAGS.batch_size]
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
while index_eps_to_save < len(
spent_eps_deltas
) and spent_eps_deltas[index_eps_to_save][1] < FLAGS.delta:
saver.save(
sess,
save_path=save_path + "/eps" +
str(spent_eps_deltas[index_eps_to_save][0]) +
"_delta" +
'%.2g' % spent_eps_deltas[index_eps_to_save][1] +
"_pcasigma" + str(FLAGS.pca_sigma) + "_sigma" +
str(FLAGS.sigma) + "/ckpt")
index_eps_to_save += 1
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
pred_train = np.argmax(predict(sess, train_images), axis=1)
train_accuracy = np.mean(pred_train == train_labels)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
# test_accuracy, mistakes = Eval(mnist_test_file, network_parameters,
# num_testing_images=NUM_TESTING_IMAGES,
# randomize=False, load_path=save_path,
# save_mistakes=FLAGS.save_mistakes)
# sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
# "test_accuracy": test_accuracy,
# "mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
# "test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
# "mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
#Flags that control privacy spending during training
tf.flags.DEFINE_float("target_delta", 1e-5, "Maximum delta for"
"--terminate_based_on_privacy.")
tf.flags.DEFINE_float("sigma", 2, "Noise sigma, used only if accountant_type"
"is Moments")
tf.flags.DEFINE_string(
"target_eps", "9.6", "Log the privacy loss for the target epsilon's. Only"
"used when accountant_type is Moments.")
tf.flags.DEFINE_float("default_gradient_l2norm_bound", 4, "norm clipping")
FLAGS = tf.flags.FLAGS
# Set accountant type to GaussianMomentsAccountant
NUM_TRAINING_IMAGES = 60000
priv_accountant = accountant.GaussianMomentsAccountant(NUM_TRAINING_IMAGES)
#Sanitizer
batch_size = FLAGS.batch_size
clipping_value = FLAGS.default_gradient_l2norm_bound
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [clipping_value / batch_size, True])
#Instantiate the Generator Network
G_sample = generator(Z)
#Instantiate the Discriminator Network
D_real, D_logit_real = discriminator(X)
D_fake, D_logit_fake = discriminator(G_sample)
# Discriminator loss for real data
def Train(cifar_train_file,
mnist_test_file,
network_parameters,
num_steps,
save_path,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
cifar_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
params.update({"sigma": FLAGS.sigma})
with tf.Graph().as_default(), tf.Session() as sess, tf.device('/cpu:0'):
# Create the basic Cifar model.
images, labels = CifarInput(cifar_train_file, batch_size,
FLAGS.randomize)
logits, projection, training_params = utils.BuildNetwork(
images, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 100))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
with_privacy = FLAGS.sigma > 0
with_privacy = False
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
init_ops = []
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
sigma=sigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
for step in range(num_steps):
epoch = step / lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
for _ in range(FLAGS.batches_per_lot):
_ = sess.run([gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or
terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(cifar_train_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=True,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
mnist_test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
print("\nTERMINATING.\n")
break
with tf.name_scope('train'):
global_step = tf.Variable(
0, dtype=tf.int32, trainable=False, name='global_step')
loss_critic_real = - tf.reduce_mean(critic_real)
loss_critic_fake = tf.reduce_mean(critic_fake)
loss_critic = loss_critic_real + loss_critic_fake
critic_vars = [x for x in tf.trainable_variables()
if x.name.startswith('critic')]
if FLAGS.with_privacy:
# assert FLAGS.sigma > 0, 'Sigma has to be positive when with_privacy=True'
with tf.name_scope('privacy_accountant'):
if use_moments_accountant:
# Moments accountant introduced in (https://arxiv.org/abs/1607.00133)
# we use same implementation of
# https://github.com/tensorflow/models/blob/master/research/differential_privacy/privacy_accountant/tf/accountant.py
priv_accountant = accountant.GaussianMomentsAccountant(
num_examples)
else:
# AmortizedAccountant which tracks the privacy spending in the amortized way.
# It uses privacy amplication via sampling to compute the privacyspending for each
# batch and strong composition (specialized for Gaussian noise) for
# accumulate the privacy spending (http://arxiv.org/pdf/1405.7085v2.pdf)
# we use the implementation of
# https://github.com/tensorflow/models/blob/master/research/differential_privacy/privacy_accountant/tf/accountant.py
priv_accountant = accountant.AmortizedAccountant(
num_examples)
# per-example Gradient l_2 norm bound.
example_gradient_l2norm_bound = FLAGS.gradient_l2norm_bound / FLAGS.batch_size
# Gaussian sanitizer, will enforce differential privacy by clipping the gradient-per-example.
# Add gaussian noise, and sum the noisy gradients at each weight update step.
def Train(mnist_train_file,
mnist_test_file,
network_parameters,
num_steps,
save_path,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"projection_dimensions": FLAGS.projection_dimensions,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
"flag_pca_eps": FLAGS.pca_eps,
"flag_pca_delta": FLAGS.pca_delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
"pca_sigma": FLAGS.pca_sigma,
})
with tf.Graph().as_default(), tf.Session() as sess, tf.device('/cpu:0'):
# Create the basic Mnist model.
images, labels = MnistInput(mnist_train_file, batch_size,
FLAGS.randomize)
logits, projection, training_params = utils.BuildNetwork(
images, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 10))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(
NUM_TRAINING_IMAGES)
sigma = None
pca_sigma = None
with_privacy = FLAGS.eps > 0
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
pca_sigma = FLAGS.pca_sigma
with_privacy = FLAGS.sigma > 0
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" %
FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
init_ops = []
if network_parameters.projection_type == "PCA":
with tf.variable_scope("pca"):
# Compute differentially private PCA.
all_data, _ = MnistInput(mnist_train_file, NUM_TRAINING_IMAGES,
False)
pca_projection = dp_pca.ComputeDPPrincipalProjection(
all_data, network_parameters.projection_dimensions,
gaussian_sanitizer, [FLAGS.pca_eps, FLAGS.pca_delta],
pca_sigma)
assign_pca_proj = tf.assign(projection, pca_projection)
init_ops.append(assign_pca_proj)
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
sigma=sigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
for step in xrange(num_steps):
epoch = step / lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
for _ in xrange(FLAGS.batches_per_lot):
_ = sess.run([gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or
terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(mnist_train_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=True,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
mnist_test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
network_parameters = utils.NetworkParameters()
# If the ASCII proto isn't specified, then construct a config protobuf based
# on 3 flags.
network_parameters.input_size = IMAGE_SIZE**2
network_parameters.default_gradient_l2norm_bound = (
FLAGS.default_gradient_l2norm_bound)
if FLAGS.projection_dimensions > 0 and FLAGS.num_conv_layers > 0:
raise ValueError("Currently you can't do PCA and have convolutions"
"at the same time. Pick one")
# could add support for PCA after convolutions.
# Currently BuildNetwork can build the network with conv followed by
# projection, but the PCA training works on data, rather than data run
# through a few layers. Will need to init the convs before running the
# PCA, and need to change the PCA subroutine to take a network and perhaps
# allow for batched inputs, to handle larger datasets.
if FLAGS.num_conv_layers > 0:
conv = utils.ConvParameters()
conv.name = "conv1"
conv.in_channels = 1
conv.out_channels = 128
conv.num_outputs = 128 * 14 * 14
network_parameters.conv_parameters.append(conv)
# defaults for the rest: 5x5,stride 1, relu, maxpool 2x2,stride 2.
# insize 28x28, bias, stddev 0.1, non-trainable.
if FLAGS.num_conv_layers > 1:
conv = network_parameters.ConvParameters()
conv.name = "conv2"
conv.in_channels = 128
conv.out_channels = 128
conv.num_outputs = 128 * 7 * 7
conv.in_size = 14
# defaults for the rest: 5x5,stride 1, relu, maxpool 2x2,stride 2.
# bias, stddev 0.1, non-trainable.
network_parameters.conv_parameters.append(conv)
if FLAGS.num_conv_layers > 2:
raise ValueError(
"Currently --num_conv_layers must be 0,1 or 2."
"Manually create a network_parameters proto for more.")
if FLAGS.projection_dimensions > 0:
network_parameters.projection_type = "PCA"
network_parameters.projection_dimensions = FLAGS.projection_dimensions
for i in xrange(FLAGS.num_hidden_layers):
hidden = utils.LayerParameters()
hidden.name = "hidden%d" % i
hidden.num_units = FLAGS.hidden_layer_num_units
hidden.relu = True
hidden.with_bias = False
hidden.trainable = not FLAGS.freeze_bottom_layers
network_parameters.layer_parameters.append(hidden)
logits = utils.LayerParameters()
logits.name = "logits"
logits.num_units = 10
logits.relu = False
logits.with_bias = False
network_parameters.layer_parameters.append(logits)
inputs = tf.placeholder(tf.float32, [None, 784], name='inputs')
outputs, _, _ = utils.BuildNetwork(inputs, network_parameters)
def Train(mnist_train_file,
mnist_test_file,
mnist_validation_file,
network_parameters,
num_steps,
save_path,
total_rho,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"projection_dimensions": FLAGS.projection_dimensions,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
"flag_pca_eps": FLAGS.pca_eps,
"flag_pca_delta": FLAGS.pca_delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
"pca_sigma": FLAGS.pca_sigma,
})
elif FLAGS.accountant_type == "zCDP":
params.update()
with tf.device('/gpu:0'), tf.Graph().as_default(), tf.Session() as sess:
# Create the basic Mnist model.
images, labels = MnistInput(mnist_train_file, batch_size,
FLAGS.randomize)
logits, projection, training_params = utils.BuildNetwork(
images, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 10))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(
NUM_TRAINING_IMAGES)
sigma = None
pca_sigma = None
with_privacy = FLAGS.eps > 0
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
pca_sigma = FLAGS.pca_sigma
with_privacy = FLAGS.sigma > 0
elif FLAGS.accountant_type == "ZCDP":
priv_accountant = accountant.DumpzCDPAccountant()
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" %
FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
varsigma = tf.placeholder(tf.float32, shape=[])
init_ops = []
if network_parameters.projection_type == "PCA":
with tf.variable_scope("pca"):
# Compute differentially private PCA.
all_data, _ = MnistInput(mnist_train_file, NUM_TRAINING_IMAGES,
False)
pca_projection = dp_pca.ComputeDPPrincipalProjection(
all_data, network_parameters.projection_dimensions,
gaussian_sanitizer, [FLAGS.pca_eps, FLAGS.pca_delta],
pca_sigma)
assign_pca_proj = tf.assign(projection, pca_projection)
init_ops.append(assign_pca_proj)
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
with_privacy = True
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
varsigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
print("No privacy")
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results" + datetime.datetime.now().strftime(
'%Y-%m-%d-%H-%M-%S') + ".json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
#
previous_epoch = -1
rho_tracking = [0]
validation_accuracy_list = []
previous_validaccuracy = 0
tracking_sigma = []
curr_sigma = 10
# total budget
rhototal = total_rho
for step in range(num_steps):
epoch = step // np.ceil(lots_per_epoch)
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
old_sigma = curr_sigma
#validation based decay
#period=10, threshold=0.01, decay_factor=0.9
period = 10
decay_factor = 0.8
threshold = 0.01
m = 5
if epoch - previous_epoch == 1 and (
epoch + 1) % period == 0: #checking epoch
current_validaccuracy = sum(validation_accuracy_list[-m:]) / m
if current_validaccuracy - previous_validaccuracy < threshold:
curr_sigma = decay_factor * curr_sigma
previous_validaccuracy = current_validaccuracy
if old_sigma != curr_sigma:
print(curr_sigma)
#for tracking by epoch
if epoch - previous_epoch == 1:
tracking_sigma.append(curr_sigma)
rho_tracking.append(rho_tracking[-1] + 1 /
(2.0 * curr_sigma**2))
previous_epoch = epoch
if with_privacy == True and rho_tracking[-1] > rhototal:
print("stop at epoch%d" % epoch)
break
print(rho_tracking)
print(rho_tracking)
print(tracking_sigma)
for _ in range(FLAGS.batches_per_lot):
_ = sess.run(
[gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta,
varsigma: curr_sigma
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or
terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
print(spent_eps_deltas)
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(mnist_train_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=True,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
mnist_test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
validation_accuracy, mistakes = Eval(
mnist_validation_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("validation_accuracy: %.2f\n" %
validation_accuracy)
validation_accuracy_list.append(validation_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
def runTensorFlow(sigma, clippingValue, batchSize, epsilon, delta):
h_dim = 128
Z_dim = 100
# Initializations for a two-layer discriminator network
mnist = input_data.read_data_sets(
baseDir + "conditional-gan-dp-base-mnist/mnist_dataset", one_hot=True)
X_dim = mnist.train.images.shape[1]
y_dim = mnist.train.labels.shape[1]
X = tf.placeholder(tf.float32, shape=[None, X_dim])
y = tf.placeholder(tf.float32, shape=[None, y_dim])
D_W1 = tf.Variable(xavier_init([X_dim + y_dim, h_dim]))
D_b1 = tf.Variable(tf.zeros(shape=[h_dim]))
D_W2 = tf.Variable(xavier_init([h_dim, 1]))
D_b2 = tf.Variable(tf.zeros(shape=[1]))
theta_D = [D_W1, D_W2, D_b1, D_b2]
# Initializations for a two-layer genrator network
Z = tf.placeholder(tf.float32, shape=[None, Z_dim])
G_W1 = tf.Variable(xavier_init([Z_dim + y_dim, h_dim]))
G_b1 = tf.Variable(tf.zeros(shape=[h_dim]))
G_W2 = tf.Variable(xavier_init([h_dim, X_dim]))
G_b2 = tf.Variable(tf.zeros(shape=[X_dim]))
theta_G = [G_W1, G_W2, G_b1, G_b2]
# Delete all Flags
del_all_flags(tf.flags.FLAGS)
# Set training parameters
tf.flags.DEFINE_string('f', '', 'kernel')
tf.flags.DEFINE_float("lr", 0.05, "start learning rate")
tf.flags.DEFINE_float("end_lr", 0.05, "end learning rate")
tf.flags.DEFINE_float(
"lr_saturate_epochs", 0,
"learning rate saturate epochs; set to 0 for a constant"
"learning rate of --lr.")
tf.flags.DEFINE_integer("batch_size", batchSize,
"The training batch size.")
tf.flags.DEFINE_integer("batches_per_lot", 1, "Number of batches per lot.")
tf.flags.DEFINE_integer(
"num_training_steps", 1, "The number of training"
"steps. This counts number of lots.")
# Flags that control privacy spending during training
tf.flags.DEFINE_float("target_delta", delta, "Maximum delta for"
"--terminate_based_on_privacy.")
tf.flags.DEFINE_float(
"sigma", sigma, "Noise sigma, used only if accountant_type"
"is Moments")
tf.flags.DEFINE_string(
"target_eps", str(epsilon),
"Log the privacy loss for the target epsilon's. Only"
"used when accountant_type is Moments.")
tf.flags.DEFINE_float("default_gradient_l2norm_bound", clippingValue,
"norm clipping")
FLAGS = tf.flags.FLAGS
# Set accountant type to GaussianMomentsAccountant
NUM_TRAINING_IMAGES = 60000
priv_accountant = accountant.GaussianMomentsAccountant(NUM_TRAINING_IMAGES)
# Sanitizer
batch_size = FLAGS.batch_size
clipping_value = FLAGS.default_gradient_l2norm_bound
# gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(priv_accountant,
# [clipping_value / batch_size, True])
# Instantiate the Generator Network
G_sample = generator(Z, y, theta_G)
# Instantiate the Discriminator Network
D_real, D_logit_real = discriminator(X, y, theta_D)
D_fake, D_logit_fake = discriminator(G_sample, y, theta_D)
# Discriminator loss for real data
D_loss_real = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits( \
logits=D_logit_real, \
labels=tf.ones_like(D_logit_real)), \
[0])
# Discriminator loss for fake data
D_loss_fake = tf.reduce_mean( \
tf.nn.sigmoid_cross_entropy_with_logits( \
logits=D_logit_fake, \
labels=tf.zeros_like(D_logit_fake)), [0])
D_loss_real_vec = tf.nn.sigmoid_cross_entropy_with_logits( \
logits=D_logit_real, \
labels=tf.ones_like(D_logit_real))
# Discriminator loss for fake data
D_loss_fake_vec = tf.nn.sigmoid_cross_entropy_with_logits( \
logits=D_logit_fake, \
labels=tf.zeros_like(D_logit_fake))
# Generator loss
G_loss = tf.reduce_mean( \
tf.nn.sigmoid_cross_entropy_with_logits( \
logits=D_logit_fake, labels=tf.ones_like(D_logit_fake)) \
, [0])
# Generator optimizer
G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G)
# Discriminator Optimizer
# ------------------------------------------------------------------------------
"""
minimize_ours :
Our method (Clipping the gradients of loss on real data and making
them noisy + Clipping the gradients of loss on fake data) is
implemented in this function .
It can be found in the following directory:
/content/gdrive/Team Drives/PrivacyGenomics/our_dp_gan/
differential_privacy/dp_sgd/dp_optimizer/dp_optimizer.py'
"""
#lr = tf.placeholder(tf.float32)
sigma = FLAGS.sigma
global_step = tf.train.get_global_step()
D_solver = base_dp_optimizer.DPGradientDescentGaussianOptimizer( \
priv_accountant,l2_norm_clip= FLAGS.default_gradient_l2norm_bound, noise_multiplier = sigma,num_microbatches=FLAGS.batch_size, learning_rate=FLAGS.lr). \
minimize(d_loss_real=D_loss_real_vec,d_loss_fake=D_loss_fake_vec, global_step=global_step, var_list=theta_D)
# ------------------------------------------------------------------------------
# Set output directory
resultDir = baseDir + "conditional-gan-dp-base-mnist/results"
if not os.path.exists(resultDir):
os.makedirs(resultDir)
resultPath = resultDir + "/bs_{}_s_{}_c_{}_d_{}_e_{}".format( \
batch_size, \
sigma, \
clipping_value, \
FLAGS.target_delta, FLAGS.target_eps)
if not os.path.exists(resultPath):
os.makedirs(resultPath)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
max_target_eps = max(target_eps)
# Main Session
with tf.Session() as sess:
init = tf.initialize_all_variables()
sess.run(init)
lot_size = FLAGS.batches_per_lot * batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
step = 0
# Is true when the spent privacy budget exceeds the target budget
should_terminate = False
# Main loop
while (step < FLAGS.num_training_steps and should_terminate == False):
epoch = step / lots_per_epoch
for _ in range(FLAGS.batches_per_lot):
# Save the generated images every 100 steps
if step % 100 == 0:
n_sample = 10
Z_sample = sample_Z(n_sample, Z_dim)
y_sample = np.zeros(shape=[n_sample, y_dim])
y_sample[0, 0] = 1
y_sample[1, 1] = 1
y_sample[2, 2] = 1
y_sample[3, 3] = 1
y_sample[4, 4] = 1
y_sample[5, 5] = 1
y_sample[6, 6] = 1
y_sample[7, 7] = 1
y_sample[8, 8] = 1
y_sample[9, 9] = 1
samples = sess.run(G_sample,
feed_dict={
Z: Z_sample,
y: y_sample
})
fig = plot(samples)
plt.savefig((resultPath + "/step_{}.png").format(
str(step).zfill(3)),
bbox_inches='tight')
plt.close(fig)
X_mb, y_mb = mnist.train.next_batch(batch_size, shuffle=True)
Z_sample = sample_Z(batch_size, Z_dim)
# Update the discriminator network
_, d_loss, _ = sess.run(
[D_solver, D_loss_real_vec, D_loss_fake_vec],
feed_dict={
X: X_mb,
Z: Z_sample,
y: y_mb
})
# _, D_loss_real_curr, D_loss_fake_curr = sess.run([D_solver, D_loss_real, D_loss_fake], \
# feed_dict={X: X_mb, \
# Z: Z_sample, \
# y: y_mb, \
# lr: curr_lr})
# Update the generator network
_, G_loss_curr = sess.run([G_solver, G_loss],
feed_dict={
Z: Z_sample,
y: y_mb
})
# Flag to terminate based on target privacy budget
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(sess, \
target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have \
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta or \
terminate_spent_eps_delta.spent_eps > max_target_eps or\
step == FLAGS.num_training_steps-1):
spent_eps_deltas = priv_accountant.get_privacy_spent( \
sess, target_eps=target_eps)
print("TERMINATE!!!!")
print("Termination Step : " + str(step))
print(spent_eps_deltas)
should_terminate = True
n_sample = 10
Z_sample = sample_Z(n_sample, Z_dim)
y_sample = np.zeros(shape=[n_sample, y_dim])
y_sample[0, 0] = 1
y_sample[1, 1] = 1
y_sample[2, 2] = 1
y_sample[3, 3] = 1
y_sample[4, 4] = 1
y_sample[5, 5] = 1
y_sample[6, 6] = 1
y_sample[7, 7] = 1
y_sample[8, 8] = 1
y_sample[9, 9] = 1
samples = sess.run(G_sample,
feed_dict={
Z: Z_sample,
y: y_sample
})
fig = plot(samples)
plt.savefig(
(resultPath + "/step_{}.png").format(str(step).zfill(3)),
bbox_inches='tight')
plt.close(fig)
n_class = np.zeros(10)
n_class[0] = 5923
n_class[1] = 6742
n_class[2] = 5958
n_class[3] = 6131
n_class[4] = 5842
n_class[5] = 5421
n_class[6] = 5918
n_class[7] = 6265
n_class[8] = 5851
n_class[9] = 5949
n_image = int(sum(n_class))
image_lables = np.zeros(shape=[n_image, len(n_class)])
image_cntr = 0
for class_cntr in np.arange(len(n_class)):
for cntr in np.arange(n_class[class_cntr]):
image_lables[image_cntr, class_cntr] = 1
image_cntr += 1
Z_sample = sample_Z(n_image, Z_dim)
images = sess.run(G_sample,
feed_dict={
Z: Z_sample,
y: image_lables
})
X_test, Y_test = loadlocal_mnist(
images_path='mnist/t10k-images.idx3-ubyte',
labels_path='mnist/t10k-labels.idx1-ubyte')
Y_test = [int(y) for y in Y_test]
classes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Y_test = label_binarize(Y_test, classes=classes)
print(" Classifying - Logistic Regression ...")
Y_score = classify(images,
image_lables,
X_test,
"lr",
random_state_value=30)
false_positive_rate, true_positive_rate, roc_auc = compute_fpr_tpr_roc(
Y_test, Y_score)
print(" AUROC: " + str(roc_auc["micro"]))
# print("\n################# Random Forest #######################")
# print(" Classifying ...")
# Y_score = classify(images, image_lables, X_test, "rf", random_state_value=30)
# print(" Computing ROC ...")
# false_positive_rate, true_positive_rate, roc_auc = compute_fpr_tpr_roc(Y_test, Y_score)
# print(" AUROC: " + str(roc_auc["micro"]))
# print("\n################# Gaussian Naive Bayes #######################")
# print(" Classifying ...")
# Y_score = classify(images, image_lables, X_test, "gnb", random_state_value=30)
# print(" Computing ROC ...")
# false_positive_rate, true_positive_rate, roc_auc = compute_fpr_tpr_roc(Y_test, Y_score)
# print(" AUROC: " + str(roc_auc["micro"]))
# print("\n################# Decision Tree #######################")
# print(" Classifying ...")
# Y_score = classify(images, image_lables, X_test, "dt", random_state_value=30)
# print(" Computing ROC ...")
# false_positive_rate, true_positive_rate, roc_auc = compute_fpr_tpr_roc(Y_test, Y_score)
# print(" AUROC: " + str(roc_auc["micro"]))
# print("\n################# Multi-layer Perceptron #######################")
# print(" Classifying ...")
# Y_score = classify(images, image_lables, X_test, "mlp", random_state_value=30)
# print(" Computing ROC ...")
# false_positive_rate, true_positive_rate, roc_auc = compute_fpr_tpr_roc(Y_test, Y_score)
# print(" AUROC: " + str(roc_auc["micro"]))
step = step + 1
def Train(train_file,
test_file,
network_parameters,
num_steps,
save_path,
total_rho,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = NUM_TRAINING_IMAGES
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
})
with tf.device('/gpu:0'), tf.Graph().as_default(), tf.Session() as sess:
#print_csv_tfrecords.print_tfrecords(train_file)
features, labels = DataInput(train_file, batch_size, False)
print("network_parameters.input_size", network_parameters.input_size)
logits, projection, training_params = utils.BuildNetwork(
features, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, LABEL_SIZE))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(
NUM_TRAINING_IMAGES)
sigma = None
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
elif FLAGS.accountant_type == "ZDCP":
priv_accountant = accountant.DumpzCDPAccountant()
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" %
FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
varsigma = tf.placeholder(tf.float32, shape=[])
init_ops = []
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
with_privacy = True
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
varsigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
print("No privacy")
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
curr_sigma = 0
previous_epoch = -1
rho_tracking = [0]
for step in range(num_steps):
epoch = step // lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
if with_privacy:
old_sigma = curr_sigma
#total budget
rhototal = total_rho
curr_sigma = get_current_sigma(epoch)
if step % 100 == 0:
print(curr_sigma)
print(rho_tracking[-1])
if epoch - previous_epoch == 1:
rho_tracking.append(rho_tracking[-1] + 1.0 /
(2.0 * curr_sigma**2))
previous_epoch = epoch
if with_privacy == True and rho_tracking[-1] > rhototal:
print("stop at epoch%d" % epoch)
print(rho_tracking[:-1])
break
for _ in range(FLAGS.batches_per_lot):
_ = sess.run(
[gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta,
varsigma: curr_sigma
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(
train_file,
network_parameters,
num_testing_images=NUM_TRAINING_IMAGES,
randomize=False,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
print(rho_tracking[:-1])
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(train_file,
network_parameters,
num_testing_images=NUM_TRAINING_IMAGES,
randomize=False,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
