def GAN_solvers(D_loss, G_loss, learning_rate, batch_size, total_examples,
l2norm_bound, batches_per_lot, sigma, dp=False):
"""
Optimizers
"""
discriminator_vars = [v for v in tf.trainable_variables() if v.name.startswith('discriminator')]
generator_vars = [v for v in tf.trainable_variables() if v.name.startswith('generator')]
if dp:
print('Using differentially private SGD to train discriminator!')
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
priv_accountant = accountant.GaussianMomentsAccountant(total_examples)
clip = True
l2norm_bound = l2norm_bound/batch_size
batches_per_lot = 1
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant,
[l2norm_bound, clip])
# the trick is that we need to calculate the gradient with respect to
# each example in the batch, during the DP SGD step
D_solver = dp_optimizer.DPGradientDescentOptimizer(learning_rate,
[eps, delta],
sanitizer=gaussian_sanitizer,
sigma=sigma,
batches_per_lot=batches_per_lot).minimize(D_loss, var_list=discriminator_vars)
else:
D_loss_mean_over_batch = tf.reduce_mean(D_loss)
D_solver = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(D_loss_mean_over_batch, var_list=discriminator_vars)
priv_accountant = None
G_loss_mean_over_batch = tf.reduce_mean(G_loss)
G_solver = tf.train.AdamOptimizer().minimize(G_loss_mean_over_batch, var_list=generator_vars)
return D_solver, G_solver, priv_accountant
def BoundGradients(training_params, priv_accountant, network_parameters, batch_size):
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant,
[network_parameters.default_gradient_l2norm_bound / batch_size, True]
)
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var]["gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(var, sanitizer.ClipOption(l2bound, True))
return gaussian_sanitizer
def DPSGD(sigma, l2norm_bound, learning_rate, total_examples):
import tensorflow as tf
from differential_privacy.dp_sgd.dp_optimizer import dp_optimizer
from differential_privacy.dp_sgd.dp_optimizer import sanitizer
from differential_privacy.privacy_accountant.tf import accountant
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
priv_accountant = accountant.GaussianMomentsAccountant(total_examples)
clip = True
batches_per_lot = 1
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [l2norm_bound, clip])
return dp_optimizer.DPGradientDescentOptimizer(
learning_rate, [eps, delta],
sanitizer=gaussian_sanitizer,
sigma=sigma,
batches_per_lot=batches_per_lot)
def runTensorFlow(sigma, clippingValue, batchSize, epsilon, delta, iteration):
h_dim = 128
Z_dim = 100
# Initializations for a two-layer discriminator network
mnist = input_data.read_data_sets(
baseDir + "our_dp_conditional_gan_mnist/mnist_dataset", one_hot=True)
X_dim = mnist.train.images.shape[1]
y_dim = mnist.train.labels.shape[1]
X = tf.placeholder(tf.float32, shape=[None, X_dim])
y = tf.placeholder(tf.float32, shape=[None, y_dim])
D_W1 = tf.Variable(xavier_init([X_dim + y_dim, h_dim]))
D_b1 = tf.Variable(tf.zeros(shape=[h_dim]))
D_W2 = tf.Variable(xavier_init([h_dim, 1]))
D_b2 = tf.Variable(tf.zeros(shape=[1]))
theta_D = [D_W1, D_W2, D_b1, D_b2]
# Initializations for a two-layer genrator network
Z = tf.placeholder(tf.float32, shape=[None, Z_dim])
G_W1 = tf.Variable(xavier_init([Z_dim + y_dim, h_dim]))
G_b1 = tf.Variable(tf.zeros(shape=[h_dim]))
G_W2 = tf.Variable(xavier_init([h_dim, X_dim]))
G_b2 = tf.Variable(tf.zeros(shape=[X_dim]))
theta_G = [G_W1, G_W2, G_b1, G_b2]
# Delete all Flags
del_all_flags(tf.flags.FLAGS)
# Set training parameters
tf.flags.DEFINE_string('f', '', 'kernel')
tf.flags.DEFINE_float("lr", 0.1, "start learning rate")
tf.flags.DEFINE_float("end_lr", 0.052, "end learning rate")
tf.flags.DEFINE_float(
"lr_saturate_epochs", 10000,
"learning rate saturate epochs; set to 0 for a constant"
"learning rate of --lr.")
tf.flags.DEFINE_integer("batch_size", batchSize,
"The training batch size.")
tf.flags.DEFINE_integer("batches_per_lot", 1, "Number of batches per lot.")
tf.flags.DEFINE_integer(
"num_training_steps", 100000, "The number of training"
"steps. This counts number of lots.")
# Flags that control privacy spending during training
tf.flags.DEFINE_float("target_delta", delta, "Maximum delta for"
"--terminate_based_on_privacy.")
tf.flags.DEFINE_float(
"sigma", sigma, "Noise sigma, used only if accountant_type"
"is Moments")
tf.flags.DEFINE_string(
"target_eps", str(epsilon),
"Log the privacy loss for the target epsilon's. Only"
"used when accountant_type is Moments.")
tf.flags.DEFINE_float("default_gradient_l2norm_bound", clippingValue,
"norm clipping")
FLAGS = tf.flags.FLAGS
# Set accountant type to GaussianMomentsAccountant
NUM_TRAINING_IMAGES = 60000
priv_accountant = accountant.GaussianMomentsAccountant(NUM_TRAINING_IMAGES)
# Sanitizer
batch_size = FLAGS.batch_size
clipping_value = FLAGS.default_gradient_l2norm_bound
# clipping_value = tf.placeholder(tf.float32)
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [clipping_value / batch_size, True])
# Instantiate the Generator Network
G_sample = generator(Z, y, theta_G)
# Instantiate the Discriminator Network
D_real, D_logit_real = discriminator(X, y, theta_D)
D_fake, D_logit_fake = discriminator(G_sample, y, theta_D)
# Discriminator loss for real data
D_loss_real = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits( \
logits=D_logit_real, \
labels=tf.ones_like(D_logit_real)), \
[0])
# Discriminator loss for fake data
D_loss_fake = tf.reduce_mean( \
tf.nn.sigmoid_cross_entropy_with_logits( \
logits=D_logit_fake, \
labels=tf.zeros_like(D_logit_fake)), [0])
# Generator loss
G_loss = tf.reduce_mean( \
tf.nn.sigmoid_cross_entropy_with_logits( \
logits=D_logit_fake, labels=tf.ones_like(D_logit_fake)) \
, [0])
# ------------------------------------------------------------------------------
"""
minimize_ours :
Our method (Clipping the gradients of loss on real data and making
them noisy + Clipping the gradients of loss on fake data) is
implemented in this function .
It can be found in the following directory:
differential_privacy/dp_sgd/dp_optimizer/dp_optimizer.py'
"""
lr = tf.placeholder(tf.float32)
sigma = FLAGS.sigma
# Generator optimizer
G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G)
# Discriminator Optimizer
D_solver = dp_optimizer.DPGradientDescentOptimizer( \
lr, [None, None], \
gaussian_sanitizer, \
sigma=sigma, \
batches_per_lot= \
FLAGS.batches_per_lot). \
minimize_ours( \
D_loss_real, \
D_loss_fake, \
var_list=theta_D)
# ------------------------------------------------------------------------------
# Set output directory
resultDir = baseDir + "out/"
if not os.path.exists(resultDir):
os.makedirs(resultDir)
resultPath = resultDir + "/run_{}_bs_{}_s_{}_c_{}_d_{}_e_{}".format( \
iteration, \
batch_size, \
sigma, \
clipping_value, \
FLAGS.target_delta, FLAGS.target_eps)
if not os.path.exists(resultPath):
os.makedirs(resultPath)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
max_target_eps = max(target_eps)
gpu_options = tf.GPUOptions(visible_device_list="0, 1")
# Main Session
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)) as sess:
init = tf.initialize_all_variables()
sess.run(init)
step = 0
# Is true when the spent privacy budget exceeds the target budget
should_terminate = False
# Main loop
while (step < FLAGS.num_training_steps and should_terminate == False):
epoch = step
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr, \
FLAGS.lr_saturate_epochs, epoch)
eps = compute_epsilon(FLAGS, (step + 1), sigma * clipping_value)
# Save the generated images every 50 steps
if step % 50 == 0:
print("step : " + str(step) + " eps : " + str(eps))
n_sample = 10
Z_sample = sample_Z(n_sample, Z_dim)
y_sample = np.zeros(shape=[n_sample, 10])
y_sample[0, 0] = 1
y_sample[1, 1] = 1
y_sample[2, 2] = 1
y_sample[3, 3] = 1
y_sample[4, 4] = 1
y_sample[5, 5] = 1
y_sample[6, 6] = 1
y_sample[7, 7] = 1
y_sample[8, 8] = 1
y_sample[9, 9] = 1
samples = sess.run(G_sample,
feed_dict={
Z: Z_sample,
y: y_sample
})
fig = plot(samples)
plt.savefig(
(resultPath + "/step_{}.png").format(str(step).zfill(3)),
bbox_inches='tight')
plt.close(fig)
X_mb, y_mb = mnist.train.next_batch(batch_size, shuffle=True)
Z_sample = sample_Z(batch_size, Z_dim)
# Update the discriminator network
_, D_loss_real_curr, D_loss_fake_curr = sess.run([D_solver, D_loss_real, D_loss_fake], \
feed_dict={X: X_mb, \
Z: Z_sample, \
y: y_mb, \
lr: curr_lr})
# Update the generator network
_, G_loss_curr = sess.run([G_solver, G_loss],
feed_dict={
Z: Z_sample,
y: y_mb,
lr: curr_lr
})
if (eps > max_target_eps):
print("TERMINATE!!!!")
print("Termination Step : " + str(step))
should_terminate = True
for i in range(0, 10):
n_sample = 10
Z_sample = sample_Z(n_sample, Z_dim)
y_sample = np.zeros(shape=[n_sample, y_dim])
y_sample[0, 0] = 1
y_sample[1, 1] = 1
y_sample[2, 2] = 1
y_sample[3, 3] = 1
y_sample[4, 4] = 1
y_sample[5, 5] = 1
y_sample[6, 6] = 1
y_sample[7, 7] = 1
y_sample[8, 8] = 1
y_sample[9, 9] = 1
samples = sess.run(G_sample,
feed_dict={
Z: Z_sample,
y: y_sample
})
fig = plot(samples)
plt.savefig((resultPath + "/Final_step_{}.png").format(
str(i).zfill(3)),
bbox_inches='tight')
plt.close(fig)
n_class = np.zeros(10)
n_class[0] = 5923
n_class[1] = 6742
n_class[2] = 5958
n_class[3] = 6131
n_class[4] = 5842
n_class[5] = 5421
n_class[6] = 5918
n_class[7] = 6265
n_class[8] = 5851
n_class[9] = 5949
n_image = int(sum(n_class))
image_lables = np.zeros(shape=[n_image, len(n_class)])
image_cntr = 0
for class_cntr in np.arange(len(n_class)):
for cntr in np.arange(n_class[class_cntr]):
image_lables[image_cntr, class_cntr] = 1
image_cntr += 1
Z_sample = sample_Z(n_image, Z_dim)
images = sess.run(G_sample,
feed_dict={
Z: Z_sample,
y: image_lables
})
X_test, Y_test = loadlocal_mnist(
images_path=baseDir + "our_dp_conditional_gan_mnist/" +
'mnist_dataset/t10k-images.idx3-ubyte',
labels_path=baseDir + "our_dp_conditional_gan_mnist/" +
'mnist_dataset/t10k-labels.idx1-ubyte')
Y_test = [int(y) for y in Y_test]
resultFile = open(resultPath + "/" + "results.txt", "w")
print("Binarizing the labels ...")
classes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Y_test = label_binarize(Y_test, classes=classes)
print(
"\n################# Logistic Regression #######################"
)
print(" Classifying ...")
Y_score = classify(images,
image_lables,
X_test,
"lr",
random_state_value=30)
print(" Computing ROC ...")
false_positive_rate, true_positive_rate, roc_auc = compute_fpr_tpr_roc(
Y_test, Y_score)
print(" AUROC: " + str(roc_auc["micro"]))
resultFile.write("LR AUROC: " + str(roc_auc["micro"]) + "\n")
print(
"\n################# Multi-layer Perceptron #######################"
)
print(" Classifying ...")
Y_score = classify(images,
image_lables,
X_test,
"mlp",
random_state_value=30)
print(" Computing ROC ...")
false_positive_rate, true_positive_rate, roc_auc = compute_fpr_tpr_roc(
Y_test, Y_score)
print(" AUROC: " + str(roc_auc["micro"]))
resultFile.write("MLP AUROC: " + str(roc_auc["micro"]) + "\n")
step = FLAGS.num_training_steps
break
step = step + 1
def Train(sess,
train_images,
train_labels,
mnist_test_file,
network_parameters,
num_steps,
save_path,
training_params,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"projection_dimensions": FLAGS.projection_dimensions,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
"flag_pca_eps": FLAGS.pca_eps,
"flag_pca_delta": FLAGS.pca_delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
"pca_sigma": FLAGS.pca_sigma,
})
# Create the basic Mnist model.
images = tf.get_default_graph().get_tensor_by_name("images:0")
labels = tf.get_default_graph().get_tensor_by_name("labels:0")
logits = tf.get_default_graph().get_tensor_by_name("logits:0")
projection = tf.get_default_graph().get_tensor_by_name("projection:0")
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 10))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(NUM_TRAINING_IMAGES)
sigma = None
pca_sigma = None
with_privacy = FLAGS.eps > 0
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
pca_sigma = FLAGS.pca_sigma
with_privacy = FLAGS.sigma > 0
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" % FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant,
[network_parameters.default_gradient_l2norm_bound / batch_size, True])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var]["gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(var,
sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
init_ops = []
if network_parameters.projection_type == "PCA":
with tf.variable_scope("pca"):
# Compute differentially private PCA.
all_data = tf.constant(train_images, dtype=tf.float32)
pca_projection = dp_pca.ComputeDPPrincipalProjection(
all_data, network_parameters.projection_dimensions,
gaussian_sanitizer, [FLAGS.pca_eps, FLAGS.pca_delta],
pca_sigma)
assign_pca_proj = tf.assign(projection, pca_projection)
init_ops.append(assign_pca_proj)
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
sigma=sigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = np.array([float(s) for s in FLAGS.target_eps.split(",")])
target_eps = -np.sort(-target_eps)
index_eps_to_save = 0
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
for step in xrange(num_steps):
epoch = step / lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
lot = np.random.choice(np.arange(NUM_TRAINING_IMAGES),
lot_size,
replace=False)
#print lot.shape
images_lot = train_images[lot, :]
labels_lot = train_labels[lot]
for i in xrange(FLAGS.batches_per_lot):
_ = sess.run(
[gd_op],
feed_dict={
lr:
curr_lr,
eps:
curr_eps,
delta:
FLAGS.delta,
images:
images_lot[i * FLAGS.batch_size:(i + 1) *
FLAGS.batch_size, :],
labels:
labels_lot[i * FLAGS.batch_size:(i + 1) * FLAGS.batch_size]
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
while index_eps_to_save < len(
spent_eps_deltas
) and spent_eps_deltas[index_eps_to_save][1] < FLAGS.delta:
saver.save(
sess,
save_path=save_path + "/eps" +
str(spent_eps_deltas[index_eps_to_save][0]) +
"_delta" +
'%.2g' % spent_eps_deltas[index_eps_to_save][1] +
"_pcasigma" + str(FLAGS.pca_sigma) + "_sigma" +
str(FLAGS.sigma) + "/ckpt")
index_eps_to_save += 1
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
pred_train = np.argmax(predict(sess, train_images), axis=1)
train_accuracy = np.mean(pred_train == train_labels)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
# test_accuracy, mistakes = Eval(mnist_test_file, network_parameters,
# num_testing_images=NUM_TESTING_IMAGES,
# randomize=False, load_path=save_path,
# save_mistakes=FLAGS.save_mistakes)
# sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
# "test_accuracy": test_accuracy,
# "mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
# "test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
# "mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
def train():
"""
"""
import time
input_sigma = FLAGS.INPUT_SIGMA
total_dp_sigma = FLAGS.TOTAL_DP_SIGMA
total_dp_delta = FLAGS.TOTAL_DP_DELTA
total_dp_epsilon = FLAGS.TOTAL_DP_EPSILON
batch_size = FLAGS.BATCH_SIZE
tf.reset_default_graph()
g = tf.get_default_graph()
# attack_target = 8
with g.as_default():
# Placeholder nodes.
data_holder = tf.placeholder(tf.float32, [
batch_size, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS
])
noised_pretrain_holder = tf.placeholder(tf.float32, [batch_size, 100])
noise_holder = tf.placeholder(tf.float32, [batch_size, 100])
label_holder = tf.placeholder(tf.float32,
[batch_size, FLAGS.NUM_CLASSES])
sgd_sigma_holder = tf.placeholder(tf.float32, ())
trans_sigma_holder = tf.placeholder(tf.float32, ())
is_training = tf.placeholder(tf.bool, ())
# model
model = model_cifar100.RDPCNN(data=data_holder,
label=label_holder,
input_sigma=input_sigma,
is_training=is_training,
noised_pretrain=noised_pretrain_holder,
noise=noise_holder)
priv_accountant = accountant.GaussianMomentsAccountant(data.train_size)
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [FLAGS.DP_GRAD_CLIPPING_L2NORM, True])
# model training
model_loss = model.loss()
model_loss_clean = model.loss_clean()
model_loss_reg = model.loss_reg()
# training
#model_op, _, _, model_lr = model.optimization(model_loss)
model_op, model_lr = model.dp_optimization(
[model_loss, model_loss_reg],
gaussian_sanitizer,
sgd_sigma_holder,
trans_sigma_holder,
FLAGS.BATCHES_PER_LOT,
is_layerwised=FLAGS.IS_MGM_LAYERWISED)
# analysis
model_M, model_sens = model.compute_M_from_input_perturbation(
[model_loss_clean, model_loss_reg],
FLAGS.DP_GRAD_CLIPPING_L2NORM,
is_layerwised=FLAGS.IS_MGM_LAYERWISED)
model_acc = model.cnn_accuracy
graph_dict = {}
graph_dict["data_holder"] = data_holder
graph_dict["noised_pretrain_holder"] = noised_pretrain_holder
graph_dict["noise_holder"] = noise_holder
graph_dict["label_holder"] = label_holder
graph_dict["sgd_sigma_holder"] = sgd_sigma_holder
graph_dict["trans_sigma_holder"] = trans_sigma_holder
graph_dict["is_training"] = is_training
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config, graph=g) as sess:
sess.run(tf.global_variables_initializer())
if FLAGS.load_pretrained:
model.tf_load_pretrained(
sess, name=FLAGS.PRETRAINED_CNN_CKPT_RESTORE_NAME)
if FLAGS.load_model:
model.tf_load(sess, name=FLAGS.CNN_CKPT_RESTORE_NAME)
if FLAGS.local:
total_train_lot = 2
total_valid_lot = 2
else:
total_train_lot = int(data.train_size / batch_size /
FLAGS.BATCHES_PER_LOT)
total_valid_lot = None
print("Training...")
itr_count = 0
itr_start_time = time.time()
for epoch in range(FLAGS.NUM_EPOCHS):
ep_start_time = time.time()
# Compute A norm
min_S_min = float("inf")
# shuffle
data.shuffle_train()
b_idx = 0
for train_idx in range(total_train_lot):
#for train_idx in range(1):
terminate = False
lot_feeds = []
lot_M = []
for _ in range(FLAGS.BATCHES_PER_LOT):
#batch_xs = keras_resnet_preprocess(data.x_train[b_idx*batch_size:(b_idx+1)*batch_size])
batch_xs = data.x_train[b_idx * batch_size:(b_idx + 1) *
batch_size]
batch_ys = data.y_train[b_idx * batch_size:(b_idx + 1) *
batch_size]
feed_dict = {data_holder: batch_xs, is_training: True}
batch_pretrain = sess.run(fetches=model.pre_trained_cnn,
feed_dict=feed_dict)
#batch_xs = np.tile(batch_xs, [1,1,1,3])
noise = np.random.normal(loc=0.0,
scale=input_sigma,
size=batch_pretrain.shape)
feed_dict = {
noise_holder: noise,
noised_pretrain_holder: batch_pretrain + noise,
label_holder: batch_ys,
is_training: True
}
#import pdb; pdb.set_trace()
#batch_S_min = sess.run(fetches=model_S_min[0], feed_dict=feed_dict)
batch_M = sess.run(fetches=model_M, feed_dict=feed_dict)
#batch_S_min = compute_S_min_from_M(batch_M, FLAGS.IS_MGM_LAYERWISED)/FLAGS.DP_GRAD_CLIPPING_L2NORM
lot_feeds.append(feed_dict)
lot_M.append(batch_M)
b_idx += 1
lot_M = sum(lot_M) / (FLAGS.BATCHES_PER_LOT**2)
lot_S_min = compute_S_min_from_M(
lot_M,
FLAGS.IS_MGM_LAYERWISED) / FLAGS.DP_GRAD_CLIPPING_L2NORM
#import pdb; pdb.set_trace()
min_S_min = lot_S_min
sigma_trans = input_sigma * min_S_min
if sigma_trans >= FLAGS.TOTAL_DP_SIGMA:
sgd_sigma = 0.0
else:
sgd_sigma = FLAGS.TOTAL_DP_SIGMA - sigma_trans
sigma_trans = FLAGS.TOTAL_DP_SIGMA
for feed_dict in lot_feeds:
# DP-SGD
feed_dict[sgd_sigma_holder] = sgd_sigma
feed_dict[trans_sigma_holder] = sigma_trans
sess.run(fetches=[model_op], feed_dict=feed_dict)
itr_count += 1
if itr_count > FLAGS.MAX_ITERATIONS:
terminate = True
# for input transofrmation
if train_idx % 1 == 0:
print("min S_min: ", min_S_min)
print("Sigma trans: ", sigma_trans)
print("Sigma grads: ", sgd_sigma)
# optimization
fetches = [model_loss, model_loss_reg, model_acc, model_lr]
loss, reg, acc, lr = sess.run(fetches=fetches,
feed_dict=feed_dict)
#import pdb; pdb.set_trace()
spent_eps_delta, selected_moment_orders = priv_accountant.get_privacy_spent(
sess, target_eps=[total_dp_epsilon])
spent_eps_delta = spent_eps_delta[0]
selected_moment_orders = selected_moment_orders[0]
if spent_eps_delta.spent_delta > total_dp_delta or spent_eps_delta.spent_eps > total_dp_epsilon:
terminate = True
# Print info
if train_idx % FLAGS.EVAL_TRAIN_FREQUENCY == (
FLAGS.EVAL_TRAIN_FREQUENCY - 1):
print("Epoch: {}".format(epoch))
print("Iteration: {}".format(itr_count))
print("Sigma used:{}".format(sigma_trans))
print("SGD Sigma: {}".format(sgd_sigma))
print("Learning rate: {}".format(lr))
print("Loss: {:.4f}, Reg loss: {:.4f}, Accuracy: {:.4f}".
format(loss, reg, acc))
print(
"Total dp eps: {:.4f}, total dp delta: {:.8f}, total dp sigma: {:.4f}, input sigma: {:.4f}"
.format(spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta, total_dp_sigma,
input_sigma))
print()
#model.tf_save(sess) # save checkpoint
with open(FLAGS.TRAIN_LOG_FILENAME, "a+") as file:
file.write("Epoch: {}\n".format(epoch))
file.write("Iteration: {}\n".format(itr_count))
file.write("Sigma used: {}\n".format(sigma_trans))
file.write("SGD Sigma: {}\n".format(sgd_sigma))
file.write("Learning rate: {}\n".format(lr))
file.write(
"Loss: {:.4f}, Reg loss: {:.4f}, Accuracy: {:.4f}\n"
.format(loss, reg, acc))
file.write(
"Total dp eps: {:.4f}, total dp delta: {:.8f}, total dp sigma: {:.4f}, input sigma: {:.4f}\n"
.format(spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta,
total_dp_sigma, input_sigma))
file.write("\n")
if itr_count % FLAGS.EVAL_VALID_FREQUENCY == 0:
#if train_idx >= 0:
end_time = time.time()
print('{} iterations completed with time {:.2f} s'.format(
itr_count, end_time - itr_start_time))
# validation
print(
"\n******************************************************************"
)
print("Epoch {} Validation".format(epoch))
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_dp_sigma,
"input_sigma": input_sigma
}
valid_dict = test_info(sess,
model,
True,
graph_dict,
dp_info,
FLAGS.VALID_LOG_FILENAME,
total_batch=100)
#np.save(FLAGS.DP_INFO_NPY, dp_info, allow_pickle=True)
'''
ckpt_name='robust_dp_cnn.epoch{}.vloss{:.6f}.vacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch,
valid_dict["loss"],
valid_dict["acc"],
input_sigma, total_dp_sigma,
spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta
)
'''
#model.tf_save(sess, name=ckpt_name) # extra store
if terminate:
break
end_time = time.time()
print('Eopch {} completed with time {:.2f} s'.format(
epoch + 1, end_time - ep_start_time))
# validation
print(
"\n******************************************************************"
)
print("Epoch {} Validation".format(epoch))
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_dp_sigma,
"input_sigma": input_sigma
}
valid_dict = test_info(sess,
model,
True,
graph_dict,
dp_info,
FLAGS.VALID_LOG_FILENAME,
total_batch=None)
np.save(FLAGS.DP_INFO_NPY, dp_info, allow_pickle=True)
ckpt_name = 'robust_dp_cnn.epoch{}.vloss{:.6f}.vacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch, valid_dict["loss"], valid_dict["acc"], input_sigma,
total_dp_sigma, spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta)
model.tf_save(sess, name=ckpt_name) # extra store
if terminate:
break
print(
"******************************************************************"
)
print()
print()
print("Optimization Finished!")
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_dp_sigma,
"input_sigma": input_sigma
}
valid_dict = test_info(sess,
model,
False,
graph_dict,
dp_info,
FLAGS.TEST_LOG_FILENAME,
total_batch=None)
np.save(FLAGS.DP_INFO_NPY, dp_info, allow_pickle=True)
ckpt_name = 'robust_dp_cnn.epoch{}.vloss{:.6f}.vacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch, valid_dict["loss"], valid_dict["acc"], input_sigma,
total_dp_sigma, spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta)
model.tf_save(sess, name=ckpt_name) # extra store
def train():
"""
"""
import time
input_sigma = FLAGS.INPUT_SIGMA
total_dp_sigma = FLAGS.TOTAL_DP_SIGMA
total_dp_delta = FLAGS.TOTAL_DP_DELTA
total_dp_epsilon = FLAGS.TOTAL_DP_EPSILON
tf.reset_default_graph()
g = tf.get_default_graph()
# attack_target = 8
with g.as_default():
# Placeholder nodes.
px_holder = [tf.placeholder(tf.float32, [1, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS]) for _ in range(FLAGS.BATCH_SIZE)]
data_holder = tf.placeholder(tf.float32, [FLAGS.BATCH_SIZE, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS])
noised_data_holder = tf.placeholder(tf.float32, [FLAGS.BATCH_SIZE, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS])
noise_holder = tf.placeholder(tf.float32, [FLAGS.BATCH_SIZE, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS])
label_holder = tf.placeholder(tf.float32, [FLAGS.BATCH_SIZE, FLAGS.NUM_CLASSES])
if FLAGS.IS_MGM_LAYERWISED:
sgd_sigma_holder = [tf.placeholder(tf.float32, ()) for _ in range(FLAGS.MAX_PARAM_SIZE)]
trans_sigma_holder = [tf.placeholder(tf.float32, ()) for _ in range(FLAGS.MAX_PARAM_SIZE)]
else:
sgd_sigma_holder = tf.placeholder(tf.float32, ())
trans_sigma_holder = tf.placeholder(tf.float32, ())
is_training = tf.placeholder(tf.bool, ())
# model
model = model_mnist.RDPCNN(noised_data=noised_data_holder, noise=noise_holder, label=label_holder, input_sigma=input_sigma, is_training=is_training)
priv_accountant = accountant.GaussianMomentsAccountant(data.train_size)
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(priv_accountant,
[FLAGS.DP_GRAD_CLIPPING_L2NORM, True])
# model training
model_loss = model.loss()
model_loss_clean = model.loss_clean()
# training
#model_op, _, _, model_lr = model.optimization(model_loss)
model_op, model_lr = model.dp_optimization(model_loss, gaussian_sanitizer, sgd_sigma_holder, trans_sigma_holder, FLAGS.BATCHES_PER_LOT, is_layerwised=FLAGS.IS_MGM_LAYERWISED)
# analysis
model_Jac, model_sens = model.compute_Jac_from_input_perturbation(model_loss_clean, FLAGS.DP_GRAD_CLIPPING_L2NORM, is_layerwised=FLAGS.IS_MGM_LAYERWISED)
model_S_min, model_res = model.compute_S_min_from_input_perturbation(model_loss_clean, is_layerwised=FLAGS.IS_MGM_LAYERWISED)
model_acc = model.cnn_accuracy
graph_dict = {}
graph_dict["px_holder"] = px_holder
graph_dict["data_holder"] = data_holder
graph_dict["noised_data_holder"] = noised_data_holder
graph_dict["noise_holder"] = noise_holder
graph_dict["label_holder"] = label_holder
graph_dict["sgd_sigma_holder"] = sgd_sigma_holder
graph_dict["trans_sigma_holder"] = trans_sigma_holder
graph_dict["is_training"] = is_training
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config, graph=g) as sess:
sess.run(tf.global_variables_initializer())
if FLAGS.load_model:
print("CNN loaded.")
model.tf_load(sess, name=FLAGS.CNN_CKPT_RESTORE_NAME)
if FLAGS.local:
total_train_lot = 2
total_valid_lot = 2
else:
total_train_lot = int(data.train_size/FLAGS.BATCH_SIZE/FLAGS.BATCHES_PER_LOT)
total_valid_lot = None
print("Training...")
itr_count = 0
for epoch in range(FLAGS.NUM_EPOCHS):
start_time = time.time()
# Compute A norm
min_S_min = float("inf")
if FLAGS.IS_MGM_LAYERWISED:
min_S_min_layerwised = [float("inf") for _ in range(FLAGS.MAX_PARAM_SIZE)]
#'''
for train_idx in range(total_train_lot):
for batch_idx in range(FLAGS.BATCHES_PER_LOT):
batch_xs, batch_ys, _ = data.next_train_batch(FLAGS.BATCH_SIZE, True)
noise = np.random.normal(loc=0.0, scale=input_sigma, size=batch_xs.shape)
feed_dict = {
noise_holder: noise,
noised_data_holder: batch_xs+noise,
label_holder: batch_ys,
is_training: True
}
#batch_S_min = sess.run(fetches=model_S_min[0], feed_dict=feed_dict)
batch_Jac, batch_sens = sess.run(fetches=[model_Jac, model_sens], feed_dict=feed_dict)
batch_S_min = compute_S_min_from_Jac(batch_Jac, batch_sens, FLAGS.IS_MGM_LAYERWISED)
#import pdb; pdb.set_trace()
if FLAGS.IS_MGM_LAYERWISED: # batch_K_norm is [b, #layer]
#import pdb; pdb.set_trace()
num_layer = batch_S_min.shape[1]
batch_S_min_layerwised = np.amin(batch_S_min, axis=0)
min_S_min_layerwised = min_S_min_layerwised[:len(batch_S_min_layerwised)]
min_S_min_layerwised = np.minimum(min_S_min_layerwised, batch_S_min_layerwised)
else: # scalalr
min_S_min = min(min_S_min, min(batch_S_min))
if train_idx % 100 == 9:
if FLAGS.IS_MGM_LAYERWISED:
print("min S_min layerwised: ", min_S_min_layerwised)
else: print("min S_min: ", min_S_min)
if FLAGS.IS_MGM_LAYERWISED:
sigma_trans = input_sigma * min_S_min_layerwised
print("Sigma trans: ", sigma_trans)
sgd_sigma = np.zeros([FLAGS.MAX_PARAM_SIZE])
for idx in range(len(sigma_trans)):
if sigma_trans[idx] < FLAGS.TOTAL_DP_SIGMA:
if FLAGS.TOTAL_DP_SIGMA - sigma_trans[idx] <= FLAGS.INPUT_DP_SIGMA_THRESHOLD:
sgd_sigma[idx] = FLAGS.INPUT_DP_SIGMA_THRESHOLD
else: sgd_sigma[idx] = FLAGS.TOTAL_DP_SIGMA - sigma_trans[idx]
print("Sigma grads: ", sgd_sigma)
#
#hetero_sgd_sigma = np.sqrt(len(min_S_min_layerwised)/np.sum(np.square(1.0/sgd_sigma[:len(min_S_min_layerwised)])))
#print("Sigma grads in Heterogeneous form: ", hetero_sgd_sigma)
else:
sigma_trans = input_sigma * min_S_min
print("Sigma trans: ", sigma_trans)
if sigma_trans >= FLAGS.TOTAL_DP_SIGMA:
sgd_sigma = 0.0
elif FLAGS.TOTAL_DP_SIGMA - sigma_trans <= FLAGS.INPUT_DP_SIGMA_THRESHOLD:
sgd_sigma = FLAGS.INPUT_DP_SIGMA_THRESHOLD
else: sgd_sigma = FLAGS.TOTAL_DP_SIGMA - sigma_trans
print("Sigma grads: ", sgd_sigma)
#'''
#sigma_trans = [34.59252105,0.71371817,16.14990762,0.59402054,0.,0.50355514,30.09081199,0.40404256,21.18426806,0.35788509,0.,0.30048024,0.,0.30312875]
#sgd_sigma = [0.,0.8,0.,0.8,1.,0.8,0.,0.8,0.,0.8,1.,0.8,1.,0.8,0.,0.,0.,0.]
#sgd_sigma = [34.59252105,1.0,16.14990762,1.0,1.,1.0,30.09081199,1.0,21.18426806,1.0,1.,1.0,1.,1.0,0.,0.,0.,0.]
for train_idx in range(total_train_lot):
terminate = False
for batch_idx in range(FLAGS.BATCHES_PER_LOT):
itr_count += 1
batch_xs, batch_ys, _ = data.next_train_batch(FLAGS.BATCH_SIZE, True)
noise = np.random.normal(loc=0.0, scale=input_sigma, size=batch_xs.shape)
feed_dict = {
noise_holder: noise,
noised_data_holder: batch_xs+noise,
label_holder: batch_ys,
is_training: True
}
if FLAGS.IS_MGM_LAYERWISED:
for idx in range(len(sigma_trans)):
feed_dict[sgd_sigma_holder[idx]] = sgd_sigma[idx]
feed_dict[trans_sigma_holder[idx]] = sigma_trans[idx]
else:
feed_dict[sgd_sigma_holder] = sgd_sigma
feed_dict[trans_sigma_holder] = sigma_trans
sess.run(fetches=[model_op], feed_dict=feed_dict)
if itr_count > FLAGS.MAX_ITERATIONS:
terminate = True
# optimization
fetches = [model_loss, model_acc, model_lr]
loss, acc, lr = sess.run(fetches=fetches, feed_dict=feed_dict)
#import pdb; pdb.set_trace()
spent_eps_delta, selected_moment_orders = priv_accountant.get_privacy_spent(sess, target_eps=[total_dp_epsilon])
spent_eps_delta = spent_eps_delta[0]
selected_moment_orders = selected_moment_orders[0]
if spent_eps_delta.spent_delta > total_dp_delta or spent_eps_delta.spent_eps > total_dp_epsilon:
terminate = True
# Print info
if train_idx % FLAGS.EVAL_TRAIN_FREQUENCY == (FLAGS.EVAL_TRAIN_FREQUENCY - 1):
print("Epoch: {}".format(epoch))
print("Iteration: {}".format(itr_count))
print("Sigma used:{}".format(sigma_trans))
print("SGD Sigma: {}".format(sgd_sigma))
print("Learning rate: {}".format(lr))
print("Loss: {:.4f}, Accuracy: {:.4f}".format(loss, acc))
print("Total dp eps: {:.4f}, total dp delta: {:.8f}, total dp sigma: {:.4f}, input sigma: {:.4f}".format(
spent_eps_delta.spent_eps, spent_eps_delta.spent_delta, total_dp_sigma, input_sigma))
print()
#model.tf_save(sess) # save checkpoint
with open(FLAGS.TRAIN_LOG_FILENAME, "a+") as file:
file.write("Epoch: {}\n".format(epoch))
file.write("Iteration: {}\n".format(itr_count))
file.write("Sigma used: {}\n".format(sigma_trans))
file.write("SGD Sigma: {}\n".format(sgd_sigma))
file.write("Learning rate: {}\n".format(lr))
file.write("Loss: {:.4f}, Accuracy: {:.4f}\n".format(loss, acc))
file.write("Total dp eps: {:.4f}, total dp delta: {:.8f}, total dp sigma: {:.4f}, input sigma: {:.4f}\n".format(
spent_eps_delta.spent_eps, spent_eps_delta.spent_delta, total_dp_sigma, input_sigma))
file.write("\n")
if terminate:
break
end_time = time.time()
print('Eopch {} completed with time {:.2f} s'.format(epoch+1, end_time-start_time))
if epoch % FLAGS.EVAL_VALID_FREQUENCY == (FLAGS.EVAL_VALID_FREQUENCY - 1):
#if epoch >= 0:
# validation
print("\n******************************************************************")
print("Validation")
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_dp_sigma,
"input_sigma": input_sigma
}
valid_dict = test_info(sess, model, None, graph_dict, dp_info, FLAGS.VALID_LOG_FILENAME, total_batch=None, valid=True)
np.save(FLAGS.DP_INFO_NPY, dp_info, allow_pickle=True)
ckpt_name='robust_dp_cnn.epoch{}.vloss{:.6f}.vacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch,
valid_dict["loss"],
valid_dict["acc"],
input_sigma, total_dp_sigma,
spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta
)
model.tf_save(sess, name=ckpt_name) # extra store
if terminate:
break
print("******************************************************************")
print()
print()
print("Optimization Finished!")
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_dp_sigma,
"input_sigma": input_sigma
}
valid_dict = test_info(sess, model, None, graph_dict, dp_info, None, total_batch=None, valid=True)
np.save(FLAGS.DP_INFO_NPY, dp_info, allow_pickle=True)
ckpt_name='robust_dp_cnn.epoch{}.vloss{:.6f}.vacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch,
valid_dict["loss"],
valid_dict["acc"],
input_sigma, total_dp_sigma,
spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta
)
model.tf_save(sess, name=ckpt_name) # extra store
"is Moments")
tf.flags.DEFINE_string(
"target_eps", "9.6", "Log the privacy loss for the target epsilon's. Only"
"used when accountant_type is Moments.")
tf.flags.DEFINE_float("default_gradient_l2norm_bound", 4, "norm clipping")
FLAGS = tf.flags.FLAGS
# Set accountant type to GaussianMomentsAccountant
NUM_TRAINING_IMAGES = 60000
priv_accountant = accountant.GaussianMomentsAccountant(NUM_TRAINING_IMAGES)
#Sanitizer
batch_size = FLAGS.batch_size
clipping_value = FLAGS.default_gradient_l2norm_bound
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [clipping_value / batch_size, True])
#Instantiate the Generator Network
G_sample = generator(Z)
#Instantiate the Discriminator Network
D_real, D_logit_real = discriminator(X)
D_fake, D_logit_fake = discriminator(G_sample)
# Discriminator loss for real data
D_loss_real = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(\
logits=D_logit_real,\
labels=tf.ones_like(D_logit_real)),\
[0])
# Discriminator loss for fake data
def Train(cifar_train_file,
mnist_test_file,
network_parameters,
num_steps,
save_path,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
cifar_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
params.update({"sigma": FLAGS.sigma})
with tf.Graph().as_default(), tf.Session() as sess, tf.device('/cpu:0'):
# Create the basic Cifar model.
images, labels = CifarInput(cifar_train_file, batch_size,
FLAGS.randomize)
logits, projection, training_params = utils.BuildNetwork(
images, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 100))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
with_privacy = FLAGS.sigma > 0
with_privacy = False
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
init_ops = []
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
sigma=sigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
for step in range(num_steps):
epoch = step / lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
for _ in range(FLAGS.batches_per_lot):
_ = sess.run([gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or
terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(cifar_train_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=True,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
mnist_test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
print("\nTERMINATING.\n")
break
# It uses privacy amplication via sampling to compute the privacyspending for each
# batch and strong composition (specialized for Gaussian noise) for
# accumulate the privacy spending (http://arxiv.org/pdf/1405.7085v2.pdf)
# we use the implementation of
# https://github.com/tensorflow/models/blob/master/research/differential_privacy/privacy_accountant/tf/accountant.py
priv_accountant = accountant.AmortizedAccountant(
num_examples)
# per-example Gradient l_2 norm bound.
example_gradient_l2norm_bound = FLAGS.gradient_l2norm_bound / FLAGS.batch_size
# Gaussian sanitizer, will enforce differential privacy by clipping the gradient-per-example.
# Add gaussian noise, and sum the noisy gradients at each weight update step.
# It will also notify the privacy accountant to update the privacy spending.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant,
[example_gradient_l2norm_bound, True])
critic_step = dp_optimizer.DPGradientDescentOptimizer(
FLAGS.lr,
# (eps, delta) unused parameters for the moments accountant which we are using
[eps_holder, delta_holder],
gaussian_sanitizer,
sigma=sigma,
batches_per_lot=1,
var_list=critic_vars).minimize((loss_critic_real, loss_critic_fake),
global_step=global_step, var_list=critic_vars)
else:
# This is used when we train without privacy.
critic_step = tf.train.RMSPropOptimizer(FLAGS.lr).minimize(
def train():
"""
"""
import time
input_sigma = FLAGS.INPUT_SIGMA
tf.reset_default_graph()
g = tf.get_default_graph()
# attack_target = 8
with g.as_default():
batch_size = FLAGS.BATCH_SIZE
# Placeholder nodes.
data_holder = tf.placeholder(tf.float32, [
batch_size, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS
])
noised_pre_holder = tf.placeholder(tf.float32, [batch_size, 256])
noise_holder = tf.placeholder(tf.float32, [batch_size, 256])
label_holder = tf.placeholder(tf.float32,
[batch_size, FLAGS.NUM_CLASSES])
sgd_sigma_holder = tf.placeholder(tf.float32, ())
trans_sigma_holder = tf.placeholder(tf.float32, ())
loss_coef_holder = tf.placeholder(tf.float32, ())
is_training = tf.placeholder(tf.bool, ())
# model
model = model_mnist.RDPCNN(data=data_holder,
label=label_holder,
input_sigma=input_sigma,
is_training=is_training,
noised_pre=noised_pre_holder,
noise=noise_holder)
priv_accountant = accountant.GaussianMomentsAccountant(data.train_size)
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [FLAGS.DP_GRAD_CLIPPING_L2NORM, True])
finetune_gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [FLAGS.FINETUNE_DP_GRAD_CLIPPING_L2NORM, True])
# model training
model_clean_loss = model.loss(loss_coef_holder, model.clean_logits)
#
model_finetune_loss = model.loss(loss_coef_holder,
model.finetune_logits)
model_finetune_clean_loss = model.loss(loss_coef_holder,
model.finetune_clean_logits)
# training
#model_op, _, _, model_lr = model.optimization(model_loss)
model_op, model_lr = model.dp_optimization(
model_clean_loss,
gaussian_sanitizer,
sgd_sigma_holder,
None,
batched_per_lot=FLAGS.BATCHES_PER_LOT,
is_layerwised=FLAGS.IS_MGM_LAYERWISED)
# finetune
model_finetune_op, model_finetune_lr = model.dp_optimization(
model_finetune_loss,
finetune_gaussian_sanitizer,
sgd_sigma_holder,
trans_sigma_holder,
is_finetune=True,
batched_per_lot=FLAGS.FINETUNE_BATCHES_PER_LOT,
is_layerwised=FLAGS.IS_MGM_LAYERWISED,
scope="FINETUNE_DP_OPT")
# analysis
model_M, model_sens = model.compute_M_from_input_perturbation(
model_finetune_clean_loss,
FLAGS.FINETUNE_DP_GRAD_CLIPPING_L2NORM,
is_layerwised=FLAGS.IS_MGM_LAYERWISED)
# acc
model_finetune_acc = model.finetune_accuracy
# acc
model_clean_acc = model.clean_accuracy
graph_dict = {}
graph_dict["data_holder"] = data_holder
graph_dict["noised_pre_holder"] = noised_pre_holder
graph_dict["noise_holder"] = noise_holder
graph_dict["label_holder"] = label_holder
graph_dict["loss_coef_holder"] = loss_coef_holder
graph_dict["sgd_sigma_holder"] = sgd_sigma_holder
graph_dict["trans_sigma_holder"] = trans_sigma_holder
graph_dict["is_training"] = is_training
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config, graph=g) as sess:
sess.run(tf.global_variables_initializer())
if FLAGS.load_model:
print("model loaded.")
model.tf_load(sess, scope=None, name=FLAGS.CNN_CKPT_RESTORE_NAME)
if FLAGS.load_pretrained:
print("model loaded.")
model.tf_load_pretrained(
sess, scope="CNN", name=FLAGS.PRETRAINED_CNN_CKPT_RESTORE_NAME)
if FLAGS.TRAIN_BEFORE_FINETUNE:
# training
if FLAGS.local:
total_train_lot = 2
total_valid_lot = 2
else:
total_train_lot = int(data.train_size / batch_size /
FLAGS.BATCHES_PER_LOT)
total_valid_lot = None
total_dp_sigma = FLAGS.TOTAL_DP_SIGMA
total_dp_delta = FLAGS.TOTAL_DP_DELTA
total_dp_epsilon = FLAGS.TOTAL_DP_EPSILON
print("Training...")
itr_count = 0
for epoch in range(FLAGS.NUM_EPOCHS):
start_time = time.time()
sgd_sigma = total_dp_sigma
sigma_trans = 0.0
for train_idx in range(total_train_lot):
#for train_idx in range(1):
terminate = False
for batch_idx in range(FLAGS.BATCHES_PER_LOT):
batch_xs, batch_ys, _ = data.next_train_batch(
batch_size, True)
feed_dict = {
data_holder: batch_xs,
label_holder: batch_ys,
loss_coef_holder: FLAGS.BETA,
sgd_sigma_holder: sgd_sigma,
trans_sigma_holder: sigma_trans,
is_training: True
}
sess.run(fetches=[model_op], feed_dict=feed_dict)
# optimization
fetches = [model_clean_loss, model_clean_acc, model_lr]
loss, acc, lr = sess.run(fetches=fetches,
feed_dict=feed_dict)
#import pdb; pdb.set_trace()
spent_eps_delta, selected_moment_orders = priv_accountant.get_privacy_spent(
sess, target_eps=[total_dp_epsilon])
spent_eps_delta = spent_eps_delta[0]
selected_moment_orders = selected_moment_orders[0]
if spent_eps_delta.spent_delta > total_dp_delta or spent_eps_delta.spent_eps > total_dp_epsilon:
terminate = True
# Print info
if train_idx % FLAGS.EVAL_TRAIN_FREQUENCY == (
FLAGS.EVAL_TRAIN_FREQUENCY - 1):
print("Epoch: {}".format(epoch))
print("Iteration: {}".format(itr_count))
print("Sigma used:{}".format(sigma_trans))
print("SGD Sigma: {}".format(sgd_sigma))
print("Learning rate: {}".format(lr))
print("Loss: {:.4f}, Accuracy: {:.4f}".format(
loss, acc))
print(
"Total dp eps: {:.4f}, total dp delta: {:.8f}, total dp sigma: {:.4f}, input sigma: {:.4f}"
.format(spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta,
total_dp_sigma, input_sigma))
print()
#model.tf_save(sess) # save checkpoint
with open(FLAGS.TRAIN_LOG_FILENAME, "a+") as file:
file.write("Epoch: {}\n".format(epoch))
file.write("Iteration: {}\n".format(itr_count))
file.write("Sigma used: {}\n".format(sigma_trans))
file.write("SGD Sigma: {}\n".format(sgd_sigma))
file.write("Learning rate: {}\n".format(lr))
file.write(
"Loss: {:.4f}, Accuracy: {:.4f}\n".format(
loss, acc))
file.write(
"Total dp eps: {:.4f}, total dp delta: {:.8f}, total dp sigma: {:.4f}, input sigma: {:.4f}\n"
.format(spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta,
total_dp_sigma, input_sigma))
file.write("\n")
if terminate:
break
end_time = time.time()
print('Eopch {} completed with time {:.2f} s'.format(
epoch + 1, end_time - start_time))
if epoch % FLAGS.EVAL_VALID_FREQUENCY == (
FLAGS.EVAL_VALID_FREQUENCY - 1):
#if epoch >= 0:
# validation
print(
"\n******************************************************************"
)
print("Validation")
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_dp_sigma,
"input_sigma": input_sigma
}
valid_dict = test_info(sess,
model,
None,
graph_dict,
dp_info,
FLAGS.VALID_LOG_FILENAME,
total_batch=None,
is_finetune=False,
valid=True)
np.save(FLAGS.DP_INFO_NPY, dp_info, allow_pickle=True)
ckpt_name = 'robust_dp_cnn.epoch{}.vloss{:.6f}.vacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch, valid_dict["loss"], valid_dict["acc"],
input_sigma, total_dp_sigma, spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta)
model.tf_save(sess, name=ckpt_name) # extra store
if epoch % FLAGS.TOTAL_DP_SIGMA_DECAY_EPOCH == FLAGS.TOTAL_DP_SIGMA_DECAY_EPOCH - 1:
total_dp_sigma = max(
model_utils.change_coef(
total_dp_sigma, FLAGS.TOTAL_DP_SIGMA_DECAY_RATE),
FLAGS.MIN_TOTAL_DP_SIGMA)
if terminate:
break
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_dp_sigma,
"input_sigma": input_sigma
}
test_dict = test_info(sess,
model,
None,
graph_dict,
dp_info,
FLAGS.TEST_LOG_FILENAME,
total_batch=None,
is_finetune=False,
valid=False)
np.save(FLAGS.DP_INFO_NPY, dp_info, allow_pickle=True)
ckpt_name = 'robust_dp_cnn.epoch{}.tloss{:.6f}.tacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch, test_dict["loss"], test_dict["acc"], input_sigma,
total_dp_sigma, spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta)
model.tf_save(sess, name=ckpt_name) # extra store
else:
print("Load model from ckpt file")
model.tf_load(sess, name=FLAGS.CNN_CKPT_RESTORE_NAME)
#finetune
if FLAGS.local:
total_train_lot = 2
total_valid_lot = 2
else:
total_train_lot = int(data.train_size / batch_size /
FLAGS.FINETUNE_BATCHES_PER_LOT)
total_valid_lot = None
total_finetune_dp_sigma = FLAGS.TOTAL_FINETUNE_DP_SIGMA
total_finetune_dp_delta = FLAGS.TOTAL_FINETUNE_DP_DELTA
total_finetune_dp_epsilon = FLAGS.TOTAL_FINETUNE_DP_EPSILON
print("Finetuning...")
itr_count = 0
for epoch in range(FLAGS.NUM_FINETUNE_EPOCHS):
start_time = time.time()
# Compute A norm
min_S_min = float("inf")
for train_idx in range(total_train_lot):
#for train_idx in range(1):
terminate = False
lot_feeds = []
lot_M = []
for batch_idx in range(FLAGS.FINETUNE_BATCHES_PER_LOT):
batch_xs, batch_ys, _ = data.next_train_batch(
batch_size, True)
feed_dict = {data_holder: batch_xs, is_training: True}
batch_pre = sess.run(fetches=model.pre_conv,
feed_dict=feed_dict)
#batch_xs = np.tile(batch_xs, [1,1,1,3])
noise = np.random.normal(loc=0.0,
scale=input_sigma,
size=batch_pre.shape)
feed_dict = {
noise_holder: noise,
noised_pre_holder: batch_pre + noise,
loss_coef_holder: FLAGS.FINETUNE_BETA,
label_holder: batch_ys,
is_training: True
}
#import pdb; pdb.set_trace()
#batch_S_min = sess.run(fetches=model_S_min[0], feed_dict=feed_dict)
batch_M = sess.run(fetches=model_M, feed_dict=feed_dict)
#batch_S_min = compute_S_min_from_M(batch_M, FLAGS.IS_MGM_LAYERWISED)/FLAGS.FINETUNE_DP_GRAD_CLIPPING_L2NORM
lot_feeds.append(feed_dict)
lot_M.append(batch_M)
lot_M = sum(lot_M) / (FLAGS.FINETUNE_BATCHES_PER_LOT**2)
lot_S_min = compute_S_min_from_M(
lot_M, FLAGS.IS_MGM_LAYERWISED
) / FLAGS.FINETUNE_DP_GRAD_CLIPPING_L2NORM
#import pdb; pdb.set_trace()
min_S_min = lot_S_min
sigma_trans = input_sigma * min_S_min
if sigma_trans >= total_finetune_dp_sigma:
sgd_sigma = 0.0
else:
sgd_sigma = total_finetune_dp_sigma - sigma_trans
sigma_trans = total_finetune_dp_sigma
for feed_dict in lot_feeds:
# DP-SGD
feed_dict[sgd_sigma_holder] = sgd_sigma
feed_dict[trans_sigma_holder] = sigma_trans
sess.run(fetches=[model_finetune_op], feed_dict=feed_dict)
itr_count += 1
if itr_count > FLAGS.MAX_FINETUNE_ITERATIONS:
terminate = True
# for input transofrmation
if train_idx % 1 == 0:
print("min S_min: ", min_S_min)
print("Sigma trans: ", sigma_trans)
print("Sigma grads: ", sgd_sigma)
# optimization
fetches = [
model_finetune_loss, model_finetune_acc, model_finetune_lr
]
loss, acc, lr = sess.run(fetches=fetches, feed_dict=feed_dict)
#import pdb; pdb.set_trace()
spent_eps_delta, selected_moment_orders = priv_accountant.get_privacy_spent(
sess, target_eps=[total_finetune_dp_epsilon])
spent_eps_delta = spent_eps_delta[0]
selected_moment_orders = selected_moment_orders[0]
if spent_eps_delta.spent_delta > total_finetune_dp_delta or spent_eps_delta.spent_eps > total_finetune_dp_epsilon:
terminate = True
# Print info
if train_idx % FLAGS.EVAL_FINETUNE_TRAIN_FREQUENCY == (
FLAGS.EVAL_FINETUNE_TRAIN_FREQUENCY - 1):
print("Finetune Epoch: {}".format(epoch))
print("Iteration: {}".format(itr_count))
print("Sigma used:{}".format(sigma_trans))
print("SGD Sigma: {}".format(sgd_sigma))
print("Learning rate: {}".format(lr))
print("Loss: {:.4f}, Accuracy: {:.4f}".format(loss, acc))
print(
"Total dp eps: {:.4f}, total dp delta: {:.8f}, total dp sigma: {:.4f}, input sigma: {:.4f}"
.format(spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta,
total_finetune_dp_sigma, input_sigma))
print()
#model.tf_save(sess) # save checkpoint
with open(FLAGS.FINETUNE_TRAIN_LOG_FILENAME, "a+") as file:
file.write("Finetune Epoch: {}\n".format(epoch))
file.write("Iteration: {}\n".format(itr_count))
file.write("Sigma used: {}\n".format(sigma_trans))
file.write("SGD Sigma: {}\n".format(sgd_sigma))
file.write("Learning rate: {}\n".format(lr))
file.write("Loss: {:.4f}, Accuracy: {:.4f}\n".format(
loss, acc))
file.write(
"Total dp eps: {:.4f}, total dp delta: {:.8f}, total dp sigma: {:.4f}, input sigma: {:.4f}\n"
.format(spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta,
total_finetune_dp_sigma, input_sigma))
file.write("\n")
if terminate:
break
end_time = time.time()
print('Eopch {} completed with time {:.2f} s'.format(
epoch + 1, end_time - start_time))
if epoch % FLAGS.EVAL_FINETUNE_VALID_FREQUENCY == (
FLAGS.EVAL_FINETUNE_VALID_FREQUENCY - 1):
#if epoch >= 0:
# validation
print(
"\n******************************************************************"
)
print("Validation")
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_finetune_dp_sigma,
"input_sigma": input_sigma
}
valid_dict = test_info(sess,
model,
None,
graph_dict,
dp_info,
FLAGS.FINETUNE_VALID_LOG_FILENAME,
total_batch=None,
is_finetune=True,
valid=True)
np.save(FLAGS.FINETUNE_DP_INFO_NPY, dp_info, allow_pickle=True)
ckpt_name = 'finetune.robust_dp_cnn.epoch{}.vloss{:.6f}.vacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch, valid_dict["loss"], valid_dict["acc"], input_sigma,
total_finetune_dp_sigma, spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta)
model.tf_save(sess, name=ckpt_name) # extra store
if epoch % FLAGS.TOTAL_FINETUNE_DP_SIGMA_DECAY_EPOCH == FLAGS.TOTAL_FINETUNE_DP_SIGMA_DECAY_EPOCH - 1:
total_finetune_dp_sigma = max(
model_utils.change_coef(
total_finetune_dp_sigma,
FLAGS.TOTAL_FINETUNE_DP_SIGMA_DECAY_RATE),
FLAGS.MIN_TOTAL_FINETUNE_DP_SIGMA)
if terminate:
break
print(
"******************************************************************"
)
print()
print()
print("Optimization Finished!")
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_finetune_dp_sigma,
"input_sigma": input_sigma
}
test_dict = test_info(sess,
model,
None,
graph_dict,
dp_info,
FLAGS.FINETUNE_TEST_LOG_FILENAME,
total_batch=None,
is_finetune=True,
valid=False)
np.save(FLAGS.FINETUNE_DP_INFO_NPY, dp_info, allow_pickle=True)
ckpt_name = 'finetune.robust_dp_cnn.epoch{}.tloss{:.6f}.tacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch, test_dict["loss"], test_dict["acc"], input_sigma,
total_finetune_dp_sigma, spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta)
model.tf_save(sess, name=ckpt_name) # extra store
return dp_info, ckpt_name
def Train(mnist_train_file,
mnist_test_file,
network_parameters,
num_steps,
save_path,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"projection_dimensions": FLAGS.projection_dimensions,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
"flag_pca_eps": FLAGS.pca_eps,
"flag_pca_delta": FLAGS.pca_delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
"pca_sigma": FLAGS.pca_sigma,
})
with tf.Graph().as_default(), tf.Session() as sess, tf.device('/cpu:0'):
# Create the basic Mnist model.
images, labels = MnistInput(mnist_train_file, batch_size,
FLAGS.randomize)
logits, projection, training_params = utils.BuildNetwork(
images, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 10))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(
NUM_TRAINING_IMAGES)
sigma = None
pca_sigma = None
with_privacy = FLAGS.eps > 0
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
pca_sigma = FLAGS.pca_sigma
with_privacy = FLAGS.sigma > 0
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" %
FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
init_ops = []
if network_parameters.projection_type == "PCA":
with tf.variable_scope("pca"):
# Compute differentially private PCA.
all_data, _ = MnistInput(mnist_train_file, NUM_TRAINING_IMAGES,
False)
pca_projection = dp_pca.ComputeDPPrincipalProjection(
all_data, network_parameters.projection_dimensions,
gaussian_sanitizer, [FLAGS.pca_eps, FLAGS.pca_delta],
pca_sigma)
assign_pca_proj = tf.assign(projection, pca_projection)
init_ops.append(assign_pca_proj)
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
sigma=sigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
for step in xrange(num_steps):
epoch = step / lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
for _ in xrange(FLAGS.batches_per_lot):
_ = sess.run([gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or
terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(mnist_train_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=True,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
mnist_test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
network_parameters = utils.NetworkParameters()
# If the ASCII proto isn't specified, then construct a config protobuf based
# on 3 flags.
network_parameters.input_size = IMAGE_SIZE**2
network_parameters.default_gradient_l2norm_bound = (
FLAGS.default_gradient_l2norm_bound)
if FLAGS.projection_dimensions > 0 and FLAGS.num_conv_layers > 0:
raise ValueError("Currently you can't do PCA and have convolutions"
"at the same time. Pick one")
# could add support for PCA after convolutions.
# Currently BuildNetwork can build the network with conv followed by
# projection, but the PCA training works on data, rather than data run
# through a few layers. Will need to init the convs before running the
# PCA, and need to change the PCA subroutine to take a network and perhaps
# allow for batched inputs, to handle larger datasets.
if FLAGS.num_conv_layers > 0:
conv = utils.ConvParameters()
conv.name = "conv1"
conv.in_channels = 1
conv.out_channels = 128
conv.num_outputs = 128 * 14 * 14
network_parameters.conv_parameters.append(conv)
# defaults for the rest: 5x5,stride 1, relu, maxpool 2x2,stride 2.
# insize 28x28, bias, stddev 0.1, non-trainable.
if FLAGS.num_conv_layers > 1:
conv = network_parameters.ConvParameters()
conv.name = "conv2"
conv.in_channels = 128
conv.out_channels = 128
conv.num_outputs = 128 * 7 * 7
conv.in_size = 14
# defaults for the rest: 5x5,stride 1, relu, maxpool 2x2,stride 2.
# bias, stddev 0.1, non-trainable.
network_parameters.conv_parameters.append(conv)
if FLAGS.num_conv_layers > 2:
raise ValueError(
"Currently --num_conv_layers must be 0,1 or 2."
"Manually create a network_parameters proto for more.")
if FLAGS.projection_dimensions > 0:
network_parameters.projection_type = "PCA"
network_parameters.projection_dimensions = FLAGS.projection_dimensions
for i in xrange(FLAGS.num_hidden_layers):
hidden = utils.LayerParameters()
hidden.name = "hidden%d" % i
hidden.num_units = FLAGS.hidden_layer_num_units
hidden.relu = True
hidden.with_bias = False
hidden.trainable = not FLAGS.freeze_bottom_layers
network_parameters.layer_parameters.append(hidden)
logits = utils.LayerParameters()
logits.name = "logits"
logits.num_units = 10
logits.relu = False
logits.with_bias = False
network_parameters.layer_parameters.append(logits)
inputs = tf.placeholder(tf.float32, [None, 784], name='inputs')
outputs, _, _ = utils.BuildNetwork(inputs, network_parameters)
def Train(mnist_train_file,
mnist_test_file,
mnist_validation_file,
network_parameters,
num_steps,
save_path,
total_rho,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = FLAGS.batch_size
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"projection_dimensions": FLAGS.projection_dimensions,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_hidden_layers": FLAGS.num_hidden_layers,
"hidden_layer_num_units": FLAGS.hidden_layer_num_units,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
"flag_pca_eps": FLAGS.pca_eps,
"flag_pca_delta": FLAGS.pca_delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
"pca_sigma": FLAGS.pca_sigma,
})
elif FLAGS.accountant_type == "zCDP":
params.update()
with tf.device('/gpu:0'), tf.Graph().as_default(), tf.Session() as sess:
# Create the basic Mnist model.
images, labels = MnistInput(mnist_train_file, batch_size,
FLAGS.randomize)
logits, projection, training_params = utils.BuildNetwork(
images, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, 10))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(
NUM_TRAINING_IMAGES)
sigma = None
pca_sigma = None
with_privacy = FLAGS.eps > 0
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
pca_sigma = FLAGS.pca_sigma
with_privacy = FLAGS.sigma > 0
elif FLAGS.accountant_type == "ZCDP":
priv_accountant = accountant.DumpzCDPAccountant()
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" %
FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
varsigma = tf.placeholder(tf.float32, shape=[])
init_ops = []
if network_parameters.projection_type == "PCA":
with tf.variable_scope("pca"):
# Compute differentially private PCA.
all_data, _ = MnistInput(mnist_train_file, NUM_TRAINING_IMAGES,
False)
pca_projection = dp_pca.ComputeDPPrincipalProjection(
all_data, network_parameters.projection_dimensions,
gaussian_sanitizer, [FLAGS.pca_eps, FLAGS.pca_delta],
pca_sigma)
assign_pca_proj = tf.assign(projection, pca_projection)
init_ops.append(assign_pca_proj)
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
with_privacy = True
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
varsigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
print("No privacy")
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results" + datetime.datetime.now().strftime(
'%Y-%m-%d-%H-%M-%S') + ".json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
#
previous_epoch = -1
rho_tracking = [0]
validation_accuracy_list = []
previous_validaccuracy = 0
tracking_sigma = []
curr_sigma = 10
# total budget
rhototal = total_rho
for step in range(num_steps):
epoch = step // np.ceil(lots_per_epoch)
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
old_sigma = curr_sigma
#validation based decay
#period=10, threshold=0.01, decay_factor=0.9
period = 10
decay_factor = 0.8
threshold = 0.01
m = 5
if epoch - previous_epoch == 1 and (
epoch + 1) % period == 0: #checking epoch
current_validaccuracy = sum(validation_accuracy_list[-m:]) / m
if current_validaccuracy - previous_validaccuracy < threshold:
curr_sigma = decay_factor * curr_sigma
previous_validaccuracy = current_validaccuracy
if old_sigma != curr_sigma:
print(curr_sigma)
#for tracking by epoch
if epoch - previous_epoch == 1:
tracking_sigma.append(curr_sigma)
rho_tracking.append(rho_tracking[-1] + 1 /
(2.0 * curr_sigma**2))
previous_epoch = epoch
if with_privacy == True and rho_tracking[-1] > rhototal:
print("stop at epoch%d" % epoch)
break
print(rho_tracking)
print(rho_tracking)
print(tracking_sigma)
for _ in range(FLAGS.batches_per_lot):
_ = sess.run(
[gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta,
varsigma: curr_sigma
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
terminate_spent_eps_delta = None
if with_privacy and FLAGS.terminate_based_on_privacy:
terminate_spent_eps_delta = priv_accountant.get_privacy_spent(
sess, target_eps=[max_target_eps])[0]
# For the Moments accountant, we should always have
# spent_eps == max_target_eps.
if (terminate_spent_eps_delta.spent_delta > FLAGS.target_delta
or
terminate_spent_eps_delta.spent_eps > max_target_eps):
should_terminate = True
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
if with_privacy:
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=target_eps)
print(spent_eps_deltas)
else:
spent_eps_deltas = [accountant.EpsDelta(0, 0)]
for spent_eps, spent_delta in spent_eps_deltas:
sys.stderr.write("spent privacy: eps %.4f delta %.5g\n" %
(spent_eps, spent_delta))
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(mnist_train_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=True,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
mnist_test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
validation_accuracy, mistakes = Eval(
mnist_validation_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("validation_accuracy: %.2f\n" %
validation_accuracy)
validation_accuracy_list.append(validation_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"spent_eps_deltas": spent_eps_deltas,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
def train():
"""
"""
import time
input_sigma = FLAGS.INPUT_SIGMA
total_dp_sigma = FLAGS.TOTAL_DP_SIGMA
total_dp_delta = FLAGS.TOTAL_DP_DELTA
total_dp_epsilon = FLAGS.TOTAL_DP_EPSILON
batch_size = FLAGS.BATCH_SIZE
tf.reset_default_graph()
g = tf.get_default_graph()
# attack_target = 8
with g.as_default():
# Placeholder nodes.
data_holder = tf.placeholder(tf.float32, [
batch_size, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS
])
noised_pretrain_holder = tf.placeholder(tf.float32,
[batch_size, 32, 32, 4])
noise_holder = tf.placeholder(tf.float32, [batch_size, 32, 32, 4])
label_holder = tf.placeholder(tf.float32,
[batch_size, FLAGS.NUM_CLASSES])
sgd_sigma_holder = tf.placeholder(tf.float32, ())
trans_sigma_holder = tf.placeholder(tf.float32, ())
is_training = tf.placeholder(tf.bool, ())
# model
model = model_cifar10.RDPCNN(data=data_holder,
label=label_holder,
input_sigma=input_sigma,
is_training=is_training,
noised_pretrain=noised_pretrain_holder,
noise=noise_holder)
priv_accountant = accountant.GaussianMomentsAccountant(data.train_size)
gaussian_sanitizer_bott = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [FLAGS.DP_GRAD_CLIPPING_L2NORM_BOTT, True])
gaussian_sanitizer_1 = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [FLAGS.DP_GRAD_CLIPPING_L2NORM_1, True])
gaussian_sanitizer_2 = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [FLAGS.DP_GRAD_CLIPPING_L2NORM_2, True])
# model training
total_opt_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
top_1_opt_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
model.opt_scope_1.name)
top_2_opt_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
model.opt_scope_2.name)
bott_opt_vars = []
for v_ in total_opt_vars:
if v_ not in top_1_opt_vars + top_2_opt_vars and "logits" not in v_.name:
bott_opt_vars.append(v_)
# loss
model_loss = model.loss(FLAGS.BETA)
model_loss_clean = model.loss_clean(FLAGS.BETA)
model_loss_bott = model.loss_bott(FLAGS.BETA_BOTT)
model_loss_reg_1 = model.loss_reg(FLAGS.REG_SCALE, top_1_opt_vars)
model_loss_reg_2 = model.loss_reg(FLAGS.REG_SCALE, top_2_opt_vars)
model_loss_reg_bott = model.loss_reg(FLAGS.REG_SCALE_BOTT,
bott_opt_vars)
# training
model_bott_op, _ = model.dp_optimization(
[model_loss_bott, model_loss_reg_bott],
gaussian_sanitizer_bott,
sgd_sigma_holder,
trans_sigma=None,
opt_vars=bott_opt_vars,
learning_rate=FLAGS.LEARNING_RATE_0,
lr_decay_steps=FLAGS.LEARNING_DECAY_STEPS_0,
lr_decay_rate=FLAGS.LEARNING_DECAY_RATE_0,
batched_per_lot=FLAGS.BATCHES_PER_LOT,
is_layerwised=FLAGS.IS_MGM_LAYERWISED)
model_op_1, _ = model.dp_optimization(
[model_loss, model_loss_reg_1],
gaussian_sanitizer_1,
sgd_sigma_holder,
trans_sigma=trans_sigma_holder,
opt_vars=top_1_opt_vars,
learning_rate=FLAGS.LEARNING_RATE_1,
lr_decay_steps=FLAGS.LEARNING_DECAY_STEPS_1,
lr_decay_rate=FLAGS.LEARNING_DECAY_RATE_1,
batched_per_lot=FLAGS.BATCHES_PER_LOT,
is_layerwised=FLAGS.IS_MGM_LAYERWISED)
model_op_2, model_lr = model.dp_optimization(
[model_loss, model_loss_reg_2],
gaussian_sanitizer_2,
sgd_sigma_holder,
trans_sigma=trans_sigma_holder,
opt_vars=top_2_opt_vars,
learning_rate=FLAGS.LEARNING_RATE_2,
lr_decay_steps=FLAGS.LEARNING_DECAY_STEPS_2,
lr_decay_rate=FLAGS.LEARNING_DECAY_RATE_2,
batched_per_lot=FLAGS.BATCHES_PER_LOT,
is_layerwised=FLAGS.IS_MGM_LAYERWISED)
# analysis
model_M_1, _ = model.compute_M_from_input_perturbation(
[model_loss_clean, model_loss_reg_1],
FLAGS.DP_GRAD_CLIPPING_L2NORM_1,
var_list=top_1_opt_vars,
is_layerwised=FLAGS.IS_MGM_LAYERWISED)
model_M_2, _ = model.compute_M_from_input_perturbation(
[model_loss_clean, model_loss_reg_2],
FLAGS.DP_GRAD_CLIPPING_L2NORM_2,
var_list=top_2_opt_vars,
is_layerwised=FLAGS.IS_MGM_LAYERWISED)
model_acc = model.cnn_accuracy
graph_dict = {}
graph_dict["data_holder"] = data_holder
graph_dict["noised_pretrain_holder"] = noised_pretrain_holder
graph_dict["noise_holder"] = noise_holder
graph_dict["label_holder"] = label_holder
graph_dict["sgd_sigma_holder"] = sgd_sigma_holder
graph_dict["trans_sigma_holder"] = trans_sigma_holder
graph_dict["is_training"] = is_training
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config, graph=g) as sess:
sess.run(tf.global_variables_initializer())
if FLAGS.load_pretrained:
model.tf_load_pretrained(sess)
if FLAGS.load_model:
model.tf_load(sess, name=FLAGS.CNN_CKPT_RESTORE_NAME)
if FLAGS.local:
total_train_lot = 2
total_valid_lot = 2
else:
total_train_lot = int(data.train_size / batch_size /
FLAGS.BATCHES_PER_LOT)
total_valid_lot = None
print("Training...")
itr_count = 0
itr_start_time = time.time()
for epoch in range(FLAGS.NUM_EPOCHS):
ep_start_time = time.time()
# Compute A norm
min_S_min = float("inf")
# shuffle
data.shuffle_train()
b_idx = 0
for train_idx in range(total_train_lot):
#for train_idx in range(1):
terminate = False
# top_2_layers
lot_feeds = []
lot_M = []
for _ in range(FLAGS.BATCHES_PER_LOT):
#batch_xs = keras_resnet_preprocess(data.x_train[b_idx*batch_size:(b_idx+1)*batch_size])
batch_xs = keras_resnet_preprocess(
data.x_train[b_idx * batch_size:(b_idx + 1) *
batch_size])
batch_ys = data.y_train[b_idx * batch_size:(b_idx + 1) *
batch_size]
feed_dict = {data_holder: batch_xs, is_training: True}
batch_pretrain = sess.run(fetches=model.pre_trained_cnn,
feed_dict=feed_dict)
#batch_xs = np.tile(batch_xs, [1,1,1,3])
noise = np.random.normal(loc=0.0,
scale=input_sigma,
size=batch_pretrain.shape)
feed_dict = {
data_holder: batch_xs,
noise_holder: noise,
noised_pretrain_holder: batch_pretrain + noise,
label_holder: batch_ys,
is_training: True
}
#if train_idx % 5 < 4:
if train_idx % FLAGS.BOTT_TRAIN_FREQ_TOTAL < FLAGS.BOTT_TRAIN_FREQ:
# run op for bott layers
feed_dict[sgd_sigma_holder] = FLAGS.TOTAL_DP_SIGMA
sess.run(fetches=model_bott_op, feed_dict=feed_dict)
#
batch_M_1 = sess.run(fetches=model_M_1,
feed_dict=feed_dict)
lot_M.append(batch_M_1)
lot_feeds.append(feed_dict)
b_idx += 1
min_S_min_1, sgd_sigma_1, sigma_trans_1 = cal_sigmas(
lot_M, input_sigma, FLAGS.DP_GRAD_CLIPPING_L2NORM_1)
# for input transofrmation
if train_idx % 1 == 0:
print("top_1_layers:")
print("min S_min: ", min_S_min_1)
print("Sigma trans: ", sigma_trans_1)
print("Sigma grads: ", sgd_sigma_1)
print()
# run op for top_1_layers
lot_M = []
for feed_dict in lot_feeds:
feed_dict[sgd_sigma_holder] = sgd_sigma_1
feed_dict[trans_sigma_holder] = sigma_trans_1
sess.run(fetches=model_op_1, feed_dict=feed_dict)
#
batch_M_2 = sess.run(fetches=model_M_2,
feed_dict=feed_dict)
lot_M.append(batch_M_2)
min_S_min_2, sgd_sigma_2, sigma_trans_2 = cal_sigmas(
lot_M, input_sigma, FLAGS.DP_GRAD_CLIPPING_L2NORM_2)
if train_idx % 1 == 0:
print("top_2_layers:")
print("min S_min: ", min_S_min_2)
print("Sigma trans: ", sigma_trans_2)
print("Sigma grads: ", sgd_sigma_2)
# run op for top_2_layers; cal M for top_1_layers
for feed_dict in lot_feeds:
feed_dict[sgd_sigma_holder] = sgd_sigma_2
feed_dict[trans_sigma_holder] = sigma_trans_2
sess.run(fetches=model_op_2, feed_dict=feed_dict)
itr_count += 1
if itr_count > FLAGS.MAX_ITERATIONS:
terminate = True
# optimization
fetches = [
model_loss, model_loss_bott, model_loss_reg_bott,
model_loss_reg_1, model_loss_reg_2, model_acc, model_lr
]
loss, loss_bott, reg0, reg1, reg2, acc, lr = sess.run(
fetches=fetches, feed_dict=feed_dict)
#import pdb; pdb.set_trace()
spent_eps_delta, selected_moment_orders = priv_accountant.get_privacy_spent(
sess, target_eps=[total_dp_epsilon])
spent_eps_delta = spent_eps_delta[0]
selected_moment_orders = selected_moment_orders[0]
if spent_eps_delta.spent_delta > total_dp_delta or spent_eps_delta.spent_eps > total_dp_epsilon:
terminate = True
# Print info
if train_idx % FLAGS.EVAL_TRAIN_FREQUENCY == (
FLAGS.EVAL_TRAIN_FREQUENCY - 1):
print("Epoch: {}".format(epoch))
print("Iteration: {}".format(itr_count))
print("Sigma used 1:{}".format(sigma_trans_1))
print("SGD Sigma 1: {}".format(sgd_sigma_1))
print("Sigma used 2:{}".format(sigma_trans_2))
print("SGD Sigma 2: {}".format(sgd_sigma_2))
print("Learning rate: {}".format(lr))
print(
"Loss: {:.4f}, Loss Bott: {:.4f}, Reg loss bott: {:.4f}, Reg loss 1: {:.4f}, Reg loss 2: {:.4f}, Accuracy: {:.4f}"
.format(loss, loss_bott, reg0, reg1, reg2, acc))
print(
"Total dp eps: {:.4f}, total dp delta: {:.8f}, total dp sigma: {:.4f}, input sigma: {:.4f}"
.format(spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta, total_dp_sigma,
input_sigma))
print()
#model.tf_save(sess) # save checkpoint
with open(FLAGS.TRAIN_LOG_FILENAME, "a+") as file:
file.write("Epoch: {}\n".format(epoch))
file.write("Iteration: {}\n".format(itr_count))
file.write("Sigma used 1: {}\n".format(sigma_trans_1))
file.write("SGD Sigma 1: {}\n".format(sgd_sigma_1))
file.write("Sigma used 2: {}\n".format(sigma_trans_2))
file.write("SGD Sigma 2: {}\n".format(sgd_sigma_2))
file.write("Learning rate: {}\n".format(lr))
file.write(
"Loss: {:.4f}, Loss Bott: {:.4f}, Reg loss bott: {:.4f}, Reg loss 1: {:.4f}, Reg loss 2: {:.4f}, Accuracy: {:.4f}\n"
.format(loss, loss_bott, reg0, reg1, reg2, acc))
file.write(
"Total dp eps: {:.4f}, total dp delta: {:.8f}, total dp sigma: {:.4f}, input sigma: {:.4f}\n"
.format(spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta,
total_dp_sigma, input_sigma))
file.write("\n")
if itr_count % FLAGS.EVAL_VALID_FREQUENCY == 0:
#if train_idx >= 0:
end_time = time.time()
print('{} iterations completed with time {:.2f} s'.format(
itr_count, end_time - itr_start_time))
# validation
print(
"\n******************************************************************"
)
print("Epoch {} Validation".format(epoch))
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_dp_sigma,
"input_sigma": input_sigma
}
valid_dict = test_info(sess,
model,
True,
graph_dict,
dp_info,
FLAGS.VALID_LOG_FILENAME,
total_batch=100)
#np.save(FLAGS.DP_INFO_NPY, dp_info, allow_pickle=True)
'''
ckpt_name='robust_dp_cnn.epoch{}.vloss{:.6f}.vacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch,
valid_dict["loss"],
valid_dict["acc"],
input_sigma, total_dp_sigma,
spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta
)
'''
#model.tf_save(sess, name=ckpt_name) # extra store
if terminate:
break
end_time = time.time()
print('Eopch {} completed with time {:.2f} s'.format(
epoch + 1, end_time - ep_start_time))
# validation
print(
"\n******************************************************************"
)
print("Epoch {} Validation".format(epoch))
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_dp_sigma,
"input_sigma": input_sigma
}
valid_dict = test_info(sess,
model,
True,
graph_dict,
dp_info,
FLAGS.VALID_LOG_FILENAME,
total_batch=None)
np.save(FLAGS.DP_INFO_NPY, dp_info, allow_pickle=True)
ckpt_name = 'robust_dp_cnn.epoch{}.vloss{:.6f}.vacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch, valid_dict["loss"], valid_dict["acc"], input_sigma,
total_dp_sigma, spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta)
model.tf_save(sess, name=ckpt_name) # extra store
if terminate:
break
print(
"******************************************************************"
)
print()
print()
print("Optimization Finished!")
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_dp_sigma,
"input_sigma": input_sigma
}
valid_dict = test_info(sess,
model,
False,
graph_dict,
dp_info,
FLAGS.TEST_LOG_FILENAME,
total_batch=None)
np.save(FLAGS.DP_INFO_NPY, dp_info, allow_pickle=True)
ckpt_name = 'robust_dp_cnn.epoch{}.vloss{:.6f}.vacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch, valid_dict["loss"], valid_dict["acc"], input_sigma,
total_dp_sigma, spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta)
model.tf_save(sess, name=ckpt_name) # extra store
def train():
"""
"""
import time
input_sigma = FLAGS.INPUT_SIGMA
total_dp_sigma = FLAGS.TOTAL_DP_SIGMA
total_dp_delta = FLAGS.TOTAL_DP_DELTA
total_dp_epsilon = FLAGS.TOTAL_DP_EPSILON
tf.reset_default_graph()
g = tf.get_default_graph()
# attack_target = 8
with g.as_default():
# Placeholder nodes.
px_holder = [
tf.placeholder(
tf.float32,
[1, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS])
for _ in range(FLAGS.BATCH_SIZE)
]
data_holder = tf.placeholder(tf.float32, [
FLAGS.BATCH_SIZE, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS,
FLAGS.NUM_CHANNELS
])
noised_data_holder = tf.placeholder(tf.float32, [
FLAGS.BATCH_SIZE, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS,
FLAGS.NUM_CHANNELS
])
noise_holder = tf.placeholder(tf.float32, [
FLAGS.BATCH_SIZE, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS,
FLAGS.NUM_CHANNELS
])
label_holder = tf.placeholder(tf.float32,
[FLAGS.BATCH_SIZE, FLAGS.NUM_CLASSES])
sgd_sigma_holder = [
tf.placeholder(tf.float32, [FLAGS.BATCH_SIZE])
for _ in range(FLAGS.MAX_PARAM_SIZE)
]
is_training = tf.placeholder(tf.bool, ())
# model
#model = model_mnist.RDPCNN(px_noised_data=px_holder, noise=noise_holder, label=label_holder, input_sigma=input_sigma, is_training=is_training)
model = model_mnist.RDPCNN(noised_data=noised_data_holder,
noise=noise_holder,
label=label_holder,
input_sigma=input_sigma,
is_training=is_training)
priv_accountant = accountant.GaussianMomentsAccountant(data.train_size)
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant,
[FLAGS.DP_GRAD_CLIPPING_L2NORM / FLAGS.BATCH_SIZE, True])
# model training
model_loss = model.loss()
model_loss_clean = model.loss_clean()
# training
#model_op, _, _, model_lr = model.optimization(model_loss)
model_op, model_lr = model.dp_optimization(model_loss,
gaussian_sanitizer,
sgd_sigma_holder,
FLAGS.BATCHES_PER_LOT)
# analysis
model_act, model_sigma_used, unmasked_sigma_used, acc_res = model.dp_accountant(
model_loss_clean, gaussian_sanitizer, total_dp_sigma, model_lr)
model_acc = model.cnn_accuracy
graph_dict = {}
graph_dict["px_holder"] = px_holder
graph_dict["data_holder"] = data_holder
graph_dict["noised_data_holder"] = noised_data_holder
graph_dict["noise_holder"] = noise_holder
graph_dict["label_holder"] = label_holder
graph_dict["sgd_sigma_holder"] = sgd_sigma_holder
graph_dict["is_training"] = is_training
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config, graph=g) as sess:
sess.run(tf.global_variables_initializer())
if FLAGS.load_model:
print("CNN loaded.")
model.tf_load(sess, name=FLAGS.CNN_CKPT_RESTORE_NAME)
if FLAGS.local:
total_train_lot = 2
total_valid_lot = 2
else:
total_train_lot = int(data.train_size / FLAGS.BATCH_SIZE /
FLAGS.BATCHES_PER_LOT)
total_valid_lot = None
print("Training...")
account_num = FLAGS.MAX_PARAM_SIZE
ori_account_num = FLAGS.ACCOUNT_NUM
sgd_sigma = [
np.zeros([FLAGS.BATCH_SIZE]) for _ in range(FLAGS.MAX_PARAM_SIZE)
]
#sgd_sigma = [np.ones([FLAGS.BATCH_SIZE])*total_dp_sigma for _ in range(FLAGS.MAX_PARAM_SIZE)]
itr_count = 0
for epoch in range(FLAGS.NUM_EPOCHS):
start_time = time.time()
for train_idx in range(total_train_lot):
terminate = False
for batch_idx in range(FLAGS.BATCHES_PER_LOT):
itr_count += 1
batch_xs, batch_ys, _ = data.next_train_batch(
FLAGS.BATCH_SIZE, True)
noise = np.random.normal(loc=0.0,
scale=input_sigma,
size=batch_xs.shape)
px_noised_xs = np.split(batch_xs + noise,
FLAGS.BATCH_SIZE,
axis=0)
feed_dict = {
noise_holder: noise,
noised_data_holder: batch_xs + noise,
label_holder: batch_ys,
is_training: True
}
for idx in range(len(sgd_sigma)):
feed_dict[sgd_sigma_holder[idx]] = sgd_sigma[idx]
'''
for idx in range(len(px_noised_xs)):
feed_dict[px_holder[idx]] = px_noised_xs[idx]
'''
sess.run(fetches=model_op, feed_dict=feed_dict)
#import pdb; pdb.set_trace()
#res = sess.run(acc_res, feed_dict=feed_dict)
#sess.run(unmasked_sigma_used, feed_dict=feed_dict)
#sess.run(model_sigma_used, feed_dict=feed_dict)
if itr_count == 1 or itr_count % FLAGS.DP_ACCOUNTANT_ITERATION == 0:
act_grads, sigma_used, unmasked = sess.run(
fetches=[
model_act, model_sigma_used,
unmasked_sigma_used
],
feed_dict=feed_dict)
# Heterogenerous: if sigma_used == 0.0, add avg additional sigma
var_used = np.zeros([FLAGS.BATCH_SIZE])
zero_count = np.zeros([FLAGS.BATCH_SIZE])
for sigma_ in sigma_used:
var_used += np.square(sigma_)
zero_count += np.array(sigma_ == 0.0, dtype=float)
avg_sgd_var = np.zeros([FLAGS.BATCH_SIZE])
total_sgd_var = total_dp_sigma**2 * len(
sigma_used) - var_used
#
heter_sigma_used = np.sqrt(var_used / len(sigma_used))
#
mask = np.array(zero_count != 0.0, dtype=int)
avg_sgd_var[mask == 1] = total_sgd_var[
mask == 1] / zero_count[mask == 1]
# filter
avg_sgd_var[avg_sgd_var <= 0] = 0
add_sgd_sigma = np.sqrt(avg_sgd_var)
add_sgd_sigma[(add_sgd_sigma > 0) & (
add_sgd_sigma < FLAGS.INPUT_DP_SIGMA_THRESHOLD
)] = FLAGS.TOTAL_DP_SIGMA
#
sgd_sigma = [
np.zeros([FLAGS.BATCH_SIZE])
for _ in range(len(sigma_used))
]
if np.any(add_sgd_sigma): # avg not all zero
account_num = 0
for idx in range(len(sigma_used)):
if np.any(sigma_used[idx]): # not all zero
account_num = max(account_num, idx + 1)
mask = np.array(sigma_used[idx] == 0.0,
dtype=int)
sgd_sigma[idx][mask == 1] = add_sgd_sigma[mask
== 1]
if np.random.rand() < 0.2:
FLAGS.ACCOUNT_NUM = len(sigma_used)
else:
FLAGS.ACCOUNT_NUM = account_num
else: # avg all zero
FLAGS.ACCOUNT_NUM = len(sigma_used)
print("Account num: ", FLAGS.ACCOUNT_NUM)
if itr_count > FLAGS.MAX_ITERATIONS:
terminate = True
#
'''
act_grads, sigma_used = sess.run(fetches=[model_act, model_sigma_used], feed_dict=feed_dict)
#import pdb; pdb.set_trace()
for idx in range(len(sigma_used)):
if np.any(sigma_used[idx]) == False: # all zero
FLAGS.ACCOUNT_NUM = min(FLAGS.ACCOUNT_NUM, idx + 1)
sgd_sigma_ = np.maximum(total_dp_sigma - sigma_used[idx], 0)
sgd_sigma_[sgd_sigma_<FLAGS.INPUT_DP_SIGMA_THRESHOLD] = 0
sgd_sigma[idx] = sgd_sigma_
print("Account num: ", FLAGS.ACCOUNT_NUM)
'''
# optimization
fetches = [model_loss, model_acc, model_lr]
loss, acc, lr = sess.run(fetches=fetches, feed_dict=feed_dict)
#import pdb; pdb.set_trace()
spent_eps_delta, selected_moment_orders = priv_accountant.get_privacy_spent(
sess, target_eps=[total_dp_epsilon])
spent_eps_delta = spent_eps_delta[0]
selected_moment_orders = selected_moment_orders[0]
if spent_eps_delta.spent_delta > total_dp_delta or spent_eps_delta.spent_eps > total_dp_epsilon:
terminate = True
# Print info
if train_idx % FLAGS.EVAL_TRAIN_FREQUENCY == (
FLAGS.EVAL_TRAIN_FREQUENCY - 1):
print("Sigma used:", sigma_used)
print("Heterogeneous Sigma used:", heter_sigma_used)
print("SGD Sigma:", sgd_sigma)
print("Epoch: {}".format(epoch))
print("Iteration: {}".format(itr_count))
print("Learning rate: {}".format(lr))
print("Loss: {:.4f}, Accuracy: {:.4f}".format(loss, acc))
print(
"Total dp eps: {:.4f}, total dp delta: {:.8f}, total dp sigma: {:.4f}, input sigma: {:.4f}"
.format(spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta, total_dp_sigma,
input_sigma))
print()
#model.tf_save(sess) # save checkpoint
if terminate:
break
end_time = time.time()
print('Eopch {} completed with time {:.2f} s'.format(
epoch + 1, end_time - start_time))
if epoch % FLAGS.EVAL_VALID_FREQUENCY == (
FLAGS.EVAL_VALID_FREQUENCY - 1):
# validation
print(
"\n******************************************************************"
)
print("Validation")
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_dp_sigma,
"input_sigma": input_sigma
}
valid_dict = test_info(sess,
model,
None,
graph_dict,
dp_info,
FLAGS.VALID_LOG_FILENAME,
total_batch=None,
valid=True)
np.save(FLAGS.DP_INFO_NPY, dp_info, allow_pickle=True)
ckpt_name = 'robust_dp_cnn.epoch{}.vloss{:.6f}.vacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch, valid_dict["loss"], valid_dict["acc"], input_sigma,
total_dp_sigma, spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta)
model.tf_save(sess, name=ckpt_name) # extra store
if terminate:
break
print(
"******************************************************************"
)
print()
print()
print("Optimization Finished!")
dp_info = {
"eps": spent_eps_delta.spent_eps,
"delta": spent_eps_delta.spent_delta,
"total_sigma": total_dp_sigma,
"input_sigma": input_sigma
}
valid_dict = test_info(sess,
model,
None,
graph_dict,
dp_info,
None,
total_batch=None,
valid=True)
np.save(FLAGS.DP_INFO_NPY, dp_info, allow_pickle=True)
ckpt_name = 'robust_dp_cnn.epoch{}.vloss{:.6f}.vacc{:.6f}.input_sigma{:.4f}.total_sigma{:.4f}.dp_eps{:.6f}.dp_delta{:.6f}.ckpt'.format(
epoch, valid_dict["loss"], valid_dict["acc"], input_sigma,
total_dp_sigma, spent_eps_delta.spent_eps,
spent_eps_delta.spent_delta)
model.tf_save(sess, name=ckpt_name) # extra store
def Train(train_file,
test_file,
network_parameters,
num_steps,
save_path,
total_rho,
eval_steps=0):
"""Train MNIST for a number of steps.
Args:
mnist_train_file: path of MNIST train data file.
mnist_test_file: path of MNIST test data file.
network_parameters: parameters for defining and training the network.
num_steps: number of steps to run. Here steps = lots
save_path: path where to save trained parameters.
eval_steps: evaluate the model every eval_steps.
Returns:
the result after the final training step.
Raises:
ValueError: if the accountant_type is not supported.
"""
batch_size = NUM_TRAINING_IMAGES
params = {
"accountant_type": FLAGS.accountant_type,
"task_id": 0,
"batch_size": FLAGS.batch_size,
"default_gradient_l2norm_bound":
network_parameters.default_gradient_l2norm_bound,
"num_examples": NUM_TRAINING_IMAGES,
"learning_rate": FLAGS.lr,
"end_learning_rate": FLAGS.end_lr,
"learning_rate_saturate_epochs": FLAGS.lr_saturate_epochs
}
# Log different privacy parameters dependent on the accountant type.
if FLAGS.accountant_type == "Amortized":
params.update({
"flag_eps": FLAGS.eps,
"flag_delta": FLAGS.delta,
})
elif FLAGS.accountant_type == "Moments":
params.update({
"sigma": FLAGS.sigma,
})
with tf.device('/gpu:0'), tf.Graph().as_default(), tf.Session() as sess:
#print_csv_tfrecords.print_tfrecords(train_file)
features, labels = DataInput(train_file, batch_size, False)
print("network_parameters.input_size", network_parameters.input_size)
logits, projection, training_params = utils.BuildNetwork(
features, network_parameters)
cost = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf.one_hot(
labels, LABEL_SIZE))
# The actual cost is the average across the examples.
cost = tf.reduce_sum(cost, [0]) / batch_size
if FLAGS.accountant_type == "Amortized":
priv_accountant = accountant.AmortizedAccountant(
NUM_TRAINING_IMAGES)
sigma = None
elif FLAGS.accountant_type == "Moments":
priv_accountant = accountant.GaussianMomentsAccountant(
NUM_TRAINING_IMAGES)
sigma = FLAGS.sigma
elif FLAGS.accountant_type == "ZDCP":
priv_accountant = accountant.DumpzCDPAccountant()
else:
raise ValueError("Undefined accountant type, needs to be "
"Amortized or Moments, but got %s" %
FLAGS.accountant)
# Note: Here and below, we scale down the l2norm_bound by
# batch_size. This is because per_example_gradients computes the
# gradient of the minibatch loss with respect to each individual
# example, and the minibatch loss (for our model) is the *average*
# loss over examples in the minibatch. Hence, the scale of the
# per-example gradients goes like 1 / batch_size.
gaussian_sanitizer = sanitizer.AmortizedGaussianSanitizer(
priv_accountant, [
network_parameters.default_gradient_l2norm_bound / batch_size,
True
])
for var in training_params:
if "gradient_l2norm_bound" in training_params[var]:
l2bound = training_params[var][
"gradient_l2norm_bound"] / batch_size
gaussian_sanitizer.set_option(
var, sanitizer.ClipOption(l2bound, True))
lr = tf.placeholder(tf.float32)
eps = tf.placeholder(tf.float32)
delta = tf.placeholder(tf.float32)
varsigma = tf.placeholder(tf.float32, shape=[])
init_ops = []
# Add global_step
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
with_privacy = True
if with_privacy:
gd_op = dp_optimizer.DPGradientDescentOptimizer(
lr, [eps, delta],
gaussian_sanitizer,
varsigma,
batches_per_lot=FLAGS.batches_per_lot).minimize(
cost, global_step=global_step)
else:
print("No privacy")
gd_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
saver = tf.train.Saver()
coord = tf.train.Coordinator()
_ = tf.train.start_queue_runners(sess=sess, coord=coord)
# We need to maintain the intialization sequence.
for v in tf.trainable_variables():
sess.run(tf.variables_initializer([v]))
sess.run(tf.global_variables_initializer())
sess.run(init_ops)
results = []
start_time = time.time()
prev_time = start_time
filename = "results-0.json"
log_path = os.path.join(save_path, filename)
target_eps = [float(s) for s in FLAGS.target_eps.split(",")]
if FLAGS.accountant_type == "Amortized":
# Only matters if --terminate_based_on_privacy is true.
target_eps = [max(target_eps)]
max_target_eps = max(target_eps)
lot_size = FLAGS.batches_per_lot * FLAGS.batch_size
lots_per_epoch = NUM_TRAINING_IMAGES / lot_size
curr_sigma = 0
previous_epoch = -1
rho_tracking = [0]
for step in range(num_steps):
epoch = step // lots_per_epoch
curr_lr = utils.VaryRate(FLAGS.lr, FLAGS.end_lr,
FLAGS.lr_saturate_epochs, epoch)
curr_eps = utils.VaryRate(FLAGS.eps, FLAGS.end_eps,
FLAGS.eps_saturate_epochs, epoch)
if with_privacy:
old_sigma = curr_sigma
#total budget
rhototal = total_rho
curr_sigma = get_current_sigma(epoch)
if step % 100 == 0:
print(curr_sigma)
print(rho_tracking[-1])
if epoch - previous_epoch == 1:
rho_tracking.append(rho_tracking[-1] + 1.0 /
(2.0 * curr_sigma**2))
previous_epoch = epoch
if with_privacy == True and rho_tracking[-1] > rhototal:
print("stop at epoch%d" % epoch)
print(rho_tracking[:-1])
break
for _ in range(FLAGS.batches_per_lot):
_ = sess.run(
[gd_op],
feed_dict={
lr: curr_lr,
eps: curr_eps,
delta: FLAGS.delta,
varsigma: curr_sigma
})
sys.stderr.write("step: %d\n" % step)
# See if we should stop training due to exceeded privacy budget:
should_terminate = False
if (eval_steps > 0 and
(step + 1) % eval_steps == 0) or should_terminate:
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(
train_file,
network_parameters,
num_testing_images=NUM_TRAINING_IMAGES,
randomize=False,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(
test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step + 1, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
if should_terminate:
break
print(rho_tracking[:-1])
saver.save(sess, save_path=save_path + "/ckpt")
train_accuracy, _ = Eval(train_file,
network_parameters,
num_testing_images=NUM_TRAINING_IMAGES,
randomize=False,
load_path=save_path)
sys.stderr.write("train_accuracy: %.2f\n" % train_accuracy)
test_accuracy, mistakes = Eval(test_file,
network_parameters,
num_testing_images=NUM_TESTING_IMAGES,
randomize=False,
load_path=save_path,
save_mistakes=FLAGS.save_mistakes)
sys.stderr.write("eval_accuracy: %.2f\n" % test_accuracy)
curr_time = time.time()
elapsed_time = curr_time - prev_time
prev_time = curr_time
results.append({
"step": step + 1, # Number of lots trained so far.
"elapsed_secs": elapsed_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"mistakes": mistakes
})
loginfo = {
"elapsed_secs": curr_time - start_time,
"train_accuracy": train_accuracy,
"test_accuracy": test_accuracy,
"num_training_steps": step, # Steps so far.
"mistakes": mistakes,
"result_series": results
}
loginfo.update(params)
if log_path:
with tf.gfile.Open(log_path, "w") as f:
json.dump(loginfo, f, indent=2)
f.write("\n")
f.close()
