def train():
"""Train CIFAR-10 for a number of steps."""
with tf.Graph().as_default():
global_step = tf.Variable(0, trainable=False)
# Get images and labels for CIFAR-10.
images, labels = cifar10.distorted_inputs()
labels = tf.one_hot(labels, 10)
# Build a Graph that computes the logits predictions from the
# inference model.
#logits = inference(images)
with tf.variable_scope('conv1') as scope:
kernel1 = _variable_with_weight_decay('weights',
shape=[3, 3, 3, 128],
stddev=5e-2,
wd=None)
conv = tf.nn.conv2d(images, kernel1, [1, 2, 2, 1], padding='SAME')
#conv = tf.nn.dropout(conv, 0.9)
biases1 = cifar10._variable_on_cpu('biases', [128], tf.constant_initializer(0.0))
pre_activation = tf.nn.bias_add(conv, biases1)
conv1 = tf.nn.relu(pre_activation, name = scope.name)
cifar10._activation_summary(conv1)
norm1 = tf.contrib.layers.batch_norm(conv1, scale=True, is_training=True, updates_collections=None)
# conv2
with tf.variable_scope('conv2') as scope:
kernel2 = _variable_with_weight_decay('weights',
shape=[5, 5, 128, 128],
stddev=5e-2,
wd=None)
conv = tf.nn.conv2d(norm1, kernel2, [1, 1, 1, 1], padding='SAME')
biases2 = cifar10._variable_on_cpu('biases', [128], tf.constant_initializer(0.1))
pre_activation = tf.nn.bias_add(conv, biases2)
conv2 = tf.nn.relu(pre_activation, name = scope.name)
#conv2 = tf.nn.dropout(conv2, 0.9)
cifar10._activation_summary(conv2)
# concat conv2 with norm1 to increase the number of features, this step does not affect the privacy preserving guarantee
current = tf.concat((conv2, norm1), axis=3)
# norm2
norm2 = tf.contrib.layers.batch_norm(current, scale=True, is_training=True, updates_collections=None)
# conv3
with tf.variable_scope('conv3') as scope:
kernel3 = _variable_with_weight_decay('weights',
shape=[5, 5, 256, 256],
stddev=5e-2,
wd=None)
conv = tf.nn.conv2d(norm2, kernel3, [1, 1, 1, 1], padding='SAME')
biases3 = cifar10._variable_on_cpu('biases', [256], tf.constant_initializer(0.1))
pre_activation = tf.nn.bias_add(conv, biases3)
conv3 = tf.nn.relu(pre_activation, name = scope.name)
#conv3 = tf.nn.dropout(conv3, 0.9)
cifar10._activation_summary(conv3)
# norm3
norm3 = tf.contrib.layers.batch_norm(conv3, scale=True, is_training=True, updates_collections=None)
#pool3, row_pooling_sequence, col_pooling_sequence = tf.nn.fractional_max_pool(norm3, pooling_ratio=[1.0, 2.0, 2.0, 1.0])
pool3 = avg_pool(norm3, 2)
# local4
with tf.variable_scope('local4') as scope:
weights1 = cifar10._variable_with_weight_decay('weights', shape=[5*5*256, hk],
stddev=0.04, wd=None)
biases4 = cifar10._variable_on_cpu('biases', [hk], tf.constant_initializer(0.1))
h_pool2_flat = tf.reshape(pool3, [-1, 5*5*256]);
z2 = tf.add(tf.matmul(h_pool2_flat, weights1), biases4, name=scope.name)
#Applying normalization for the flat connected layer h_fc1#
batch_mean2, batch_var2 = tf.nn.moments(z2,[0])
scale2 = tf.Variable(tf.ones([hk]))
beta2 = tf.Variable(tf.zeros([hk]))
BN_norm = tf.nn.batch_normalization(z2,batch_mean2,batch_var2,beta2,scale2,1e-3)
###
local4 = max_out(BN_norm, hk)
cifar10._activation_summary(local4)
# linear layer(WX + b),
# We don't apply softmax here because
# tf.nn.sparse_softmax_cross_entropy_with_logits accepts the unscaled logits
# and performs the softmax internally for efficiency.
weights2 = cifar10._variable_with_weight_decay('weights', [hk, 10],
stddev=1/(hk*1.0), wd=0.0)
biases5 = cifar10._variable_on_cpu('biases', [10], tf.constant_initializer(0.0))
logits = tf.add(tf.matmul(local4, weights2), biases5, name=scope.name)
cifar10._activation_summary(logits)
# Calculate loss. Apply Taylor Expansion for the output layer
#loss = cifar10.loss(logits, labels)
# Calculate the average cross entropy loss across the batch.
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
labels=labels, logits=logits, name='cross_entropy_per_example')
loss = tf.reduce_mean(cross_entropy, name='cross_entropy')
opt = tf.train.GradientDescentOptimizer(lr)
gw_K1 = tf.gradients(loss, kernel1)[0]
gb1 = tf.gradients(loss, biases1)[0]
gw_K2 = tf.gradients(loss, kernel2)[0]
gb2 = tf.gradients(loss, biases2)[0]
gw_K3 = tf.gradients(loss, kernel3)[0]
gb3 = tf.gradients(loss, biases3)[0]
gw_W1 = tf.gradients(loss, weights1)[0]
gb4 = tf.gradients(loss, biases4)[0]
gw_W2 = tf.gradients(loss, weights2)[0]
gb5 = tf.gradients(loss, biases5)[0]
#clip gradient
gw_K1 = tf.clip_by_norm(gw_K1,clip_bound)
gw_K2 = tf.clip_by_norm(gw_K2,clip_bound)
gw_K3 = tf.clip_by_norm(gw_K3,clip_bound)
gw_W1 = tf.clip_by_norm(gw_W1,clip_bound)
gw_W2 = tf.clip_by_norm(gw_W2,clip_bound)
#perturb
gw_K1 += tf.random_normal(shape=tf.shape(gw_K1), mean=0.0, stddev = sigma * (sensitivity**2), dtype=tf.float32)
gw_K2 += tf.random_normal(shape=tf.shape(gw_K2), mean=0.0, stddev = sigma * (sensitivity**2), dtype=tf.float32)
gw_K3 += tf.random_normal(shape=tf.shape(gw_K3), mean=0.0, stddev = sigma * (sensitivity**2), dtype=tf.float32)
gw_W1 += tf.random_normal(shape=tf.shape(gw_W1), mean=0.0, stddev = sigma * (sensitivity**2), dtype=tf.float32)
gw_W2 += tf.random_normal(shape=tf.shape(gw_W2), mean=0.0, stddev = sigma * (sensitivity**2), dtype=tf.float32)
gb1 += tf.random_normal(shape=tf.shape(gb1), mean=0.0, stddev = sigma * (sensitivity**2), dtype=tf.float32)
gb2 += tf.random_normal(shape=tf.shape(gb2), mean=0.0, stddev = sigma * (sensitivity**2), dtype=tf.float32)
gb3 += tf.random_normal(shape=tf.shape(gb3), mean=0.0, stddev = sigma * (sensitivity**2), dtype=tf.float32)
gb4 += tf.random_normal(shape=tf.shape(gb4), mean=0.0, stddev = sigma * (sensitivity**2), dtype=tf.float32)
gb5 += tf.random_normal(shape=tf.shape(gb5), mean=0.0, stddev = sigma * (sensitivity**2), dtype=tf.float32)
# apply gradients and keep tracking moving average of the parameters
apply_gradient_op = opt.apply_gradients([(gw_K1,kernel1),(gb1,biases1),(gw_K2,kernel2),(gb2,biases2),(gw_K3,kernel3),(gb3,biases3),(gw_W1,weights1),(gb4,biases4),(gw_W2,weights2),(gb5,biases5)], global_step=global_step);
variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
with tf.control_dependencies([apply_gradient_op, variables_averages_op]):
train_op = tf.no_op(name='train')
# Create a saver.
saver = tf.train.Saver(tf.all_variables())
# Privacy accountant
priv_accountant = accountant.GaussianMomentsAccountant(D)
privacy_accum_op = priv_accountant.accumulate_privacy_spending([None, None], sigma, batch_size)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
# Build an initialization operation to run below.
init = tf.initialize_all_variables()
# Start running operations on the Graph.
sess = tf.Session(config=tf.ConfigProto(log_device_placement=False))
sess.run(init)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
summary_writer = tf.summary.FileWriter(os.getcwd() + '/tmp/cifar10_train', sess.graph)
# load the most recent models
_global_step = 0
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
print(ckpt.model_checkpoint_path);
saver.restore(sess, ckpt.model_checkpoint_path)
_global_step = int(ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1])
else:
print('No checkpoint file found')
T = int(int(math.ceil(D/batch_size))*epochs + 1) # number of steps
step_for_epoch = int(math.ceil(D/batch_size)); #number of steps for one epoch
for step in xrange(_global_step, _global_step + T):
start_time = time.time()
_, loss_value = sess.run([train_op, loss])
duration = time.time() - start_time
assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
# report the result periodically
if step % (5*step_for_epoch) == 0:
num_examples_per_step = batch_size
examples_per_sec = num_examples_per_step / duration
sec_per_batch = float(duration)
format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
'sec/batch)')
print (format_str % (datetime.now(), step, loss_value,
examples_per_sec, sec_per_batch))
if step % (5*step_for_epoch) == 0:
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, step)
# Save the model checkpoint periodically.
if step % (5*step_for_epoch) == 0 and (step > _global_step):
checkpoint_path = os.path.join(os.getcwd() + '/tmp/cifar10_train', 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step);
sess.run([privacy_accum_op])
spent_eps_deltas = priv_accountant.get_privacy_spent(sess, target_eps=target_eps)
print(step, spent_eps_deltas)
sess.run([privacy_accum_op])
spent_eps_deltas = priv_accountant.get_privacy_spent(sess, target_eps=target_eps)
_break = False;
for _eps, _delta in spent_eps_deltas:
if _delta >= delta:
_break = True;
break;
if _break == True:
break;
def inference(images):
"""Build the CIFAR-10 model.
Args:
images: Images returned from distorted_inputs() or inputs().
Returns:
Logits.
"""
###
# We instantiate all variables using tf.get_variable() instead of
# tf.Variable() in order to share variables across multiple GPU training runs.
# If we only ran this model on a single GPU, we could simplify this function
# by replacing all instances of tf.get_variable() with tf.Variable().
#
# conv1
#xavier = tf.contrib.layers.xavier_initializer_conv2d()
with tf.variable_scope('conv1') as scope:
kernel1 = _variable_with_weight_decay('weights',
shape=[3, 3, 3, 128],
stddev=5e-2,
wd=None)
conv = tf.nn.conv2d(images, kernel1, [1, 2, 2, 1], padding='SAME')
#conv = tf.nn.dropout(conv, 0.9)
biases1 = cifar10._variable_on_cpu('biases', [128], tf.constant_initializer(0.0))
pre_activation = tf.nn.bias_add(conv, biases1)
conv1 = tf.nn.relu(pre_activation, name = scope.name)
cifar10._activation_summary(conv1)
norm1 = tf.contrib.layers.batch_norm(conv1, scale=True, is_training=True, updates_collections=None)
# conv2
with tf.variable_scope('conv2') as scope:
kernel2 = _variable_with_weight_decay('weights',
shape=[5, 5, 128, 128],
stddev=5e-2,
wd=None)
conv = tf.nn.conv2d(norm1, kernel2, [1, 1, 1, 1], padding='SAME')
biases2 = cifar10._variable_on_cpu('biases', [128], tf.constant_initializer(0.1))
pre_activation = tf.nn.bias_add(conv, biases2)
conv2 = tf.nn.relu(pre_activation, name = scope.name)
#conv2 = tf.nn.dropout(conv2, 0.9)
cifar10._activation_summary(conv2)
# concat conv2 with norm1 to increase the number of features, this step does not affect the privacy preserving guarantee
current = tf.concat((conv2, norm1), axis=3)
# norm2
norm2 = tf.contrib.layers.batch_norm(current, scale=True, is_training=True, updates_collections=None)
# conv3
with tf.variable_scope('conv3') as scope:
kernel3 = _variable_with_weight_decay('weights',
shape=[5, 5, 256, 256],
stddev=5e-2,
wd=None)
conv = tf.nn.conv2d(norm2, kernel3, [1, 1, 1, 1], padding='SAME')
biases3 = cifar10._variable_on_cpu('biases', [256], tf.constant_initializer(0.1))
pre_activation = tf.nn.bias_add(conv, biases3)
conv3 = tf.nn.relu(pre_activation, name = scope.name)
#conv3 = tf.nn.dropout(conv3, 0.9)
cifar10._activation_summary(conv3)
# norm3
norm3 = tf.contrib.layers.batch_norm(conv3, scale=True, is_training=True, updates_collections=None)
#pool3, row_pooling_sequence, col_pooling_sequence = tf.nn.fractional_max_pool(norm3, pooling_ratio=[1.0, 2.0, 2.0, 1.0])
pool3 = avg_pool(norm3, 2)
# local4
with tf.variable_scope('local4') as scope:
weights1 = cifar10._variable_with_weight_decay('weights', shape=[5*5*256, hk],
stddev=0.04, wd=None)
biases4 = cifar10._variable_on_cpu('biases', [hk], tf.constant_initializer(0.1))
h_pool2_flat = tf.reshape(pool3, [-1, 5*5*256]);
z2 = tf.add(tf.matmul(h_pool2_flat, weights1), biases4, name=scope.name)
#Applying normalization for the flat connected layer h_fc1#
batch_mean2, batch_var2 = tf.nn.moments(z2,[0])
scale2 = tf.Variable(tf.ones([hk]))
beta2 = tf.Variable(tf.zeros([hk]))
BN_norm = tf.nn.batch_normalization(z2,batch_mean2,batch_var2,beta2,scale2,1e-3)
###
local4 = max_out(BN_norm, hk)
cifar10._activation_summary(local4)
"""print(images.get_shape());
print(norm1.get_shape());
print(norm2.get_shape());
print(pool3.get_shape());
print(local4.get_shape());"""
# linear layer(WX + b),
# We don't apply softmax here because
# tf.nn.sparse_softmax_cross_entropy_with_logits accepts the unscaled logits
# and performs the softmax internally for efficiency.
weights2 = cifar10._variable_with_weight_decay('weights', [hk, 10],
stddev=1/(hk*1.0), wd=0.0)
biases5 = cifar10._variable_on_cpu('biases', [10],
tf.constant_initializer(0.0))
softmax_linear = tf.add(tf.matmul(local4, weights2), biases5, name=scope.name)
cifar10._activation_summary(softmax_linear)
return softmax_linear
def test_inference(images, params, image_size, adv_noise):
kernel1 = params[0]
biases1 = params[1]
with tf.variable_scope('conv1') as scope:
conv = tf.nn.conv2d(images + adv_noise,
kernel1, [1, 2, 2, 1],
padding='SAME')
#conv = tf.nn.dropout(conv, 1.0)
pre_activation = tf.nn.bias_add(conv, biases1)
conv1 = tf.nn.relu(pre_activation, name=scope.name)
cifar10._activation_summary(conv1)
# norm1
norm1 = tf.contrib.layers.batch_norm(conv1,
scale=True,
is_training=True,
updates_collections=None)
# conv2
kernel2 = params[2]
biases2 = params[3]
with tf.variable_scope('conv2') as scope:
conv = tf.nn.conv2d(norm1, kernel2, [1, 1, 1, 1], padding='SAME')
pre_activation = tf.nn.bias_add(conv, biases2)
conv2 = tf.nn.relu(pre_activation, name=scope.name)
#conv2 = tf.nn.dropout(conv2, 1.0)
cifar10._activation_summary(conv2)
# concat conv2 with norm1 to increase the number of features, this step does not affect the privacy preserving guarantee
current = tf.concat((conv2, norm1), axis=3)
# norm2
norm2 = tf.contrib.layers.batch_norm(current,
scale=True,
is_training=False,
updates_collections=None)
# conv3
kernel3 = params[4]
biases3 = params[5]
with tf.variable_scope('conv3') as scope:
conv = tf.nn.conv2d(norm2, kernel3, [1, 1, 1, 1],
padding='SAME') #noiseless model
pre_activation = tf.nn.bias_add(conv, biases3)
conv3 = tf.nn.relu(pre_activation, name=scope.name)
#conv3 = tf.nn.dropout(conv3, 1.0)
cifar10._activation_summary(conv3)
# norm3
norm3 = tf.contrib.layers.batch_norm(conv3,
scale=True,
is_training=False,
updates_collections=None)
#pool3, row_pooling_sequence, col_pooling_sequence = tf.nn.fractional_max_pool(norm3, pooling_ratio=[1.0, 2.0, 2.0, 1.0])
pool3 = avg_pool(norm3, 2)
# local4, note that we do not need to inject Laplace noise into the testing phase
kernel4 = params[6]
biases4 = params[7]
with tf.variable_scope('local4') as scope:
h_pool2_flat = tf.reshape(pool3, [-1, int(image_size / 4)**2 * 256])
z2 = tf.add(tf.matmul(h_pool2_flat, kernel4), biases4, name=scope.name)
#Applying normalization for the flat connected layer h_fc1#
#batch_mean2, batch_var2 = tf.nn.moments(z2,[0])
#scale2 = params[10] #tf.Variable(tf.ones([hk]))
#beta2 = params[11] #tf.Variable(tf.zeros([hk]))
BN_norm = tf.contrib.layers.batch_norm(z2,
scale=True,
is_training=False,
updates_collections=None)
#BN_norm = tf.nn.batch_normalization(z2,batch_mean2,batch_var2,beta2,scale2,1e-3)
###
local4 = max_out(BN_norm, hk)
local4 = tf.clip_by_value(local4, -1, 1)
cifar10._activation_summary(local4)
#with tf.variable_scope('local5') as scope:
kernel5 = params[8]
biases5 = params[9]
softmax_linear = tf.add(tf.matmul(local4, kernel5),
biases5,
name=scope.name)
cifar10._activation_summary(softmax_linear)
return softmax_linear
def inference(images, perturbH, params):
"""Build the CIFAR-10 model.
Args:
images: Images returned from distorted_inputs() or inputs().
Returns:
Logits.
"""
###
# We instantiate all variables using tf.get_variable() instead of
# tf.Variable() in order to share variables across multiple GPU training runs.
# If we only ran this model on a single GPU, we could simplify this function
# by replacing all instances of tf.get_variable() with tf.Variable().
#
# conv1
#xavier = tf.contrib.layers.xavier_initializer_conv2d()
with tf.variable_scope('conv1') as scope:
#kernel = _variable_with_weight_decay('weights',
# shape=[3, 3, 3, 128],
# stddev=5e-2,
# wd=0.0)
conv = tf.nn.conv2d(images, params[0], [1, 2, 2, 1],
padding='SAME') + perturbH
#conv = tf.nn.dropout(conv, 0.9)
#biases = cifar10._variable_on_cpu('biases', [128], tf.constant_initializer(0.0))
pre_activation = tf.nn.bias_add(conv, params[1])
conv1 = tf.nn.relu(pre_activation, name=scope.name)
cifar10._activation_summary(conv1)
norm1 = tf.contrib.layers.batch_norm(conv1,
scale=True,
is_training=True,
updates_collections=[CONV_VARIABLES])
# conv2
with tf.variable_scope('conv2') as scope:
#kernel = _variable_with_weight_decay('weights',
# shape=[5, 5, 128, 128],
# stddev=5e-2,
# wd=0.0)
conv = tf.nn.conv2d(norm1, params[2], [1, 1, 1, 1], padding='SAME')
#biases = cifar10._variable_on_cpu('biases', [128], tf.constant_initializer(0.1))
pre_activation = tf.nn.bias_add(conv, params[3])
conv2 = tf.nn.relu(pre_activation, name=scope.name)
#conv2 = tf.nn.dropout(conv2, 0.9)
cifar10._activation_summary(conv2)
# concat conv2 with norm1 to increase the number of features, this step does not affect the privacy preserving guarantee
current = tf.concat((conv2, norm1), axis=3)
# norm2
norm2 = tf.contrib.layers.batch_norm(current,
scale=True,
is_training=True,
updates_collections=[CONV_VARIABLES])
# conv3
with tf.variable_scope('conv3') as scope:
#kernel = _variable_with_weight_decay('weights',
# shape=[5, 5, 256, 256],
# stddev=5e-2,
# wd=0.0)
conv = tf.nn.conv2d(norm2, params[4], [1, 1, 1, 1], padding='SAME')
#biases = cifar10._variable_on_cpu('biases', [256], tf.constant_initializer(0.1))
pre_activation = tf.nn.bias_add(conv, params[5])
conv3 = tf.nn.relu(pre_activation, name=scope.name)
#conv3 = tf.nn.dropout(conv3, 0.9)
cifar10._activation_summary(conv3)
# norm3
norm3 = tf.contrib.layers.batch_norm(conv3,
scale=True,
is_training=True,
updates_collections=[CONV_VARIABLES])
#pool3, row_pooling_sequence, col_pooling_sequence = tf.nn.fractional_max_pool(norm3, pooling_ratio=[1.0, 2.0, 2.0, 1.0])
pool3 = avg_pool(norm3, 2)
# local4
with tf.variable_scope('local4') as scope:
#weights = cifar10._variable_with_weight_decay('weights', shape=[5*5*256, hk],
# stddev=0.04, wd=0.004)
#biases = cifar10._variable_on_cpu('biases', [hk], tf.constant_initializer(0.1))
h_pool2_flat = tf.reshape(pool3, [-1, int(image_size / 4)**2 * 256])
z2 = tf.add(tf.matmul(h_pool2_flat, params[6]),
params[7],
name=scope.name)
#Applying normalization for the flat connected layer h_fc1#
#batch_mean2, batch_var2 = tf.nn.moments(z2,[0])
#scale2 = tf.Variable(tf.ones([hk]))
#beta2 = tf.Variable(tf.zeros([hk]))
BN_norm = tf.contrib.layers.batch_norm(
z2, scale=True, is_training=True, updates_collections=[CONV_VARIABLES])
###
local4 = max_out(BN_norm, hk)
local4 = tf.clip_by_value(local4, -1,
1) # hidden neurons must be bounded in [-1, 1]
#local4 += perturbFM; # perturb hidden neurons, which are considered coefficients in the differentially private logistic regression layer
cifar10._activation_summary(local4)
"""print(images.get_shape());
print(norm1.get_shape());
print(norm2.get_shape());
print(pool3.get_shape());
print(local4.get_shape());"""
# linear layer(WX + b),
# We don't apply softmax here because
# tf.nn.sparse_softmax_cross_entropy_with_logits accepts the unscaled logits
# and performs the softmax internally for efficiency.
#with tf.variable_scope('local5') as scope:
#weights = cifar10._variable_with_weight_decay('weights', [hk, 10],
# stddev=1/(hk*1.0), wd=0.0)
#biases = cifar10._variable_on_cpu('biases', [10],
# tf.constant_initializer(0.0))
softmax_linear = tf.add(tf.matmul(local4, params[8]),
params[9],
name=scope.name)
cifar10._activation_summary(softmax_linear)
return softmax_linear
def inference(images, params, dp_mult):
"""Build the CIFAR-10 model.
Args:
images: Images returned from distorted_inputs() or inputs().
Returns:
Logits.
"""
###
# We instantiate all variables using tf.get_variable() instead of
# tf.Variable() in order to share variables across multiple GPU training runs.
# If we only ran this model on a single GPU, we could simplify this function
# by replacing all instances of tf.get_variable() with tf.Variable().
#
# conv1
#xavier = tf.contrib.layers.xavier_initializer_conv2d()
with tf.variable_scope('conv1') as scope:
conv = tf.nn.conv2d(images, params[0], [1, 2, 2, 1], padding='SAME')
#conv = tf.nn.dropout(conv, 0.9)
pre_activation = tf.nn.bias_add(conv, params[1])
conv1 = tf.nn.relu(pre_activation + dp_mult, name=scope.name)
cifar10._activation_summary(conv1)
norm1 = tf.contrib.layers.batch_norm(conv1,
scale=True,
is_training=True,
updates_collections=None)
# conv2
with tf.variable_scope('conv2') as scope:
conv = tf.nn.conv2d(norm1, params[2], [1, 1, 1, 1], padding='SAME')
pre_activation = tf.nn.bias_add(conv, params[3])
conv2 = tf.nn.relu(pre_activation, name=scope.name)
#conv2 = tf.nn.dropout(conv2, 0.9)
cifar10._activation_summary(conv2)
# concat conv2 with norm1 to increase the number of features, this step does not affect the privacy preserving guarantee
current = tf.concat((conv2, norm1), axis=3)
# norm2
norm2 = tf.contrib.layers.batch_norm(current,
scale=True,
is_training=True,
updates_collections=None)
# conv3
with tf.variable_scope('conv3') as scope:
conv = tf.nn.conv2d(norm2, params[4], [1, 1, 1, 1], padding='SAME')
pre_activation = tf.nn.bias_add(conv, params[5])
conv3 = tf.nn.relu(pre_activation, name=scope.name)
#conv3 = tf.nn.dropout(conv3, 0.9)
cifar10._activation_summary(conv3)
# norm3
norm3 = tf.contrib.layers.batch_norm(conv3,
scale=True,
is_training=True,
updates_collections=None)
#pool3, row_pooling_sequence, col_pooling_sequence = tf.nn.fractional_max_pool(norm3, pooling_ratio=[1.0, 2.0, 2.0, 1.0])
pool3 = avg_pool(norm3, 2)
# local4
with tf.variable_scope('local4') as scope:
h_pool2_flat = tf.reshape(pool3, [-1, int(image_size / 4)**2 * 256])
z2 = tf.add(tf.matmul(h_pool2_flat, params[6]),
params[7],
name=scope.name)
#Applying normalization for the flat connected layer h_fc1#
batch_mean2, batch_var2 = tf.nn.moments(z2, [0])
BN_norm = tf.nn.batch_normalization(z2, batch_mean2, batch_var2,
params[11], params[10], 1e-3)
###
local4 = max_out(BN_norm, hk)
cifar10._activation_summary(local4)
"""print(images.get_shape());
print(norm1.get_shape());
print(norm2.get_shape());
print(pool3.get_shape());
print(local4.get_shape());"""
# linear layer(WX + b),
# We don't apply softmax here because
# tf.nn.sparse_softmax_cross_entropy_with_logits accepts the unscaled logits
# and performs the softmax internally for efficiency.
softmax_linear = tf.add(tf.matmul(local4, params[8]),
params[9],
name=scope.name)
cifar10._activation_summary(softmax_linear)
return softmax_linear, conv1
def test_inference(images):
with tf.variable_scope('conv1') as scope:
kernel = _variable_with_weight_decay('weights',
shape=[3, 3, 3, 128],
stddev=5e-2,
wd=0.0)
conv = tf.nn.conv2d(images, kernel, [1, 2, 2, 1], padding='SAME')
#conv = tf.nn.dropout(conv, 1.0)
biases = cifar10._variable_on_cpu('biases', [128],
tf.constant_initializer(0.0))
pre_activation = tf.nn.bias_add(conv, biases)
conv1 = tf.nn.relu(pre_activation, name=scope.name)
cifar10._activation_summary(conv1)
# norm1
norm1 = tf.contrib.layers.batch_norm(conv1,
scale=True,
is_training=True,
updates_collections=None)
# conv2
with tf.variable_scope('conv2') as scope:
kernel = _variable_with_weight_decay('weights',
shape=[5, 5, 128, 128],
stddev=5e-2,
wd=0.0)
conv = tf.nn.conv2d(norm1, kernel, [1, 1, 1, 1], padding='SAME')
biases = cifar10._variable_on_cpu('biases', [128],
tf.constant_initializer(0.1))
pre_activation = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(pre_activation, name=scope.name)
#conv2 = tf.nn.dropout(conv2, 1.0)
cifar10._activation_summary(conv2)
# concat conv2 with norm1 to increase the number of features, this step does not affect the privacy preserving guarantee
current = tf.concat((conv2, norm1), axis=3)
# norm2
norm2 = tf.contrib.layers.batch_norm(current,
scale=True,
is_training=True,
updates_collections=None)
# conv3
with tf.variable_scope('conv3') as scope:
kernel = _variable_with_weight_decay('weights',
shape=[5, 5, 256, 256],
stddev=5e-2,
wd=0.0)
conv = tf.nn.conv2d(norm2, kernel, [1, 1, 1, 1],
padding='SAME') #noiseless model
biases = cifar10._variable_on_cpu('biases', [256],
tf.constant_initializer(0.1))
pre_activation = tf.nn.bias_add(conv, biases)
conv3 = tf.nn.relu(pre_activation, name=scope.name)
#conv3 = tf.nn.dropout(conv3, 1.0)
cifar10._activation_summary(conv3)
# norm3
norm3 = tf.contrib.layers.batch_norm(conv3,
scale=True,
is_training=True,
updates_collections=None)
#pool3, row_pooling_sequence, col_pooling_sequence = tf.nn.fractional_max_pool(norm3, pooling_ratio=[1.0, 2.0, 2.0, 1.0])
pool3 = avg_pool(norm3, 2)
# local4, note that we do not need to inject Laplace noise into the testing phase
with tf.variable_scope('local4') as scope:
weights = cifar10._variable_with_weight_decay('weights',
shape=[5 * 5 * 256, hk],
stddev=0.04,
wd=0.004)
biases = cifar10._variable_on_cpu('biases', [hk],
tf.constant_initializer(0.1))
h_pool2_flat = tf.reshape(pool3, [-1, 5 * 5 * 256])
z2 = tf.add(tf.matmul(h_pool2_flat, weights), biases, name=scope.name)
#Applying normalization for the flat connected layer h_fc1#
batch_mean2, batch_var2 = tf.nn.moments(z2, [0])
scale2 = tf.Variable(tf.ones([hk]))
beta2 = tf.Variable(tf.zeros([hk]))
BN_norm = tf.nn.batch_normalization(z2, batch_mean2, batch_var2, beta2,
scale2, 1e-3)
###
local4 = max_out(BN_norm, hk)
cifar10._activation_summary(local4)
weights = cifar10._variable_with_weight_decay('weights', [hk, 10],
stddev=1 / (hk * 1.0),
wd=0.0)
biases = cifar10._variable_on_cpu('biases', [10],
tf.constant_initializer(0.0))
softmax_linear = tf.add(tf.matmul(local4, weights),
biases,
name=scope.name)
cifar10._activation_summary(softmax_linear)
return softmax_linear
