def test_batch_norm_pop_stats_decays_correctly_2_itrs(self):
in_size = 1
input_layer = tf.placeholder(tf.float32, shape=(None, in_size))
bn_config = tf_util.BatchNormConfig(decay=0.5)
scope_name = 'test_bn_scope'
with tf.variable_scope(scope_name):
output_train, batch_ops = tf_util.batch_norm(
input_layer, True, batch_norm_config=bn_config)
input_values = np.array([[1, 2, 3, 4]]).T
# variance([1, 2, 3, 4]) = 1.25
# mean([1, 2, 3, 4]) = 2.5
input_values2 = np.array([[2, 4]]).T
# variance([2, 4]) = 1
# mean([2, 4]) = 3
self.sess.run(tf.global_variables_initializer())
mean = None
variance = None
for values in [input_values, input_values2]:
training_values, mean, variance = self.sess.run([
output_train, batch_ops.update_pop_mean_op,
batch_ops.update_pop_var_op
], {input_layer: values})
self.assertAlmostEqual(mean[0], ((2.5 / 2) + 3) / 2)
self.assertAlmostEqual(variance[0], (((1.25 + 1) / 2) + 1) / 2)
def test_batch_norm_whitens_training_data(self):
in_size = 1
input_layer = tf.placeholder(tf.float32, shape=(None, in_size))
bn_config = tf_util.BatchNormConfig(epsilon=0.)
scope_name = 'test_bn_scope'
with tf.variable_scope(scope_name):
output_train = tf_util.batch_norm(input_layer,
True,
batch_norm_config=bn_config)[0]
input_values = np.array([[-2], [2]])
self.sess.run(tf.global_variables_initializer())
training_values = self.sess.run(output_train,
{input_layer: input_values})
expected_training_values = np.array([[-1], [1]])
self.assertNpArraysEqual(training_values, expected_training_values)
def test_batch_norm_eval_uses_pop_stats_correctly(self):
in_size = 1
input_layer = tf.placeholder(tf.float32, shape=(None, in_size))
epsilon = 1e-5
bn_config = tf_util.BatchNormConfig(decay=0, epsilon=epsilon)
scope_name = 'test_bn_scope'
with tf.variable_scope(scope_name) as scope:
output_train, batch_ops = tf_util.batch_norm(
input_layer, True, batch_norm_config=bn_config)
scope.reuse_variables()
output_eval = tf_util.batch_norm(input_layer,
False,
batch_norm_config=bn_config)[0]
input_values = np.array([[1, 2, 3, 4]]).T
# variance([1, 2, 3, 4]) = 1.25
# mean([1, 2, 3, 4]) = 2.5
self.sess.run(tf.global_variables_initializer())
training_values, mean, variance, gamma, beta = self.sess.run(
[
output_train,
batch_ops.update_pop_mean_op,
batch_ops.update_pop_var_op,
batch_ops.scale,
batch_ops.offset,
], {input_layer: input_values})
self.assertEqual(mean[0], 2.5)
self.assertEqual(variance[0], 1.25)
self.assertEqual(gamma[0], 1)
self.assertEqual(beta[0], 0)
expected_eval_values = np.array([[-2, 2]]).T
expected_pop_mean = 2.5
expected_pop_var = 1.25
eval_input_values = (
expected_eval_values * np.sqrt(expected_pop_var + epsilon) +
expected_pop_mean)
eval_values = self.sess.run(output_eval,
{input_layer: eval_input_values})
self.assertNpArraysAlmostEqual(expected_eval_values, eval_values)
def test_batch_norm_untrained_is_a_noop(self):
in_size = 1
input_layer = tf.placeholder(tf.float32, shape=(None, in_size))
bn_config = tf_util.BatchNormConfig(epsilon=0., decay=0.5)
scope_name = 'test_bn_scope'
with tf.variable_scope(scope_name) as scope:
output_train = tf_util.batch_norm(input_layer,
True,
batch_norm_config=bn_config)[0]
scope.reuse_variables()
output_eval = tf_util.batch_norm(input_layer,
False,
batch_norm_config=bn_config)[0]
input_values = np.array([[-2], [2]])
self.sess.run(tf.global_variables_initializer())
eval_values, _ = self.sess.run([output_eval, output_train],
{input_layer: input_values})
expected_eval_values = np.array([[-2], [2]])
self.assertNpArraysEqual(eval_values, expected_eval_values)
def test_batch_norm_pop_stats_are_computed_correctly(self):
in_size = 1
input_layer = tf.placeholder(tf.float32, shape=(None, in_size))
bn_config = tf_util.BatchNormConfig(decay=0.)
scope_name = 'test_bn_scope'
with tf.variable_scope(scope_name):
output_train, batch_ops = tf_util.batch_norm(
input_layer, True, batch_norm_config=bn_config)
input_values = np.array([[1, 2, 3, 4]]).T
# variance([1, 2, 3, 4]) = 1.25
# mean([1, 2, 3, 4]) = 2.5
self.sess.run(tf.global_variables_initializer())
training_values, mean, variance = self.sess.run(
[
output_train, batch_ops.update_pop_mean_op,
batch_ops.update_pop_var_op
], {input_layer: input_values})
self.assertAlmostEqual(mean[0], 2.5)
self.assertAlmostEqual(variance[0], 1.25)