def test_untrainable_batch_norm_vars_not_trainable(self): input_layer = tf.placeholder(tf.float32, shape=(None, 1)) scope_name = 'test_bn_scope' with tf.variable_scope(scope_name): tf_util.batch_norm(input_layer, True) with tf.variable_scope(scope_name, reuse=True) as bn_scope: tf_util.batch_norm(input_layer, False) self.sess.run(tf.global_variables_initializer()) trainable_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) bn_params = tf_util.get_untrainable_batch_norm_vars(bn_scope) self.assertEqual(0, len(set(trainable_params).intersection(bn_params)))
def test_batch_norm_pop_stats_decays_correctly_2_itrs(self): in_size = 1 input_layer = tf.placeholder(tf.float32, shape=(None, in_size)) bn_config = tf_util.BatchNormConfig(decay=0.5) scope_name = 'test_bn_scope' with tf.variable_scope(scope_name): output_train, batch_ops = tf_util.batch_norm( input_layer, True, batch_norm_config=bn_config) input_values = np.array([[1, 2, 3, 4]]).T # variance([1, 2, 3, 4]) = 1.25 # mean([1, 2, 3, 4]) = 2.5 input_values2 = np.array([[2, 4]]).T # variance([2, 4]) = 1 # mean([2, 4]) = 3 self.sess.run(tf.global_variables_initializer()) mean = None variance = None for values in [input_values, input_values2]: training_values, mean, variance = self.sess.run([ output_train, batch_ops.update_pop_mean_op, batch_ops.update_pop_var_op ], {input_layer: values}) self.assertAlmostEqual(mean[0], ((2.5 / 2) + 3) / 2) self.assertAlmostEqual(variance[0], (((1.25 + 1) / 2) + 1) / 2)
def _process_layer(self, previous_layer, scope_name="process_layer"): """ This should be done called between every layer, i.e. a = self.process_layer(linear(x)) b = self.process_layer(linear(relu(a))) output = linear(relu(b)) This method should NOT be called on the output. If batch norm is disabled, this just returns `previous_layer` immediately. If batch norm is enabled, this returns a layer with bn enabled, either for training or eval based on self._is_bn_in_training_mode :param previous_layer: Either the input layer or the output to a previous call to `_process_layer` :param scope_name: Scope name for any variables that may be created while processing the layer :return: If batch norm is disabled, this returns previous_layer. Otherwise, it returns a tuple (batch norm'd layer for training, batch norm'd layer for eval) """ if not self._batch_norm: return previous_layer with tf.variable_scope(scope_name): processed_layer, _ = tf_util.batch_norm( previous_layer, is_training=self._is_bn_in_training_mode, batch_norm_config=self._batch_norm_config, ) return processed_layer
def test_batch_norm_eval_uses_pop_stats_correctly(self): in_size = 1 input_layer = tf.placeholder(tf.float32, shape=(None, in_size)) epsilon = 1e-5 bn_config = tf_util.BatchNormConfig(decay=0, epsilon=epsilon) scope_name = 'test_bn_scope' with tf.variable_scope(scope_name) as scope: output_train, batch_ops = tf_util.batch_norm( input_layer, True, batch_norm_config=bn_config) scope.reuse_variables() output_eval = tf_util.batch_norm(input_layer, False, batch_norm_config=bn_config)[0] input_values = np.array([[1, 2, 3, 4]]).T # variance([1, 2, 3, 4]) = 1.25 # mean([1, 2, 3, 4]) = 2.5 self.sess.run(tf.global_variables_initializer()) training_values, mean, variance, gamma, beta = self.sess.run( [ output_train, batch_ops.update_pop_mean_op, batch_ops.update_pop_var_op, batch_ops.scale, batch_ops.offset, ], {input_layer: input_values}) self.assertEqual(mean[0], 2.5) self.assertEqual(variance[0], 1.25) self.assertEqual(gamma[0], 1) self.assertEqual(beta[0], 0) expected_eval_values = np.array([[-2, 2]]).T expected_pop_mean = 2.5 expected_pop_var = 1.25 eval_input_values = ( expected_eval_values * np.sqrt(expected_pop_var + epsilon) + expected_pop_mean) eval_values = self.sess.run(output_eval, {input_layer: eval_input_values}) self.assertNpArraysAlmostEqual(expected_eval_values, eval_values)
def test_batch_norm_untrained_is_a_noop(self): in_size = 1 input_layer = tf.placeholder(tf.float32, shape=(None, in_size)) bn_config = tf_util.BatchNormConfig(epsilon=0., decay=0.5) scope_name = 'test_bn_scope' with tf.variable_scope(scope_name) as scope: output_train = tf_util.batch_norm(input_layer, True, batch_norm_config=bn_config)[0] scope.reuse_variables() output_eval = tf_util.batch_norm(input_layer, False, batch_norm_config=bn_config)[0] input_values = np.array([[-2], [2]]) self.sess.run(tf.global_variables_initializer()) eval_values, _ = self.sess.run([output_eval, output_train], {input_layer: input_values}) expected_eval_values = np.array([[-2], [2]]) self.assertNpArraysEqual(eval_values, expected_eval_values)
def test_get_untrainable_batch_norm_vars(self): input_layer = tf.placeholder(tf.float32, shape=(None, 1)) scope_name = 'test_bn_scope' with tf.variable_scope(scope_name) as bn_scope: output_train, bn_ops = tf_util.batch_norm( input_layer, True, ) update_ops = [bn_ops.pop_mean, bn_ops.pop_var] update_ops_v2 = tf_util.get_untrainable_batch_norm_vars(bn_scope) self.assertEqual(set(update_ops), set(update_ops_v2))
def test_batch_norm_whitens_training_data(self): in_size = 1 input_layer = tf.placeholder(tf.float32, shape=(None, in_size)) bn_config = tf_util.BatchNormConfig(epsilon=0.) scope_name = 'test_bn_scope' with tf.variable_scope(scope_name): output_train = tf_util.batch_norm(input_layer, True, batch_norm_config=bn_config)[0] input_values = np.array([[-2], [2]]) self.sess.run(tf.global_variables_initializer()) training_values = self.sess.run(output_train, {input_layer: input_values}) expected_training_values = np.array([[-1], [1]]) self.assertNpArraysEqual(training_values, expected_training_values)
def test_scope_does_not_affect_get_batch_norm_update_pop_stats_ops(self): scope_name = 'test_bn_scope' with tf.variable_scope('test'): with tf.variable_scope('foo'): input_layer = tf.placeholder(tf.float32, shape=(None, 1)) with tf.variable_scope(scope_name) as bn_scope: output_train, bn_ops = tf_util.batch_norm( input_layer, True, ) with tf.variable_scope('bar'): update_ops = bn_ops.update_pop_stats_ops with tf.variable_scope('zoo'): update_ops_v2 = tf_util.get_batch_norm_update_pop_stats_ops( bn_scope) self.assertEqual(set(update_ops), set(update_ops_v2))
def test_batch_norm_pop_stats_are_computed_correctly(self): in_size = 1 input_layer = tf.placeholder(tf.float32, shape=(None, in_size)) bn_config = tf_util.BatchNormConfig(decay=0.) scope_name = 'test_bn_scope' with tf.variable_scope(scope_name): output_train, batch_ops = tf_util.batch_norm( input_layer, True, batch_norm_config=bn_config) input_values = np.array([[1, 2, 3, 4]]).T # variance([1, 2, 3, 4]) = 1.25 # mean([1, 2, 3, 4]) = 2.5 self.sess.run(tf.global_variables_initializer()) training_values, mean, variance = self.sess.run( [ output_train, batch_ops.update_pop_mean_op, batch_ops.update_pop_var_op ], {input_layer: input_values}) self.assertAlmostEqual(mean[0], 2.5) self.assertAlmostEqual(variance[0], 1.25)