def testEmptyUpdateOps(self):
with tf.Graph().as_default():
tf_inputs = tf.constant(self._inputs, dtype=tf.dtypes.float32)
tf_labels = tf.constant(self._labels, dtype=tf.dtypes.float32)
tf_predictions = self.batchnorm_classifier(tf_inputs)
loss = tf.losses.log_loss(tf_labels, tf_predictions)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0)
train_op = utils.create_train_op(loss, optimizer, update_ops=[])
moving_mean = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
'.*moving_mean:')[0]
moving_variance = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
'.*moving_variance:')[0]
with self.cached_session() as session:
# Initialize all variables
session.run(tf.global_variables_initializer())
mean, variance = session.run([moving_mean, moving_variance])
# After initialization moving_mean == 0 and moving_variance == 1.
self.assertAllClose(mean, [0] * 4)
self.assertAllClose(variance, [1] * 4)
for _ in range(10):
session.run(train_op)
mean = moving_mean.eval()
variance = moving_variance.eval()
# Since we skip update_ops the moving_vars are not updated.
self.assertAllClose(mean, [0] * 4)
self.assertAllClose(variance, [1] * 4)
def stage_2_train_op():
return utils.create_train_op(
stage_2_loss,
stage_2_optimizer,
update_ops=(None if
self.hparams.stage_2.training.update_encoder_batch_norm
else []))
def testTrainOpInCollection(self):
with tf.Graph().as_default():
tf_inputs = tf.constant(self._inputs, dtype=tf.dtypes.float32)
tf_labels = tf.constant(self._labels, dtype=tf.dtypes.float32)
tf_predictions = self.batchnorm_classifier(tf_inputs)
loss = tf.losses.log_loss(tf_labels, tf_predictions)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0)
train_op = utils.create_train_op(loss, optimizer)
# Make sure the training op was recorded in the proper collection
self.assertIn(train_op, tf.get_collection(tf.GraphKeys.TRAIN_OP))
def testGlobalStepNotIncrementedWhenSetToNone(self):
with tf.Graph().as_default():
tf_inputs = tf.constant(self._inputs, dtype=tf.dtypes.float32)
tf_labels = tf.constant(self._labels, dtype=tf.dtypes.float32)
tf_predictions = self.batchnorm_classifier(tf_inputs)
loss = tf.losses.log_loss(tf_labels, tf_predictions)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0)
train_op = utils.create_train_op(loss, optimizer, global_step=None)
global_step = tf.train.get_or_create_global_step()
with self.cached_session() as session:
# Initialize all variables
session.run(tf.global_variables_initializer())
for _ in range(10):
session.run(train_op)
# Since train_op don't use global_step it shouldn't change.
self.assertAllClose(global_step.eval(), 0)
def testUseUpdateOps(self):
with tf.Graph().as_default():
tf_inputs = tf.constant(self._inputs, dtype=tf.dtypes.float32)
tf_labels = tf.constant(self._labels, dtype=tf.dtypes.float32)
expected_mean = np.mean(self._inputs, axis=(0))
expected_var = np.var(self._inputs, axis=(0))
tf_predictions = self.batchnorm_classifier(tf_inputs)
loss = tf.losses.log_loss(tf_labels, tf_predictions)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0)
train_op = utils.create_train_op(loss, optimizer)
moving_mean = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
'.*moving_mean:')[0]
moving_variance = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
'.*moving_variance:')[0]
with self.cached_session() as session:
# Initialize all variables
session.run(tf.global_variables_initializer())
mean, variance = session.run([moving_mean, moving_variance])
# After initialization moving_mean == 0 and moving_variance == 1.
self.assertAllClose(mean, [0] * 4)
self.assertAllClose(variance, [1] * 4)
for _ in range(200):
session.run(train_op)
mean = moving_mean.eval()
variance = moving_variance.eval()
# After 10 updates with decay 0.1 moving_mean == expected_mean and
# moving_variance == expected_var.
self.assertAllClose(mean, expected_mean)
self.assertAllClose(variance, expected_var)
