def testTrainingSubsetsOfVariablesOnlyUpdatesThoseVariables(self): # First, train only the weights of the model. with ops.Graph().as_default(): random_seed.set_random_seed(0) total_loss = self.ModelLoss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1.0) weights, biases = variables_lib2.get_variables() train_op = learning.create_train_op(total_loss, optimizer) train_weights = learning.create_train_op( total_loss, optimizer, variables_to_train=[weights]) train_biases = learning.create_train_op( total_loss, optimizer, variables_to_train=[biases]) with session.Session() as sess: # Initialize the variables. sess.run(variables_lib.global_variables_initializer()) # Get the intial weights and biases values. weights_values, biases_values = sess.run([weights, biases]) self.assertGreater(np.linalg.norm(weights_values), 0) self.assertAlmostEqual(np.linalg.norm(biases_values), 0) # Update weights and biases. loss = sess.run(train_op) self.assertGreater(loss, .5) new_weights, new_biases = sess.run([weights, biases]) # Check that the weights and biases have been updated. self.assertGreater( np.linalg.norm(weights_values - new_weights), 0) self.assertGreater(np.linalg.norm(biases_values - new_biases), 0) weights_values, biases_values = new_weights, new_biases # Update only weights. loss = sess.run(train_weights) self.assertGreater(loss, .5) new_weights, new_biases = sess.run([weights, biases]) # Check that the weights have been updated, but biases have not. self.assertGreater( np.linalg.norm(weights_values - new_weights), 0) self.assertAlmostEqual( np.linalg.norm(biases_values - new_biases), 0) weights_values = new_weights # Update only biases. loss = sess.run(train_biases) self.assertGreater(loss, .5) new_weights, new_biases = sess.run([weights, biases]) # Check that the biases have been updated, but weights have not. self.assertAlmostEqual( np.linalg.norm(weights_values - new_weights), 0) self.assertGreater(np.linalg.norm(biases_values - new_biases), 0)
def testTrainAllVarsHasLowerLossThanTrainSubsetOfVars(self): logdir1 = os.path.join(tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs1') # First, train only the weights of the model. with ops.Graph().as_default(): random_seed.set_random_seed(0) total_loss = self.ModelLoss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1.0) weights = variables_lib2.get_variables_by_name('weights') train_op = learning.create_train_op(total_loss, optimizer, variables_to_train=weights) loss = learning.train(train_op, logdir1, number_of_steps=200, log_every_n_steps=10) self.assertGreater(loss, .015) self.assertLess(loss, .05) # Next, train the biases of the model. with ops.Graph().as_default(): random_seed.set_random_seed(1) total_loss = self.ModelLoss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1.0) biases = variables_lib2.get_variables_by_name('biases') train_op = learning.create_train_op(total_loss, optimizer, variables_to_train=biases) loss = learning.train(train_op, logdir1, number_of_steps=300, log_every_n_steps=10) self.assertGreater(loss, .015) self.assertLess(loss, .05) # Finally, train both weights and bias to get lower loss. with ops.Graph().as_default(): random_seed.set_random_seed(2) total_loss = self.ModelLoss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train(train_op, logdir1, number_of_steps=400, log_every_n_steps=10) self.assertIsNotNone(loss) self.assertLess(loss, .015)
def testTrainingSubsetsOfVariablesOnlyUpdatesThoseVariables(self): # First, train only the weights of the model. with ops.Graph().as_default(): random_seed.set_random_seed(0) total_loss = self.ModelLoss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) weights, biases = variables_lib2.get_variables() train_op = learning.create_train_op(total_loss, optimizer) train_weights = learning.create_train_op( total_loss, optimizer, variables_to_train=[weights]) train_biases = learning.create_train_op( total_loss, optimizer, variables_to_train=[biases]) with session.Session() as sess: # Initialize the variables. sess.run(variables_lib.global_variables_initializer()) # Get the initial weights and biases values. weights_values, biases_values = sess.run([weights, biases]) self.assertGreater(np.linalg.norm(weights_values), 0) self.assertAlmostEqual(np.linalg.norm(biases_values), 0) # Update weights and biases. loss = sess.run(train_op) self.assertGreater(loss, .5) new_weights, new_biases = sess.run([weights, biases]) # Check that the weights and biases have been updated. self.assertGreater(np.linalg.norm(weights_values - new_weights), 0) self.assertGreater(np.linalg.norm(biases_values - new_biases), 0) weights_values, biases_values = new_weights, new_biases # Update only weights. loss = sess.run(train_weights) self.assertGreater(loss, .5) new_weights, new_biases = sess.run([weights, biases]) # Check that the weights have been updated, but biases have not. self.assertGreater(np.linalg.norm(weights_values - new_weights), 0) self.assertAlmostEqual(np.linalg.norm(biases_values - new_biases), 0) weights_values = new_weights # Update only biases. loss = sess.run(train_biases) self.assertGreater(loss, .5) new_weights, new_biases = sess.run([weights, biases]) # Check that the biases have been updated, but weights have not. self.assertAlmostEqual(np.linalg.norm(weights_values - new_weights), 0) self.assertGreater(np.linalg.norm(biases_values - new_biases), 0)
def testTrainWithLocalVariable(self): logdir = os.path.join(tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) local_multiplier = variables_lib2.local_variable(1.0) tf_predictions = LogisticClassifier(tf_inputs) * local_multiplier loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train(train_op, logdir, number_of_steps=300, log_every_n_steps=10) self.assertIsNotNone(loss) self.assertLess(loss, .015)
def testTrainWithSessionConfig(self): with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) session_config = config_pb2.ConfigProto(allow_soft_placement=True) loss = learning.train(train_op, None, number_of_steps=300, log_every_n_steps=10, session_config=session_config) self.assertIsNotNone(loss) self.assertLess(loss, .015)
def testTrainWithNoInitAssignCanAchieveZeroLoss(self): logdir = os.path.join(tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') g = ops.Graph() with g.as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = BatchNormClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train(train_op, logdir, number_of_steps=300, log_every_n_steps=10) self.assertLess(loss, .1)
def testNoneGlobalStep(self): with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = BatchNormClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op( total_loss, optimizer, global_step=None) global_step = variables_lib2.get_or_create_global_step() with session.Session() as sess: # Initialize all variables sess.run(variables_lib.global_variables_initializer()) for _ in range(10): sess.run([train_op]) global_step = global_step.eval() # Since train_op don't use global_step it shouldn't change. self.assertAllClose(global_step, 0)
def testTrainWithSessionWrapper(self): """Test that slim.learning.train can take `session_wrapper` args. One of the applications of `session_wrapper` is the wrappers of TensorFlow Debugger (tfdbg), which intercept methods calls to `tf.Session` (e.g., run) to achieve debugging. `DumpingDebugWrapperSession` is used here for testing purpose. """ dump_root = tempfile.mkdtemp() def dumping_wrapper(sess): # pylint: disable=invalid-name return dumping_wrapper_lib.DumpingDebugWrapperSession(sess, dump_root) with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train( train_op, None, number_of_steps=1, session_wrapper=dumping_wrapper) self.assertIsNotNone(loss) run_root = glob.glob(os.path.join(dump_root, 'run_*'))[-1] dump = debug_data.DebugDumpDir(run_root) self.assertAllEqual(0, dump.get_tensors('global_step', 0, 'DebugIdentity')[0])
def testTrainWithEpochLimit(self): logdir = os.path.join(tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_inputs_limited = input_lib.limit_epochs(tf_inputs, num_epochs=300) tf_labels_limited = input_lib.limit_epochs(tf_labels, num_epochs=300) tf_predictions = LogisticClassifier(tf_inputs_limited) loss_ops.log_loss(tf_predictions, tf_labels_limited) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train(train_op, logdir, log_every_n_steps=10) self.assertIsNotNone(loss) self.assertLess(loss, .015) self.assertTrue( os.path.isfile('{}/model.ckpt-300.index'.format(logdir))) self.assertTrue( os.path.isfile( '{}/model.ckpt-300.data-00000-of-00001'.format(logdir)))
def testResumeTrainAchievesRoughlyTheSameLoss(self): logdir = os.path.join( tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') number_of_steps = [300, 301, 305] for i in range(len(number_of_steps)): with ops.Graph().as_default(): random_seed.set_random_seed(i) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train( train_op, logdir, number_of_steps=number_of_steps[i], log_every_n_steps=10) self.assertIsNotNone(loss) self.assertLess(loss, .015)
def testTrainWithTrace(self): logdir = os.path.join( tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() summary.scalar('total_loss', total_loss) optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train( train_op, logdir, number_of_steps=300, log_every_n_steps=10, trace_every_n_steps=100) self.assertIsNotNone(loss) for trace_step in [1, 101, 201]: trace_filename = 'tf_trace-%d.json' % trace_step self.assertTrue(os.path.isfile(os.path.join(logdir, trace_filename)))
def testEmptyUpdateOps(self): with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = BatchNormClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer, update_ops=[]) moving_mean = variables_lib2.get_variables_by_name('moving_mean')[0] moving_variance = variables_lib2.get_variables_by_name('moving_variance')[ 0] with session.Session() as sess: # Initialize all variables sess.run(variables_lib.global_variables_initializer()) mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. self.assertAllClose(mean, [0] * 4) self.assertAllClose(variance, [1] * 4) for _ in range(10): sess.run([train_op]) mean = moving_mean.eval() variance = moving_variance.eval() # Since we skip update_ops the moving_vars are not updated. self.assertAllClose(mean, [0] * 4) self.assertAllClose(variance, [1] * 4)
def testTrainAllVarsHasLowerLossThanTrainSubsetOfVars(self): logdir1 = os.path.join( tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs1') # First, train only the weights of the model. with ops.Graph().as_default(): random_seed.set_random_seed(0) total_loss = self.ModelLoss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) weights = variables_lib2.get_variables_by_name('weights') train_op = learning.create_train_op( total_loss, optimizer, variables_to_train=weights) loss = learning.train( train_op, logdir1, number_of_steps=200, log_every_n_steps=10) self.assertGreater(loss, .015) self.assertLess(loss, .05) # Next, train the biases of the model. with ops.Graph().as_default(): random_seed.set_random_seed(1) total_loss = self.ModelLoss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) biases = variables_lib2.get_variables_by_name('biases') train_op = learning.create_train_op( total_loss, optimizer, variables_to_train=biases) loss = learning.train( train_op, logdir1, number_of_steps=300, log_every_n_steps=10) self.assertGreater(loss, .015) self.assertLess(loss, .05) # Finally, train both weights and bias to get lower loss. with ops.Graph().as_default(): random_seed.set_random_seed(2) total_loss = self.ModelLoss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train( train_op, logdir1, number_of_steps=400, log_every_n_steps=10) self.assertIsNotNone(loss) self.assertLess(loss, .015)
def build_train_op(self, loss, train_step): learning_rate = lr_decay**( train_step / learning_rate_decrease_step) * learning_rate_init momentum = 1 - learning_rate optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate, momentum=momentum) train_op = learning.create_train_op(loss, optimizer=optimizer) return train_op
def optimize_nn(deep_nn_template, images, labels, device_id, name_prefix, algorithm_params, loss_func): model_input_params = ModelInputParams(x=images, y=labels, device_id=device_id, is_training=True) tensor_outputs = deep_nn_template(model_input_params, algorithm_params=algorithm_params) with tf.name_scope(name_prefix + '_loss'): cross_entropy_l = loss_func(tensor_outputs, labels) cross_entropy = tf.reduce_mean(cross_entropy_l) with tf.name_scope(name_prefix + '_optimizer'): global_step = tf.train.get_or_create_global_step() learning_rate = tf.train.exponential_decay( algorithm_params["learning_rate"], global_step, algorithm_params["learning_rate_decay_step"], algorithm_params["learning_rate_decay_factor"], staircase=True) if isinstance(algorithm_params["optimizer"], tuple) or isinstance( algorithm_params["optimizer"], list): if algorithm_params["optimizer"][0] == "MomentumOptimizer": optimizer = tf.train.MomentumOptimizer( learning_rate, momentum=algorithm_params["optimizer"][1], name="nn_core/Momentum") else: if algorithm_params["optimizer"] == "AdamOptimizer": optimizer = tf.train.AdamOptimizer(learning_rate, name="nn_core/Adam") # None means TPU if device_id is None: optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) train_step = create_train_op(cross_entropy, optimizer, global_step=global_step) # This part is required for batch normalization to work update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) if update_ops: updates = tf.group(*update_ops) cross_entropy = control_flow_ops.with_dependencies([updates], cross_entropy) return tensor_outputs.y_conv, cross_entropy, learning_rate, train_step
def testRecordTrainOpInCollection(self): with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) # Make sure the training op was recorded in the proper collection self.assertTrue(train_op in ops.get_collection(ops.GraphKeys.TRAIN_OP))
def testTrainWithNoneAsLogdirWhenUsingTraceRaisesError(self): with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) with self.assertRaises(ValueError): learning.train( train_op, None, number_of_steps=300, trace_every_n_steps=10)
def testTrainWithNoneAsInitWhenUsingVarsRaisesError(self): logdir = os.path.join( tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) with self.assertRaises(RuntimeError): learning.train(train_op, logdir, init_op=None, number_of_steps=300)
def create_train_op(self, learning_rate=1.0, gradient_multiplier=1.0): tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=learning_rate) if gradient_multiplier != 1.0: variables = variables_lib.trainable_variables() gradient_multipliers = {var: gradient_multiplier for var in variables} else: gradient_multipliers = None return learning.create_train_op( total_loss, optimizer, gradient_multipliers=gradient_multipliers)
def testTrainWithNoInitAssignCanAchieveZeroLoss(self): logdir = os.path.join( tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train( train_op, logdir, number_of_steps=300, log_every_n_steps=10) self.assertIsNotNone(loss) self.assertLess(loss, .015)
def testUseUpdateOps(self): with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) expected_mean = np.mean(self._inputs, axis=(0)) expected_var = np.var(self._inputs, axis=(0)) expected_var = self._addBesselsCorrection(16, expected_var) tf_predictions = BatchNormClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) moving_mean = variables_lib2.get_variables_by_name( 'moving_mean')[0] moving_variance = variables_lib2.get_variables_by_name( 'moving_variance')[0] with session.Session() as sess: # Initialize all variables sess.run(variables_lib.global_variables_initializer()) mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. self.assertAllClose(mean, [0] * 4) self.assertAllClose(variance, [1] * 4) for _ in range(10): sess.run([train_op]) mean = moving_mean.eval() variance = moving_variance.eval() # After 10 updates with decay 0.1 moving_mean == expected_mean and # moving_variance == expected_var. self.assertAllClose(mean, expected_mean) self.assertAllClose(variance, expected_var)
def testTrainWithEpochLimit(self): logdir = os.path.join(tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_inputs_limited = input_lib.limit_epochs(tf_inputs, num_epochs=300) tf_labels_limited = input_lib.limit_epochs(tf_labels, num_epochs=300) tf_predictions = LogisticClassifier(tf_inputs_limited) loss_ops.log_loss(tf_predictions, tf_labels_limited) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train(train_op, logdir, log_every_n_steps=10) self.assertIsNotNone(loss) self.assertLess(loss, .015) self.assertTrue(os.path.isfile('{}/model.ckpt-300.index'.format(logdir))) self.assertTrue(os.path.isfile('{}/model.ckpt-300.data-00000-of-00001'.format(logdir)))
def testTrainWithSessionConfig(self): with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) session_config = config_pb2.ConfigProto(allow_soft_placement=True) loss = learning.train( train_op, None, number_of_steps=300, log_every_n_steps=10, session_config=session_config) self.assertIsNotNone(loss) self.assertLess(loss, .015)
def testUseUpdateOps(self): with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) expected_mean = np.mean(self._inputs, axis=(0)) expected_var = np.var(self._inputs, axis=(0)) expected_var = self._addBesselsCorrection(16, expected_var) tf_predictions = BatchNormClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) moving_mean = variables_lib2.get_variables_by_name('moving_mean')[0] moving_variance = variables_lib2.get_variables_by_name('moving_variance')[ 0] with session.Session() as sess: # Initialize all variables sess.run(variables_lib.global_variables_initializer()) mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. self.assertAllClose(mean, [0] * 4) self.assertAllClose(variance, [1] * 4) for _ in range(10): sess.run([train_op]) mean = moving_mean.eval() variance = moving_variance.eval() # After 10 updates with decay 0.1 moving_mean == expected_mean and # moving_variance == expected_var. self.assertAllClose(mean, expected_mean) self.assertAllClose(variance, expected_var)