def testTrainingStats(self):
model = _make_seq2seq_model(self.get_temp_dir())
optimizer = tf.keras.optimizers.SGD(1.0)
stats = training.TrainingStats(model, optimizer, warmup_steps=2)
def _generate_example(length):
return tf.constant(" ".join(map(str, range(length))))
def _step(source_length, target_length, step, loss):
source_example = _generate_example(source_length)
target_example = _generate_example(target_length)
source_features = model.features_inputter.make_features(
source_example)
target_features = model.labels_inputter.make_features(
target_example)
stats.update_on_example(source_features, target_features)
stats.update_on_step(tf.constant(step, dtype=tf.int64),
tf.constant(loss))
def _is_json_serializable(summary):
json.dumps(summary)
return True
_step(24, 23, 5, 9.8)
_step(10, 8, 10, 9.6)
summary = stats.get_last_summary()
self.assertTrue(_is_json_serializable(summary))
self.assertEqual(summary["learning_rate"], 1.0)
self.assertEqual(summary["step"], 10)
self.assertAllClose(summary["loss"], 9.6)
# Throughput values are ignored in the 2 first steps.
self.assertEqual(summary["steps_per_sec"], 0)
self.assertEqual(summary["words_per_sec"]["source"], 0)
self.assertEqual(summary["words_per_sec"]["target"], 0)
_step(14, 21, 15, 9.4)
summary = stats.get_last_summary()
self.assertNotEqual(summary["steps_per_sec"], 0)
self.assertNotEqual(summary["words_per_sec"]["source"], 0)
self.assertNotEqual(summary["words_per_sec"]["target"], 0)
stats.log()
stats.reset_throughput()
summary = stats.get_last_summary()
self.assertEqual(summary["steps_per_sec"], 0)
self.assertEqual(summary["words_per_sec"]["source"], 0)
self.assertEqual(summary["words_per_sec"]["target"], 0)
summary = stats.get_global_summary()
self.assertTrue(_is_json_serializable(summary))
self.assertEqual(summary["last_learning_rate"], 1.0)
self.assertEqual(summary["last_step"], 15)
self.assertAllClose(summary["last_loss"], 9.4)
self.assertAllClose(summary["average_loss"], 9.6)
self.assertEqual(summary["num_steps"], 3)
def testTrainingStats(self):
model = _make_seq2seq_model(self.get_temp_dir())
optimizer = tf.keras.optimizers.SGD(1.0)
stats = training.TrainingStats(model, optimizer, warmup_steps=2)
def _step(source_length, target_length, step, loss):
source_features = {"length": source_length}
target_features = {"length": target_length}
stats.update_on_example(source_features, target_features)
stats.update_on_step(step, loss)
_step(24, 23, 5, 9.8)
_step(10, 8, 10, 9.6)
summary = stats.get_last_summary()
self.assertEqual(summary["learning_rate"], 1.0)
self.assertEqual(summary["step"], 10)
self.assertEqual(summary["loss"], 9.6)
# Throughput values are ignored in the 2 first steps.
self.assertEqual(summary["steps_per_sec"], 0)
self.assertEqual(summary["words_per_sec"]["source"], 0)
self.assertEqual(summary["words_per_sec"]["target"], 0)
_step(14, 21, 15, 9.4)
summary = stats.get_last_summary()
self.assertNotEqual(summary["steps_per_sec"], 0)
self.assertNotEqual(summary["words_per_sec"]["source"], 0)
self.assertNotEqual(summary["words_per_sec"]["target"], 0)
stats.log()
stats.reset_throughput()
summary = stats.get_last_summary()
self.assertEqual(summary["steps_per_sec"], 0)
self.assertEqual(summary["words_per_sec"]["source"], 0)
self.assertEqual(summary["words_per_sec"]["target"], 0)
summary = stats.get_global_summary()
self.assertEqual(summary["last_learning_rate"], 1.0)
self.assertEqual(summary["last_step"], 15)
self.assertEqual(summary["last_loss"], 9.4)
self.assertEqual(summary["average_loss"], 9.6)
self.assertEqual(summary["num_steps"], 3)