def testTrainingStats(self): model = _make_seq2seq_model(self.get_temp_dir()) optimizer = tf.keras.optimizers.SGD(1.0) stats = training.TrainingStats(model, optimizer, warmup_steps=2) def _generate_example(length): return tf.constant(" ".join(map(str, range(length)))) def _step(source_length, target_length, step, loss): source_example = _generate_example(source_length) target_example = _generate_example(target_length) source_features = model.features_inputter.make_features( source_example) target_features = model.labels_inputter.make_features( target_example) stats.update_on_example(source_features, target_features) stats.update_on_step(tf.constant(step, dtype=tf.int64), tf.constant(loss)) def _is_json_serializable(summary): json.dumps(summary) return True _step(24, 23, 5, 9.8) _step(10, 8, 10, 9.6) summary = stats.get_last_summary() self.assertTrue(_is_json_serializable(summary)) self.assertEqual(summary["learning_rate"], 1.0) self.assertEqual(summary["step"], 10) self.assertAllClose(summary["loss"], 9.6) # Throughput values are ignored in the 2 first steps. self.assertEqual(summary["steps_per_sec"], 0) self.assertEqual(summary["words_per_sec"]["source"], 0) self.assertEqual(summary["words_per_sec"]["target"], 0) _step(14, 21, 15, 9.4) summary = stats.get_last_summary() self.assertNotEqual(summary["steps_per_sec"], 0) self.assertNotEqual(summary["words_per_sec"]["source"], 0) self.assertNotEqual(summary["words_per_sec"]["target"], 0) stats.log() stats.reset_throughput() summary = stats.get_last_summary() self.assertEqual(summary["steps_per_sec"], 0) self.assertEqual(summary["words_per_sec"]["source"], 0) self.assertEqual(summary["words_per_sec"]["target"], 0) summary = stats.get_global_summary() self.assertTrue(_is_json_serializable(summary)) self.assertEqual(summary["last_learning_rate"], 1.0) self.assertEqual(summary["last_step"], 15) self.assertAllClose(summary["last_loss"], 9.4) self.assertAllClose(summary["average_loss"], 9.6) self.assertEqual(summary["num_steps"], 3)
def testTrainingStats(self): model = _make_seq2seq_model(self.get_temp_dir()) optimizer = tf.keras.optimizers.SGD(1.0) stats = training.TrainingStats(model, optimizer, warmup_steps=2) def _step(source_length, target_length, step, loss): source_features = {"length": source_length} target_features = {"length": target_length} stats.update_on_example(source_features, target_features) stats.update_on_step(step, loss) _step(24, 23, 5, 9.8) _step(10, 8, 10, 9.6) summary = stats.get_last_summary() self.assertEqual(summary["learning_rate"], 1.0) self.assertEqual(summary["step"], 10) self.assertEqual(summary["loss"], 9.6) # Throughput values are ignored in the 2 first steps. self.assertEqual(summary["steps_per_sec"], 0) self.assertEqual(summary["words_per_sec"]["source"], 0) self.assertEqual(summary["words_per_sec"]["target"], 0) _step(14, 21, 15, 9.4) summary = stats.get_last_summary() self.assertNotEqual(summary["steps_per_sec"], 0) self.assertNotEqual(summary["words_per_sec"]["source"], 0) self.assertNotEqual(summary["words_per_sec"]["target"], 0) stats.log() stats.reset_throughput() summary = stats.get_last_summary() self.assertEqual(summary["steps_per_sec"], 0) self.assertEqual(summary["words_per_sec"]["source"], 0) self.assertEqual(summary["words_per_sec"]["target"], 0) summary = stats.get_global_summary() self.assertEqual(summary["last_learning_rate"], 1.0) self.assertEqual(summary["last_step"], 15) self.assertEqual(summary["last_loss"], 9.4) self.assertEqual(summary["average_loss"], 9.6) self.assertEqual(summary["num_steps"], 3)