def test_callback(optimized_metric, initial_best, train_metrics, eval_metrics,
improved_seq):
with tempfile.TemporaryDirectory() as tmpdir:
monitor = callback.TrainingMonitor(output_folder=tmpdir,
optimized_metric=optimized_metric)
assert monitor.optimized_metric == optimized_metric
assert monitor.get_best_validation_score() == initial_best
metrics_fname = os.path.join(tmpdir, C.METRICS_NAME)
for checkpoint, (train_metric, eval_metric,
expected_improved) in enumerate(
zip(train_metrics, eval_metrics, improved_seq),
1):
monitor.checkpoint_callback(checkpoint, train_metric)
assert len(monitor.metrics) == checkpoint
assert monitor.metrics[-1] == {
k + "-train": v
for k, v in train_metric.items()
}
improved, best_checkpoint = monitor.eval_end_callback(
checkpoint, DummyMetric(eval_metric))
assert {k + "-val"
for k in eval_metric.keys()} <= monitor.metrics[-1].keys()
assert improved == expected_improved
assert os.path.exists(metrics_fname)
metrics = utils.read_metrics_file(metrics_fname)
_compare_metrics(metrics, monitor.metrics)
def test_write_read_metric_file():
expected_metrics = [{'float_metric':3.45, 'bool_metric': True},
{'float_metric':1.0, 'bool_metric': False}]
with TemporaryDirectory(prefix="metric_file") as work_dir:
metric_path = os.path.join(work_dir, "metrics")
utils.write_metrics_file(expected_metrics, metric_path)
read_metrics = utils.read_metrics_file(metric_path)
assert len(read_metrics) == len(expected_metrics)
assert expected_metrics == read_metrics