def test_update():
tracker = metrics_tracking.MetricsTracker()
tracker.update("new_metric", 0.5) # automatic registration
assert set(tracker.metrics.keys()) == {"new_metric"}
assert tracker.metrics["new_metric"].direction == "min" # default direction
assert tracker.get_history("new_metric") == [
metrics_tracking.MetricObservation(0.5, step=0)
]
def test_register():
tracker = metrics_tracking.MetricsTracker()
tracker.register("new_metric", direction="max")
assert set(tracker.metrics.keys()) == {"new_metric"}
assert tracker.metrics["new_metric"].direction == "max"
with pytest.raises(ValueError, match="`direction` should be one of"):
tracker.register("another_metric", direction="wrong")
with pytest.raises(ValueError, match="already exists"):
tracker.register("new_metric", direction="max")
def __init__(self,
hyperparameters,
trial_id=None,
status=TrialStatus.RUNNING):
self.hyperparameters = hyperparameters
self.trial_id = generate_trial_id() if trial_id is None else trial_id
self.metrics = metrics_tracking.MetricsTracker()
self.score = None
self.best_step = None
self.status = status
def test_register_from_metrics():
# As well as direction inference.
tracker = metrics_tracking.MetricsTracker(
metrics=[metrics.CategoricalAccuracy(),
metrics.MeanSquaredError()])
assert set(tracker.metrics.keys()) == {
"categorical_accuracy",
"mean_squared_error",
}
assert tracker.metrics["categorical_accuracy"].direction == "max"
assert tracker.metrics["mean_squared_error"].direction == "min"
def test_get_history():
tracker = metrics_tracking.MetricsTracker()
tracker.update("new_metric", 0.5, step=0)
tracker.update("new_metric", 1.5, step=1)
tracker.update("new_metric", 2.0, step=2)
assert tracker.get_history("new_metric") == [
metrics_tracking.MetricObservation(0.5, 0),
metrics_tracking.MetricObservation(1.5, 1),
metrics_tracking.MetricObservation(2.0, 2),
]
with pytest.raises(ValueError, match="Unknown metric"):
tracker.get_history("another_metric")
def test_get_last_value():
tracker = metrics_tracking.MetricsTracker()
tracker.register("new_metric", "min")
assert tracker.get_last_value("new_metric") is None
tracker.set_history(
"new_metric",
[
metrics_tracking.MetricObservation(1.0, 0),
metrics_tracking.MetricObservation(2.0, 1),
metrics_tracking.MetricObservation(3.0, 2),
],
)
assert tracker.get_last_value("new_metric") == 3.0
def test_set_history():
tracker = metrics_tracking.MetricsTracker()
tracker.set_history(
"new_metric",
[
metrics_tracking.MetricObservation(0.5, 0),
metrics_tracking.MetricObservation(1.5, 1),
metrics_tracking.MetricObservation(2.0, 2),
],
)
values = [obs.value for obs in tracker.get_history("new_metric")]
steps = [obs.step for obs in tracker.get_history("new_metric")]
assert values == [[0.5], [1.5], [2.0]]
assert steps == [0, 1, 2]
def test_get_statistics():
tracker = metrics_tracking.MetricsTracker()
history = [
metrics_tracking.MetricObservation(random.random(), i)
for i in range(10)
]
tracker.set_history("new_metric", history)
stats = tracker.get_statistics("new_metric")
assert set(stats.keys()) == {"min", "max", "mean", "median", "var", "std"}
history = [obs.value for obs in history]
assert stats["min"] == np.min(history)
assert stats["max"] == np.max(history)
assert stats["mean"] == np.mean(history)
assert stats["median"] == np.median(history)
assert stats["var"] == np.var(history)
assert stats["std"] == np.std(history)
def test_metricstracker_proto():
tracker = metrics_tracking.MetricsTracker()
tracker.register("score", direction="max")
tracker.update("score", value=10, step=1)
tracker.update("score", value=20, step=1)
tracker.update("score", value=30, step=2)
proto = tracker.to_proto()
obs = proto.metrics["score"].observations
assert obs[0].value == [10, 20]
assert obs[0].step == 1
assert obs[1].value == [30]
assert obs[1].step == 2
assert proto.metrics["score"].maximize
new_tracker = metrics_tracking.MetricsTracker.from_proto(proto)
assert new_tracker.metrics["score"].direction == "max"
assert new_tracker.metrics["score"].get_history() == [
metrics_tracking.MetricObservation([10, 20], 1),
metrics_tracking.MetricObservation(30, 2),
]
def test_serialization():
tracker = metrics_tracking.MetricsTracker()
tracker.register("metric_min", "min")
tracker.register("metric_max", "max")
tracker.set_history(
"metric_min",
[
metrics_tracking.MetricObservation(1.0, 0),
metrics_tracking.MetricObservation(2.0, 1),
metrics_tracking.MetricObservation(3.0, 2),
],
)
tracker.set_history(
"metric_max",
[
metrics_tracking.MetricObservation(1.0, 0),
metrics_tracking.MetricObservation(2.0, 1),
metrics_tracking.MetricObservation(3.0, 2),
],
)
new_tracker = metrics_tracking.MetricsTracker.from_config(
tracker.get_config())
assert new_tracker.metrics.keys() == tracker.metrics.keys()
def test_exists():
tracker = metrics_tracking.MetricsTracker()
tracker.register("new_metric", direction="max")
assert tracker.exists("new_metric")
assert not tracker.exists("another_metric")
