def test_update():
tracker = metric.MetricsTracker()
tracker.update('new_metric', 0.5) # automatic registration
assert set(tracker.metrics.keys()) == {'new_metric'}
assert tracker.metrics['new_metric'].direction == 'min' # default direction
assert (tracker.get_history('new_metric') == [
metric.MetricObservation(0.5, step=0)
])
def test_register():
tracker = metric.MetricsTracker()
tracker.register('new_metric', direction='max')
assert set(tracker.metrics.keys()) == {'new_metric'}
assert tracker.metrics['new_metric'].direction == 'max'
with pytest.raises(ValueError, match='`direction` should be one of'):
tracker.register('another_metric', direction='wrong')
with pytest.raises(ValueError, match='already exists'):
tracker.register('new_metric', direction='max')
def test_get_last_value():
tracker = metric.MetricsTracker()
tracker.register('new_metric', 'min')
assert tracker.get_last_value('new_metric') is None
tracker.set_history('new_metric', [
metric.MetricObservation(1., 0),
metric.MetricObservation(2., 1),
metric.MetricObservation(3., 2)
])
assert tracker.get_last_value('new_metric') == 3.
def test_register_from_metrics():
# As well as direction inference.
tracker = metric.MetricsTracker(
metrics=[metrics.CategoricalAccuracy(),
metrics.MeanSquaredError()])
assert set(tracker.metrics.keys()) == {
'categorical_accuracy', 'mean_squared_error'
}
assert tracker.metrics['categorical_accuracy'].direction == 'max'
assert tracker.metrics['mean_squared_error'].direction == 'min'
def __init__(self,
hyperparameters,
trial_id=None,
status=TrialStatus.RUNNING):
self.hyperparameters = hyperparameters
self.trial_id = generate_trial_id() if trial_id is None else trial_id
self.metrics = metric.MetricsTracker()
self.score = None
self.best_step = None
self.status = status
def test_get_history():
tracker = metric.MetricsTracker()
tracker.update('new_metric', 0.5, step=0)
tracker.update('new_metric', 1.5, step=1)
tracker.update('new_metric', 2., step=2)
assert tracker.get_history('new_metric') == [
metric.MetricObservation(0.5, 0),
metric.MetricObservation(1.5, 1),
metric.MetricObservation(2., 2),
]
with pytest.raises(ValueError, match='Unknown metric'):
tracker.get_history('another_metric')
def test_serialization():
tracker = metric.MetricsTracker()
tracker.register('metric_min', 'min')
tracker.register('metric_max', 'max')
tracker.set_history('metric_min', [
metric.MetricObservation(1., 0),
metric.MetricObservation(2., 1),
metric.MetricObservation(3., 2)
])
tracker.set_history('metric_max', [
metric.MetricObservation(1., 0),
metric.MetricObservation(2., 1),
metric.MetricObservation(3., 2)
])
new_tracker = metric.MetricsTracker.from_config(tracker.get_config())
assert new_tracker.metrics.keys() == tracker.metrics.keys()
def test_exists():
tracker = metric.MetricsTracker()
tracker.register('new_metric', direction='max')
assert tracker.exists('new_metric')
assert not tracker.exists('another_metric')