def test_update(): tracker = metrics_tracking.MetricsTracker() tracker.update("new_metric", 0.5) # automatic registration assert set(tracker.metrics.keys()) == {"new_metric"} assert tracker.metrics["new_metric"].direction == "min" # default direction assert tracker.get_history("new_metric") == [ metrics_tracking.MetricObservation(0.5, step=0) ]
def test_get_history(): tracker = metrics_tracking.MetricsTracker() tracker.update('new_metric', 0.5) tracker.update('new_metric', 1.5) tracker.update('new_metric', 2.) assert tracker.get_history('new_metric') == [0.5, 1.5, 2.] with pytest.raises(ValueError, match='Unknown metric'): tracker.get_history('another_metric')
def test_update(): tracker = metrics_tracking.MetricsTracker() tracker.update('new_metric', 0.5) # automatic registration assert set(tracker.metrics.keys()) == {'new_metric'} assert tracker.metrics['new_metric'].direction == 'min' # default direction assert (tracker.get_history('new_metric') == [ metrics_tracking.MetricObservation(0.5, step=0) ])
def test_register_from_metrics(): # As well as direction inference. tracker = metrics_tracking.MetricsTracker( metrics=[metrics.CategoricalAccuracy(), metrics.MeanSquaredError()]) assert tracker.names == ['categorical_accuracy', 'mean_squared_error'] assert tracker.directions['categorical_accuracy'] == 'max' assert tracker.directions['mean_squared_error'] == 'min'
def test_register(): tracker = metrics_tracking.MetricsTracker() tracker.register("new_metric", direction="max") assert set(tracker.metrics.keys()) == {"new_metric"} assert tracker.metrics["new_metric"].direction == "max" with pytest.raises(ValueError, match="`direction` should be one of"): tracker.register("another_metric", direction="wrong") with pytest.raises(ValueError, match="already exists"): tracker.register("new_metric", direction="max")
def test_register(): tracker = metrics_tracking.MetricsTracker() tracker.register('new_metric', direction='max') assert set(tracker.metrics.keys()) == {'new_metric'} assert tracker.metrics['new_metric'].direction == 'max' with pytest.raises(ValueError, match='`direction` should be one of'): tracker.register('another_metric', direction='wrong') with pytest.raises(ValueError, match='already exists'): tracker.register('new_metric', direction='max')
def test_register(): tracker = metrics_tracking.MetricsTracker() tracker.register('new_metric', direction='max') assert tracker.names == ['new_metric'] assert tracker.directions['new_metric'] == 'max' with pytest.raises(ValueError, match='`direction` should be one of'): tracker.register('another_metric', direction='wrong') with pytest.raises(ValueError, match='already exists'): tracker.register('new_metric', direction='max')
def test_get_last_value(): tracker = metrics_tracking.MetricsTracker() tracker.register('new_metric', 'min') assert tracker.get_last_value('new_metric') is None tracker.set_history('new_metric', [ metrics_tracking.MetricObservation(1., 0), metrics_tracking.MetricObservation(2., 1), metrics_tracking.MetricObservation(3., 2) ]) assert tracker.get_last_value('new_metric') == 3.
def test_get_best_value(): tracker = metrics_tracking.MetricsTracker() tracker.register('metric_min', 'min') tracker.register('metric_max', 'max') assert tracker.get_best_value('metric_min') is None tracker.set_history('metric_min', [1., 2., 3.]) tracker.set_history('metric_max', [1., 2., 3.]) assert tracker.get_best_value('metric_min') == 1. assert tracker.get_best_value('metric_max') == 3.
def test_set_history(): tracker = metrics_tracking.MetricsTracker() tracker.set_history('new_metric', [ metrics_tracking.MetricObservation(0.5, 0), metrics_tracking.MetricObservation(1.5, 1), metrics_tracking.MetricObservation(2., 2), ]) values = [obs.value for obs in tracker.get_history('new_metric')] steps = [obs.step for obs in tracker.get_history('new_metric')] assert values == [[0.5], [1.5], [2.]] assert steps == [0, 1, 2]
def test_register_from_metrics(): # As well as direction inference. tracker = metrics_tracking.MetricsTracker( metrics=[metrics.CategoricalAccuracy(), metrics.MeanSquaredError()] ) assert set(tracker.metrics.keys()) == { "categorical_accuracy", "mean_squared_error", } assert tracker.metrics["categorical_accuracy"].direction == "max" assert tracker.metrics["mean_squared_error"].direction == "min"
def test_serialization(): tracker = metrics_tracking.MetricsTracker() tracker.register('metric_min', 'min') tracker.register('metric_max', 'max') tracker.set_history('metric_min', [1., 2., 3.]) tracker.set_history('metric_max', [1., 2., 3.]) new_tracker = metrics_tracking.MetricsTracker.from_config( tracker.get_config()) assert new_tracker.names == tracker.names assert new_tracker.directions == tracker.directions assert new_tracker.metrics_history == tracker.metrics_history
def test_get_statistics(): tracker = metrics_tracking.MetricsTracker() history = [random.random() for _ in range(10)] tracker.set_history('new_metric', history) stats = tracker.get_statistics('new_metric') assert set(stats.keys()) == {'min', 'max', 'mean', 'median', 'var', 'std'} assert stats['min'] == np.min(history) assert stats['max'] == np.max(history) assert stats['mean'] == np.mean(history) assert stats['median'] == np.median(history) assert stats['var'] == np.var(history) assert stats['std'] == np.std(history)
def test_get_history(): tracker = metrics_tracking.MetricsTracker() tracker.update('new_metric', 0.5, step=0) tracker.update('new_metric', 1.5, step=1) tracker.update('new_metric', 2., step=2) assert tracker.get_history('new_metric') == [ metrics_tracking.MetricObservation(0.5, 0), metrics_tracking.MetricObservation(1.5, 1), metrics_tracking.MetricObservation(2., 2), ] with pytest.raises(ValueError, match='Unknown metric'): tracker.get_history('another_metric')
def test_get_last_value(): tracker = metrics_tracking.MetricsTracker() tracker.register("new_metric", "min") assert tracker.get_last_value("new_metric") is None tracker.set_history( "new_metric", [ metrics_tracking.MetricObservation(1.0, 0), metrics_tracking.MetricObservation(2.0, 1), metrics_tracking.MetricObservation(3.0, 2), ], ) assert tracker.get_last_value("new_metric") == 3.0
def test_get_statistics(): tracker = metrics_tracking.MetricsTracker() history = [ metrics_tracking.MetricObservation(random.random(), i) for i in range(10) ] tracker.set_history("new_metric", history) stats = tracker.get_statistics("new_metric") assert set(stats.keys()) == {"min", "max", "mean", "median", "var", "std"} history = [obs.value for obs in history] assert stats["min"] == np.min(history) assert stats["max"] == np.max(history) assert stats["mean"] == np.mean(history) assert stats["median"] == np.median(history) assert stats["var"] == np.var(history) assert stats["std"] == np.std(history)
def test_get_statistics(): tracker = metrics_tracking.MetricsTracker() history = [ metrics_tracking.MetricObservation(random.random(), i) for i in range(10) ] tracker.set_history('new_metric', history) stats = tracker.get_statistics('new_metric') assert set(stats.keys()) == {'min', 'max', 'mean', 'median', 'var', 'std'} history = [obs.value for obs in history] assert stats['min'] == np.min(history) assert stats['max'] == np.max(history) assert stats['mean'] == np.mean(history) assert stats['median'] == np.median(history) assert stats['var'] == np.var(history) assert stats['std'] == np.std(history)
def test_serialization(): tracker = metrics_tracking.MetricsTracker() tracker.register('metric_min', 'min') tracker.register('metric_max', 'max') tracker.set_history('metric_min', [ metrics_tracking.MetricObservation(1., 0), metrics_tracking.MetricObservation(2., 1), metrics_tracking.MetricObservation(3., 2) ]) tracker.set_history('metric_max', [ metrics_tracking.MetricObservation(1., 0), metrics_tracking.MetricObservation(2., 1), metrics_tracking.MetricObservation(3., 2) ]) new_tracker = metrics_tracking.MetricsTracker.from_config( tracker.get_config()) assert new_tracker.metrics.keys() == tracker.metrics.keys()
def test_get_best_value(): tracker = metrics_tracking.MetricsTracker() tracker.register('metric_min', 'min') tracker.register('metric_max', 'max') assert tracker.get_best_value('metric_min') is None tracker.set_history('metric_min', [ metrics_tracking.MetricObservation(1., 0), metrics_tracking.MetricObservation(2., 1), metrics_tracking.MetricObservation(3., 2) ]) tracker.set_history('metric_max', [ metrics_tracking.MetricObservation(1., 0), metrics_tracking.MetricObservation(2., 1), metrics_tracking.MetricObservation(3., 2) ]) assert tracker.get_best_value('metric_min') == 1. assert tracker.get_best_value('metric_max') == 3.
def test_metricstracker_proto(): tracker = metrics_tracking.MetricsTracker() tracker.register('score', direction='max') tracker.update('score', value=10, step=1) tracker.update('score', value=20, step=1) tracker.update('score', value=30, step=2) proto = tracker.to_proto() obs = proto.metrics['score'].observations assert obs[0].value == [10, 20] assert obs[0].step == 1 assert obs[1].value == [30] assert obs[1].step == 2 assert proto.metrics['score'].maximize new_tracker = metrics_tracking.MetricsTracker.from_proto(proto) assert new_tracker.metrics['score'].direction == 'max' assert new_tracker.metrics['score'].get_history() == [ metrics_tracking.MetricObservation([10, 20], 1), metrics_tracking.MetricObservation(30, 2) ]
def test_serialization(): tracker = metrics_tracking.MetricsTracker() tracker.register("metric_min", "min") tracker.register("metric_max", "max") tracker.set_history( "metric_min", [ metrics_tracking.MetricObservation(1.0, 0), metrics_tracking.MetricObservation(2.0, 1), metrics_tracking.MetricObservation(3.0, 2), ], ) tracker.set_history( "metric_max", [ metrics_tracking.MetricObservation(1.0, 0), metrics_tracking.MetricObservation(2.0, 1), metrics_tracking.MetricObservation(3.0, 2), ], ) new_tracker = metrics_tracking.MetricsTracker.from_config(tracker.get_config()) assert new_tracker.metrics.keys() == tracker.metrics.keys()
def test_exists(): tracker = metrics_tracking.MetricsTracker() tracker.register("new_metric", direction="max") assert tracker.exists("new_metric") assert not tracker.exists("another_metric")
def test_set_history(): tracker = metrics_tracking.MetricsTracker() tracker.set_history('new_metric', [1., 2., 3.]) tracker.get_history('new_metric') == [1., 2., 3.]
def test_exists(): tracker = metrics_tracking.MetricsTracker() tracker.register('new_metric', direction='max') assert tracker.exists('new_metric') assert not tracker.exists('another_metric')
def test_get_last_value(): tracker = metrics_tracking.MetricsTracker() tracker.register('new_metric', 'min') assert tracker.get_last_value('new_metric') is None tracker.set_history('new_metric', [1., 2., 3.]) assert tracker.get_last_value('new_metric') is 3.
def test_update(): tracker = metrics_tracking.MetricsTracker() tracker.update('new_metric', 0.5) # automatic registration assert tracker.names == ['new_metric'] assert tracker.directions['new_metric'] == 'min' # default direction assert tracker.get_history('new_metric') == [0.5]