def test_metric_creation():
metric = Metric('test', 'min')
assert metric.name == 'test'
assert metric.direction == 'min'
assert int(metric.start_time) == int(time.time())
assert metric.wall_time == []
assert metric.history == []
def add(self, metric):
""" Add a metric to the collection
Args:
metric (Metric or str): Metric object or metric name
"""
# our own metric object -> direct add
if isinstance(metric, Metric):
# our own metric, do nothing
metric_name = metric.name
else:
if isinstance(metric, str):
# metric by name
metric_name = metric
else:
# keras metric
metric_name = metric.name
metric_name = self._replace_alias(metric_name)
# canonalize metric name (val_metric vs metric)
no_val_name = metric_name.replace('val_', '')
if no_val_name in _METRIC_DIRECTION:
direction = _METRIC_DIRECTION[no_val_name]
else:
fatal('Unknown metric %s' % metric_name)
# create a metric object
metric = Metric(metric_name, direction)
if metric_name in self._objects:
fatal('Duplicate metric:%s' % metric_name)
self._objects[metric_name] = metric
self._last_insert_idx = metric_name
def test_metric_obj_add(mc):
mm = Metric('test', 'min')
mc.add(mm)
metric = mc.get('test')
assert isinstance(metric, Metric)
assert metric.direction == 'min'
assert metric.name == 'test'
assert mm == metric
def test_from_config_to_config(mm):
mm.is_objective = True
config = mm.to_config()
mm2 = Metric.from_config(config)
assert mm2.name == 'name'
assert mm2.direction == 'min'
assert mm2.get_history() == [10, 11]
assert mm2.is_objective
def test_from_config_to_config(mc):
m = Metric("loss", "min")
mc = MetricsCollection()
mc.add(m)
mc.update("loss", .5)
config = mc.to_config()
mc2 = MetricsCollection.from_config(config)
mcl = mc.to_list()
mc2l = mc2.to_list()
assert mc2._objective_name == mc._objective_name
for idx in range(len(mcl)):
assert mcl[idx].name == mc2l[idx].name
assert mcl[idx].get_last_value() == mc2l[idx].get_last_value()
def test_from_config_to_config_no_val():
m = Metric("loss", "min")
mc = MetricsCollection()
mc.add(m)
mc.update("loss", .5)
config = mc.to_config()
mc2 = MetricsCollection.from_config(config, with_values=False)
mcl = mc.to_list()
mc2l = mc2.to_list()
assert mc2._objective_name == mc._objective_name
assert len(mcl) == 1
assert len(mc2l) == 1
assert mcl[0].get_last_value() == .5
assert mc2l[0].get_last_value() is None
def from_config(config, with_values=True):
"""Generate a MetricsCollection from a configuration dictionary
Args:
config - (dict) The configuration dict returned from to_config.
with_values - (bool) If True, metric values are copied. If False,
only the metadata is copied. Defaults to True.
Returns:
(MetricsCollection) The collection of metrics defined by the config.
"""
col = MetricsCollection()
for metric_config in config:
metric = Metric.from_config(metric_config, with_values=with_values)
col.add(metric)
if metric.is_objective:
col._objective_name = metric.name
return col
def test_duplicate_obj(mc):
mm = Metric('acc', 'min')
mc.add(mm)
with pytest.raises(ValueError):
mc.add(mm)
def test_single_update():
mm = Metric('min', 'min')
assert mm.update(10)
def test_get_empty_last_value():
mm = Metric('min', 'min')
assert not mm.get_last_value()
def test_best_max_value():
mm = Metric('max', 'max')
mm.update(10)
mm.update(8)
assert mm.get_best_value() == 10
def test_metric_invalid_direction():
with pytest.raises(ValueError):
Metric('test', 'invalid')
def test_metric_wall_time():
mm = Metric('acc', 'max')
mm.update(10)
time.sleep(1)
mm.update(11)
assert mm.wall_time[1] > 1
def mm():
mm = Metric('name', 'min')
mm.update(10)
mm.update(11)
return mm