def test_metric_creation(): metric = Metric('test', 'min') assert metric.name == 'test' assert metric.direction == 'min' assert int(metric.start_time) == int(time.time()) assert metric.wall_time == [] assert metric.history == []
def add(self, metric): """ Add a metric to the collection Args: metric (Metric or str): Metric object or metric name """ # our own metric object -> direct add if isinstance(metric, Metric): # our own metric, do nothing metric_name = metric.name else: if isinstance(metric, str): # metric by name metric_name = metric else: # keras metric metric_name = metric.name metric_name = self._replace_alias(metric_name) # canonalize metric name (val_metric vs metric) no_val_name = metric_name.replace('val_', '') if no_val_name in _METRIC_DIRECTION: direction = _METRIC_DIRECTION[no_val_name] else: fatal('Unknown metric %s' % metric_name) # create a metric object metric = Metric(metric_name, direction) if metric_name in self._objects: fatal('Duplicate metric:%s' % metric_name) self._objects[metric_name] = metric self._last_insert_idx = metric_name
def test_metric_obj_add(mc): mm = Metric('test', 'min') mc.add(mm) metric = mc.get('test') assert isinstance(metric, Metric) assert metric.direction == 'min' assert metric.name == 'test' assert mm == metric
def test_from_config_to_config(mm): mm.is_objective = True config = mm.to_config() mm2 = Metric.from_config(config) assert mm2.name == 'name' assert mm2.direction == 'min' assert mm2.get_history() == [10, 11] assert mm2.is_objective
def test_from_config_to_config(mc): m = Metric("loss", "min") mc = MetricsCollection() mc.add(m) mc.update("loss", .5) config = mc.to_config() mc2 = MetricsCollection.from_config(config) mcl = mc.to_list() mc2l = mc2.to_list() assert mc2._objective_name == mc._objective_name for idx in range(len(mcl)): assert mcl[idx].name == mc2l[idx].name assert mcl[idx].get_last_value() == mc2l[idx].get_last_value()
def test_from_config_to_config_no_val(): m = Metric("loss", "min") mc = MetricsCollection() mc.add(m) mc.update("loss", .5) config = mc.to_config() mc2 = MetricsCollection.from_config(config, with_values=False) mcl = mc.to_list() mc2l = mc2.to_list() assert mc2._objective_name == mc._objective_name assert len(mcl) == 1 assert len(mc2l) == 1 assert mcl[0].get_last_value() == .5 assert mc2l[0].get_last_value() is None
def from_config(config, with_values=True): """Generate a MetricsCollection from a configuration dictionary Args: config - (dict) The configuration dict returned from to_config. with_values - (bool) If True, metric values are copied. If False, only the metadata is copied. Defaults to True. Returns: (MetricsCollection) The collection of metrics defined by the config. """ col = MetricsCollection() for metric_config in config: metric = Metric.from_config(metric_config, with_values=with_values) col.add(metric) if metric.is_objective: col._objective_name = metric.name return col
def test_duplicate_obj(mc): mm = Metric('acc', 'min') mc.add(mm) with pytest.raises(ValueError): mc.add(mm)
def test_single_update(): mm = Metric('min', 'min') assert mm.update(10)
def test_get_empty_last_value(): mm = Metric('min', 'min') assert not mm.get_last_value()
def test_best_max_value(): mm = Metric('max', 'max') mm.update(10) mm.update(8) assert mm.get_best_value() == 10
def test_metric_invalid_direction(): with pytest.raises(ValueError): Metric('test', 'invalid')
def test_metric_wall_time(): mm = Metric('acc', 'max') mm.update(10) time.sleep(1) mm.update(11) assert mm.wall_time[1] > 1
def mm(): mm = Metric('name', 'min') mm.update(10) mm.update(11) return mm