def test_metric_glob_twice_norm(publish_util): history = [] history.append(dict(step=0, data=dict(metric=1,))) m1a = pb.MetricRecord(glob_name="*") m1a.summary.best = True m1a.summary.max = True m1a.step_metric = "thestep" m1b = pb.MetricRecord(glob_name="*") m1b.summary.min = True metrics = _make_metrics([m1a, m1b]) ctx_util = publish_util(history=history, metrics=metrics) metrics = ctx_util.metrics summary = ctx_util.summary assert metrics and len(metrics) == 2 mstep, mmetric = metrics assert mstep == {"1": "thestep"} assert mmetric == {"1": "metric", "5": 1, "7": [1, 2, 4]} assert summary == { "_step": 0, "metric": 1, "metric": {"best": 1, "max": 1, "min": 1}, }
def test_metric_dot_glob(publish_util): """glob should escape the defined metric name.""" history = [] history.append(dict(step=0, data={"this.has.dots": 2})) history.append(dict(step=1, data={"this.also": 2})) history.append(dict(step=2, data={"nodots": 3})) history.append(dict(step=3, data={"this.also": 1})) m1 = pb.MetricRecord(name="this\\.also") m1.options.defined = True m1.summary.max = True m2 = pb.MetricRecord(glob_name="*") m2.options.defined = True m2.summary.min = True metrics = _make_metrics([m1, m2]) ctx_util = publish_util(history=history, metrics=metrics) metrics = ctx_util.metrics summary = ctx_util.summary assert metrics and len(metrics) == 3 # order doesnt really matter assert metrics[0] == {"1": "this\\.also", "7": [2], "6": [3]} assert metrics[1] == {"1": "this\\.has\\.dots", "7": [1]} assert metrics[2] == {"1": "nodots", "7": [1]} assert summary == { "_step": 3, "this.also": {"max": 2}, "nodots": {"min": 3}, "this.has.dots": {"min": 2}, }
def _handle_glob_metric(self, record: wandb_internal_pb2.Record) -> None: metric = record.metric if metric._control.overwrite: self._metric_globs.setdefault( metric.glob_name, wandb_internal_pb2.MetricRecord() ).CopyFrom(metric) else: self._metric_globs.setdefault( metric.glob_name, wandb_internal_pb2.MetricRecord() ).MergeFrom(metric) self._dispatch_record(record)
def test_metric_run_metric_obj(user_test): run = user_test.get_run() m1 = run._define_metric("glob") m2 = run._define_metric("val", step_metric=m1) assert m2.step_metric == "glob" r = user_test.get_records() assert len(r.records) == 2 assert len(r.metric) == 2 mr1, mr2 = r.metric assert mr1 == pb.MetricRecord(name="glob") assert mr2 == pb.MetricRecord(name="val", step_metric="glob")
def test_metric_again(publish_util): history = _gen_history() m1 = pb.MetricRecord(name="mystep") m2 = pb.MetricRecord(name="v1", step_metric="mystep") m3 = pb.MetricRecord(name="v2") m4 = pb.MetricRecord(name="v2", step_metric="mystep") metrics = _make_metrics([m1, m2, m3, m4]) ctx_util = publish_util(history=history, metrics=metrics) summary = ctx_util.summary assert {"v1": 2, "v2": 3, "v3": "pizza", "mystep": 3, "_step": 2,} == summary metrics = ctx_util.metrics assert metrics and len(metrics) == 3
def test_metric_twice_norm(publish_util): m1a = pb.MetricRecord(name="metric") m1a.summary.best = True m1a.summary.max = True m1a.step_metric = "thestep" m1b = pb.MetricRecord(name="metric") m1b.summary.min = True metrics = _make_metrics([m1a, m1b]) ctx_util = publish_util(metrics=metrics) metrics = ctx_util.metrics assert len(metrics) == 2 mstep, mmetric = metrics assert mstep == {"1": "thestep"} assert mmetric == {"1": "metric", "5": 1, "7": [1, 2, 4]}
def _make_metrics(mitems): metrics = [] for mitem in mitems: m = pb.MetricRecord() m.CopyFrom(mitem) metrics.append(m) return metrics
def _commit(self): m = pb.MetricRecord() if self._name.endswith("*"): m.glob_name = self._name else: m.name = self._name if self._step_metric: m.step_metric = self._step_metric if self._step_sync: m.options.step_sync = self._step_sync if self._hidden: m.options.hidden = self._hidden if self._summary: summary_set = set(self._summary) if "min" in summary_set: m.summary.min = True if "max" in summary_set: m.summary.max = True if "mean" in summary_set: m.summary.mean = True if "best" in summary_set: m.summary.best = True if self._goal == "min": m.goal = m.GOAL_MINIMIZE if self._goal == "max": m.goal = m.GOAL_MAXIMIZE if self._overwrite: m._control.overwrite = self._overwrite if self._callback: self._callback(m)
def test_metric_run_metric_obj(user_test): run = user_test.get_run() m1 = run.define_metric("glob") m2 = run.define_metric("val", step_metric=m1) assert m2.step_metric == "glob" r = user_test.get_records() assert len(r.records) == 2 assert len(r.metric) == 2 mr1, mr2 = r.metric glob_metric = pb.MetricRecord(name="glob") glob_metric.options.defined = True step_metric = pb.MetricRecord(name="val", step_metric="glob") step_metric.options.defined = True step_metric.options.step_sync = True assert mr1 == glob_metric assert mr2 == step_metric
def test_metric_sum_none(publish_util): history = _gen_history() m1 = pb.MetricRecord(name="v2") metrics = _make_metrics([m1]) ctx_util = publish_util(history=history, metrics=metrics) summary = ctx_util.summary assert {"v1": 2, "v2": 3, "v3": "pizza", "mystep": 3, "_step": 2,} == summary
def test_metric_min_str(publish_util): history = _gen_history() m1 = pb.MetricRecord(name="v3") m1.summary.min = True metrics = _make_metrics([m1]) ctx_util = publish_util(history=history, metrics=metrics) summary = ctx_util.summary assert {"v1": 2, "v2": 3, "mystep": 3, "_step": 2,} == summary
def test_metric_mult(publish_util): history = _gen_history() m1 = pb.MetricRecord(name="mystep") m2 = pb.MetricRecord(name="v1", step_metric="mystep") m2.summary.max = True m3 = pb.MetricRecord(name="v2", step_metric="mystep") m3.summary.min = True metrics = _make_metrics([m1, m2, m3]) ctx_util = publish_util(history=history, metrics=metrics) summary = ctx_util.summary assert { "v1": {"max": 3}, "v2": {"min": 2}, "v3": "pizza", "mystep": 3, "_step": 2, } == summary
def test_metric_step(publish_util): history = _gen_history() metrics = [ pb.MetricRecord(glob_name="*", step_metric="mystep"), ] metrics = _make_metrics(metrics) ctx_util = publish_util(history=history, metrics=metrics) summary = ctx_util.summary assert dict(v1=2, v2=3, v3="pizza", mystep=3, _step=2) == summary
def test_metric_glob_twice_over(publish_util): history = [] history.append(dict(step=0, data=dict(metric=1,))) m1a = pb.MetricRecord(glob_name="*") m1a.summary.best = True m1a.summary.max = True m1a.step_metric = "thestep" m1b = pb.MetricRecord(glob_name="*") m1b.summary.min = True m1b._control.overwrite = True metrics = _make_metrics([m1a, m1b]) ctx_util = publish_util(history=history, metrics=metrics) metrics = ctx_util.metrics summary = ctx_util.summary assert metrics and len(metrics) == 1 mmetric = metrics[0] assert mmetric == {"1": "metric", "7": [1]} assert summary == {"_step": 0, "metric": {"min": 1}}
def test_metric_nan_max(publish_util): history = [] history.append(dict(step=0, data=dict(v2=2))) history.append(dict(step=1, data=dict(v2=8))) history.append(dict(step=2, data=dict(v2=float("nan")))) m1 = pb.MetricRecord(name="v2") m1.summary.max = True metrics = _make_metrics([m1]) ctx_util = publish_util(history=history, metrics=metrics) summary = ctx_util.summary assert summary.get("v2") == {"max": 8}
def _handle_defined_metric(self, record: wandb_internal_pb2.Record) -> None: metric = record.metric if metric._control.overwrite: self._metric_defines.setdefault( metric.name, wandb_internal_pb2.MetricRecord() ).CopyFrom(metric) else: self._metric_defines.setdefault( metric.name, wandb_internal_pb2.MetricRecord() ).MergeFrom(metric) # before dispatching, make sure step_metric is defined, if not define it and # dispatch it locally first metric = self._metric_defines[metric.name] if metric.step_metric and metric.step_metric not in self._metric_defines: m = wandb_internal_pb2.MetricRecord(name=metric.step_metric) self._metric_defines[metric.step_metric] = m mr = wandb_internal_pb2.Record() mr.metric.CopyFrom(m) mr.control.local = True # Dont store this, just send it self._dispatch_record(mr) self._dispatch_record(record)
def _history_define_metric(self, hkey): """check for hkey match in glob metrics, return defined metric.""" # Dont define metric for internal metrics if hkey.startswith("_"): return None for k, mglob in six.iteritems(self._metric_globs): if k.endswith("*"): if hkey.startswith(k[:-1]): m = wandb_internal_pb2.MetricRecord() m.CopyFrom(mglob) m.ClearField("glob_name") m.name = hkey return m return None
def test_metric_best(publish_util): history = _gen_history() m1 = pb.MetricRecord(name="mystep") m2 = pb.MetricRecord(name="v1", step_metric="mystep") m2.summary.best = True m2.goal = m2.GOAL_MAXIMIZE m3 = pb.MetricRecord(name="v2", step_metric="mystep") m3.summary.best = True m3.goal = m3.GOAL_MINIMIZE metrics = _make_metrics([m1, m2, m3]) ctx_util = publish_util(history=history, metrics=metrics) summary = ctx_util.summary assert { "v1": 2, "v1": {"best": 3}, "v2": 3, "v2": {"best": 2}, "v3": "pizza", "mystep": 3, "_step": 2, } == summary
def test_metric_stepsync(publish_util): history = [] history.append(dict(step=0, data=dict(a1=1,))) history.append(dict(step=1, data=dict(s1=2))) history.append(dict(step=2, data=dict(a1=3,))) history.append(dict(step=3, data=dict(a1=5, s1=4))) history.append(dict(step=3, data=dict(s1=6))) history.append(dict(step=4, data=dict(a1=7,))) history.append(dict(step=5, data=dict(a1=9, s1=8))) m0 = pb.MetricRecord(name="s1") m1 = pb.MetricRecord(name="a1", step_metric="s1") m1.options.step_sync = True metrics = _make_metrics([m0, m1]) ctx_util = publish_util(history=history, metrics=metrics) summary = ctx_util.summary history = ctx_util.history assert {"a1": 9, "s1": 8, "_step": 5,} == summary history_val = [(h.get("a1"), h.get("s1")) for h in history if "a1" in h] assert history_val == [(1, None), (3, 2), (5, 4), (7, 6), (9, 8)]
def test_metric_mean(publish_util): history = _gen_history() m1 = pb.MetricRecord(name="v2", step_metric="mystep") m1.summary.mean = True metrics = _make_metrics([m1]) ctx_util = publish_util(history=history, metrics=metrics) summary = ctx_util.summary assert { "v1": 2, "v2": {"mean": 13.0 / 3}, "v3": "pizza", "mystep": 3, "_step": 2, } == summary
def send_metric(self, data: wandb_internal_pb2.Record) -> None: metric = data.metric if metric.glob_name: logger.warning("Seen metric with glob (shouldnt happen)") return # merge or overwrite old_metric = self._config_metric_dict.get( metric.name, wandb_internal_pb2.MetricRecord()) if metric._control.overwrite: old_metric.CopyFrom(metric) else: old_metric.MergeFrom(metric) self._config_metric_dict[metric.name] = old_metric metric = old_metric # TODO(jhr): remove this code before shipping (only for prototype UI) if metric.step_metric: if metric.step_metric != self._config_default_xaxis: self._config_default_xaxis = metric.step_metric self._update_config() # convert step_metric to index if metric.step_metric: find_step_idx = self._config_metric_index_dict.get( metric.step_metric) if find_step_idx is not None: # make a copy of this metric as we will be modifying it rec = wandb_internal_pb2.Record() rec.metric.CopyFrom(metric) metric = rec.metric metric.ClearField("step_metric") metric.step_metric_index = find_step_idx + 1 md: Dict[int, Any] = proto_util.proto_encode_to_dict(metric) find_idx = self._config_metric_index_dict.get(metric.name) if find_idx is not None: self._config_metric_pbdict_list[find_idx] = md else: next_idx = len(self._config_metric_pbdict_list) self._config_metric_pbdict_list.append(md) self._config_metric_index_dict[metric.name] = next_idx self._update_config()
def test_metric_dot_flat_notescaped(publish_util): """match doesnt work if metric is not escaped.""" history = [] history.append(dict(step=0, data={"this.has.dots": 2})) history.append(dict(step=1, data={"this.also": 2})) history.append(dict(step=2, data={"nodots": 2})) history.append(dict(step=3, data={"this.also": 1})) m1 = pb.MetricRecord(name="this.also") m1.summary.max = True metrics = _make_metrics([m1]) ctx_util = publish_util(history=history, metrics=metrics) metrics = ctx_util.metrics summary = ctx_util.summary assert metrics and len(metrics) == 1 mmetric = metrics[0] assert mmetric == {"1": "this.also", "7": [2]} assert summary == { "_step": 3, "this.also": 1, "nodots": 2, "this.has.dots": 2, }