def test_metric_glob_twice_norm(publish_util):
history = []
history.append(dict(step=0, data=dict(metric=1,)))
m1a = pb.MetricRecord(glob_name="*")
m1a.summary.best = True
m1a.summary.max = True
m1a.step_metric = "thestep"
m1b = pb.MetricRecord(glob_name="*")
m1b.summary.min = True
metrics = _make_metrics([m1a, m1b])
ctx_util = publish_util(history=history, metrics=metrics)
metrics = ctx_util.metrics
summary = ctx_util.summary
assert metrics and len(metrics) == 2
mstep, mmetric = metrics
assert mstep == {"1": "thestep"}
assert mmetric == {"1": "metric", "5": 1, "7": [1, 2, 4]}
assert summary == {
"_step": 0,
"metric": 1,
"metric": {"best": 1, "max": 1, "min": 1},
}
def test_metric_dot_glob(publish_util):
"""glob should escape the defined metric name."""
history = []
history.append(dict(step=0, data={"this.has.dots": 2}))
history.append(dict(step=1, data={"this.also": 2}))
history.append(dict(step=2, data={"nodots": 3}))
history.append(dict(step=3, data={"this.also": 1}))
m1 = pb.MetricRecord(name="this\\.also")
m1.options.defined = True
m1.summary.max = True
m2 = pb.MetricRecord(glob_name="*")
m2.options.defined = True
m2.summary.min = True
metrics = _make_metrics([m1, m2])
ctx_util = publish_util(history=history, metrics=metrics)
metrics = ctx_util.metrics
summary = ctx_util.summary
assert metrics and len(metrics) == 3
# order doesnt really matter
assert metrics[0] == {"1": "this\\.also", "7": [2], "6": [3]}
assert metrics[1] == {"1": "this\\.has\\.dots", "7": [1]}
assert metrics[2] == {"1": "nodots", "7": [1]}
assert summary == {
"_step": 3,
"this.also": {"max": 2},
"nodots": {"min": 3},
"this.has.dots": {"min": 2},
}
def _handle_glob_metric(self, record: wandb_internal_pb2.Record) -> None:
metric = record.metric
if metric._control.overwrite:
self._metric_globs.setdefault(
metric.glob_name, wandb_internal_pb2.MetricRecord()
).CopyFrom(metric)
else:
self._metric_globs.setdefault(
metric.glob_name, wandb_internal_pb2.MetricRecord()
).MergeFrom(metric)
self._dispatch_record(record)
def test_metric_run_metric_obj(user_test):
run = user_test.get_run()
m1 = run._define_metric("glob")
m2 = run._define_metric("val", step_metric=m1)
assert m2.step_metric == "glob"
r = user_test.get_records()
assert len(r.records) == 2
assert len(r.metric) == 2
mr1, mr2 = r.metric
assert mr1 == pb.MetricRecord(name="glob")
assert mr2 == pb.MetricRecord(name="val", step_metric="glob")
def test_metric_again(publish_util):
history = _gen_history()
m1 = pb.MetricRecord(name="mystep")
m2 = pb.MetricRecord(name="v1", step_metric="mystep")
m3 = pb.MetricRecord(name="v2")
m4 = pb.MetricRecord(name="v2", step_metric="mystep")
metrics = _make_metrics([m1, m2, m3, m4])
ctx_util = publish_util(history=history, metrics=metrics)
summary = ctx_util.summary
assert {"v1": 2, "v2": 3, "v3": "pizza", "mystep": 3, "_step": 2,} == summary
metrics = ctx_util.metrics
assert metrics and len(metrics) == 3
def test_metric_twice_norm(publish_util):
m1a = pb.MetricRecord(name="metric")
m1a.summary.best = True
m1a.summary.max = True
m1a.step_metric = "thestep"
m1b = pb.MetricRecord(name="metric")
m1b.summary.min = True
metrics = _make_metrics([m1a, m1b])
ctx_util = publish_util(metrics=metrics)
metrics = ctx_util.metrics
assert len(metrics) == 2
mstep, mmetric = metrics
assert mstep == {"1": "thestep"}
assert mmetric == {"1": "metric", "5": 1, "7": [1, 2, 4]}
def _make_metrics(mitems):
metrics = []
for mitem in mitems:
m = pb.MetricRecord()
m.CopyFrom(mitem)
metrics.append(m)
return metrics
def _commit(self):
m = pb.MetricRecord()
if self._name.endswith("*"):
m.glob_name = self._name
else:
m.name = self._name
if self._step_metric:
m.step_metric = self._step_metric
if self._step_sync:
m.options.step_sync = self._step_sync
if self._hidden:
m.options.hidden = self._hidden
if self._summary:
summary_set = set(self._summary)
if "min" in summary_set:
m.summary.min = True
if "max" in summary_set:
m.summary.max = True
if "mean" in summary_set:
m.summary.mean = True
if "best" in summary_set:
m.summary.best = True
if self._goal == "min":
m.goal = m.GOAL_MINIMIZE
if self._goal == "max":
m.goal = m.GOAL_MAXIMIZE
if self._overwrite:
m._control.overwrite = self._overwrite
if self._callback:
self._callback(m)
def test_metric_run_metric_obj(user_test):
run = user_test.get_run()
m1 = run.define_metric("glob")
m2 = run.define_metric("val", step_metric=m1)
assert m2.step_metric == "glob"
r = user_test.get_records()
assert len(r.records) == 2
assert len(r.metric) == 2
mr1, mr2 = r.metric
glob_metric = pb.MetricRecord(name="glob")
glob_metric.options.defined = True
step_metric = pb.MetricRecord(name="val", step_metric="glob")
step_metric.options.defined = True
step_metric.options.step_sync = True
assert mr1 == glob_metric
assert mr2 == step_metric
def test_metric_sum_none(publish_util):
history = _gen_history()
m1 = pb.MetricRecord(name="v2")
metrics = _make_metrics([m1])
ctx_util = publish_util(history=history, metrics=metrics)
summary = ctx_util.summary
assert {"v1": 2, "v2": 3, "v3": "pizza", "mystep": 3, "_step": 2,} == summary
def test_metric_min_str(publish_util):
history = _gen_history()
m1 = pb.MetricRecord(name="v3")
m1.summary.min = True
metrics = _make_metrics([m1])
ctx_util = publish_util(history=history, metrics=metrics)
summary = ctx_util.summary
assert {"v1": 2, "v2": 3, "mystep": 3, "_step": 2,} == summary
def test_metric_mult(publish_util):
history = _gen_history()
m1 = pb.MetricRecord(name="mystep")
m2 = pb.MetricRecord(name="v1", step_metric="mystep")
m2.summary.max = True
m3 = pb.MetricRecord(name="v2", step_metric="mystep")
m3.summary.min = True
metrics = _make_metrics([m1, m2, m3])
ctx_util = publish_util(history=history, metrics=metrics)
summary = ctx_util.summary
assert {
"v1": {"max": 3},
"v2": {"min": 2},
"v3": "pizza",
"mystep": 3,
"_step": 2,
} == summary
def test_metric_step(publish_util):
history = _gen_history()
metrics = [
pb.MetricRecord(glob_name="*", step_metric="mystep"),
]
metrics = _make_metrics(metrics)
ctx_util = publish_util(history=history, metrics=metrics)
summary = ctx_util.summary
assert dict(v1=2, v2=3, v3="pizza", mystep=3, _step=2) == summary
def test_metric_glob_twice_over(publish_util):
history = []
history.append(dict(step=0, data=dict(metric=1,)))
m1a = pb.MetricRecord(glob_name="*")
m1a.summary.best = True
m1a.summary.max = True
m1a.step_metric = "thestep"
m1b = pb.MetricRecord(glob_name="*")
m1b.summary.min = True
m1b._control.overwrite = True
metrics = _make_metrics([m1a, m1b])
ctx_util = publish_util(history=history, metrics=metrics)
metrics = ctx_util.metrics
summary = ctx_util.summary
assert metrics and len(metrics) == 1
mmetric = metrics[0]
assert mmetric == {"1": "metric", "7": [1]}
assert summary == {"_step": 0, "metric": {"min": 1}}
def test_metric_nan_max(publish_util):
history = []
history.append(dict(step=0, data=dict(v2=2)))
history.append(dict(step=1, data=dict(v2=8)))
history.append(dict(step=2, data=dict(v2=float("nan"))))
m1 = pb.MetricRecord(name="v2")
m1.summary.max = True
metrics = _make_metrics([m1])
ctx_util = publish_util(history=history, metrics=metrics)
summary = ctx_util.summary
assert summary.get("v2") == {"max": 8}
def _handle_defined_metric(self, record: wandb_internal_pb2.Record) -> None:
metric = record.metric
if metric._control.overwrite:
self._metric_defines.setdefault(
metric.name, wandb_internal_pb2.MetricRecord()
).CopyFrom(metric)
else:
self._metric_defines.setdefault(
metric.name, wandb_internal_pb2.MetricRecord()
).MergeFrom(metric)
# before dispatching, make sure step_metric is defined, if not define it and
# dispatch it locally first
metric = self._metric_defines[metric.name]
if metric.step_metric and metric.step_metric not in self._metric_defines:
m = wandb_internal_pb2.MetricRecord(name=metric.step_metric)
self._metric_defines[metric.step_metric] = m
mr = wandb_internal_pb2.Record()
mr.metric.CopyFrom(m)
mr.control.local = True # Dont store this, just send it
self._dispatch_record(mr)
self._dispatch_record(record)
def _history_define_metric(self, hkey):
"""check for hkey match in glob metrics, return defined metric."""
# Dont define metric for internal metrics
if hkey.startswith("_"):
return None
for k, mglob in six.iteritems(self._metric_globs):
if k.endswith("*"):
if hkey.startswith(k[:-1]):
m = wandb_internal_pb2.MetricRecord()
m.CopyFrom(mglob)
m.ClearField("glob_name")
m.name = hkey
return m
return None
def test_metric_best(publish_util):
history = _gen_history()
m1 = pb.MetricRecord(name="mystep")
m2 = pb.MetricRecord(name="v1", step_metric="mystep")
m2.summary.best = True
m2.goal = m2.GOAL_MAXIMIZE
m3 = pb.MetricRecord(name="v2", step_metric="mystep")
m3.summary.best = True
m3.goal = m3.GOAL_MINIMIZE
metrics = _make_metrics([m1, m2, m3])
ctx_util = publish_util(history=history, metrics=metrics)
summary = ctx_util.summary
assert {
"v1": 2,
"v1": {"best": 3},
"v2": 3,
"v2": {"best": 2},
"v3": "pizza",
"mystep": 3,
"_step": 2,
} == summary
def test_metric_stepsync(publish_util):
history = []
history.append(dict(step=0, data=dict(a1=1,)))
history.append(dict(step=1, data=dict(s1=2)))
history.append(dict(step=2, data=dict(a1=3,)))
history.append(dict(step=3, data=dict(a1=5, s1=4)))
history.append(dict(step=3, data=dict(s1=6)))
history.append(dict(step=4, data=dict(a1=7,)))
history.append(dict(step=5, data=dict(a1=9, s1=8)))
m0 = pb.MetricRecord(name="s1")
m1 = pb.MetricRecord(name="a1", step_metric="s1")
m1.options.step_sync = True
metrics = _make_metrics([m0, m1])
ctx_util = publish_util(history=history, metrics=metrics)
summary = ctx_util.summary
history = ctx_util.history
assert {"a1": 9, "s1": 8, "_step": 5,} == summary
history_val = [(h.get("a1"), h.get("s1")) for h in history if "a1" in h]
assert history_val == [(1, None), (3, 2), (5, 4), (7, 6), (9, 8)]
def test_metric_mean(publish_util):
history = _gen_history()
m1 = pb.MetricRecord(name="v2", step_metric="mystep")
m1.summary.mean = True
metrics = _make_metrics([m1])
ctx_util = publish_util(history=history, metrics=metrics)
summary = ctx_util.summary
assert {
"v1": 2,
"v2": {"mean": 13.0 / 3},
"v3": "pizza",
"mystep": 3,
"_step": 2,
} == summary
def send_metric(self, data: wandb_internal_pb2.Record) -> None:
metric = data.metric
if metric.glob_name:
logger.warning("Seen metric with glob (shouldnt happen)")
return
# merge or overwrite
old_metric = self._config_metric_dict.get(
metric.name, wandb_internal_pb2.MetricRecord())
if metric._control.overwrite:
old_metric.CopyFrom(metric)
else:
old_metric.MergeFrom(metric)
self._config_metric_dict[metric.name] = old_metric
metric = old_metric
# TODO(jhr): remove this code before shipping (only for prototype UI)
if metric.step_metric:
if metric.step_metric != self._config_default_xaxis:
self._config_default_xaxis = metric.step_metric
self._update_config()
# convert step_metric to index
if metric.step_metric:
find_step_idx = self._config_metric_index_dict.get(
metric.step_metric)
if find_step_idx is not None:
# make a copy of this metric as we will be modifying it
rec = wandb_internal_pb2.Record()
rec.metric.CopyFrom(metric)
metric = rec.metric
metric.ClearField("step_metric")
metric.step_metric_index = find_step_idx + 1
md: Dict[int, Any] = proto_util.proto_encode_to_dict(metric)
find_idx = self._config_metric_index_dict.get(metric.name)
if find_idx is not None:
self._config_metric_pbdict_list[find_idx] = md
else:
next_idx = len(self._config_metric_pbdict_list)
self._config_metric_pbdict_list.append(md)
self._config_metric_index_dict[metric.name] = next_idx
self._update_config()
def test_metric_dot_flat_notescaped(publish_util):
"""match doesnt work if metric is not escaped."""
history = []
history.append(dict(step=0, data={"this.has.dots": 2}))
history.append(dict(step=1, data={"this.also": 2}))
history.append(dict(step=2, data={"nodots": 2}))
history.append(dict(step=3, data={"this.also": 1}))
m1 = pb.MetricRecord(name="this.also")
m1.summary.max = True
metrics = _make_metrics([m1])
ctx_util = publish_util(history=history, metrics=metrics)
metrics = ctx_util.metrics
summary = ctx_util.summary
assert metrics and len(metrics) == 1
mmetric = metrics[0]
assert mmetric == {"1": "this.also", "7": [2]}
assert summary == {
"_step": 3,
"this.also": 1,
"nodots": 2,
"this.has.dots": 2,
}
