def test_log_metric(tracking_uri_mock, tmpdir):
with start_run() as active_run, mock.patch("time.time") as time_mock:
time_mock.side_effect = range(300, 400)
run_uuid = active_run.info.run_uuid
mlflow.log_metric("name_1", 25)
mlflow.log_metric("name_2", -3)
mlflow.log_metric("name_1", 30, 5)
mlflow.log_metric("name_1", 40, -2)
mlflow.log_metric("nested/nested/name", 40)
finished_run = tracking.MlflowClient().get_run(run_uuid)
# Validate metrics
assert len(finished_run.data.metrics) == 3
expected_pairs = {"name_1": 30, "name_2": -3, "nested/nested/name": 40}
for key, value in finished_run.data.metrics.items():
assert expected_pairs[key] == value
# TODO: use client get_metric_history API here instead once it exists
fs = FileStore(os.path.join(tmpdir.strpath, "mlruns"))
metric_history_name1 = fs.get_metric_history(run_uuid, "name_1")
assert set([(m.value, m.timestamp, m.step)
for m in metric_history_name1]) == set([
(25, 300, 0),
(30, 302, 5),
(40, 303, -2),
])
metric_history_name2 = fs.get_metric_history(run_uuid, "name_2")
assert set([(m.value, m.timestamp, m.step)
for m in metric_history_name2]) == set([
(-3, 301, 0),
])
def test_log_batch(tracking_uri_mock, tmpdir):
expected_metrics = {"metric-key0": 1.0, "metric-key1": 4.0}
expected_params = {"param-key0": "param-val0", "param-key1": "param-val1"}
exact_expected_tags = {"tag-key0": "tag-val0", "tag-key1": "tag-val1"}
approx_expected_tags = set([MLFLOW_SOURCE_NAME, MLFLOW_SOURCE_TYPE])
t = int(time.time())
sorted_expected_metrics = sorted(expected_metrics.items(),
key=lambda kv: kv[0])
metrics = [
Metric(key=key, value=value, timestamp=t, step=i)
for i, (key, value) in enumerate(sorted_expected_metrics)
]
params = [
Param(key=key, value=value) for key, value in expected_params.items()
]
tags = [
RunTag(key=key, value=value)
for key, value in exact_expected_tags.items()
]
with start_run() as active_run:
run_uuid = active_run.info.run_uuid
mlflow.tracking.MlflowClient().log_batch(run_id=run_uuid,
metrics=metrics,
params=params,
tags=tags)
finished_run = tracking.MlflowClient().get_run(run_uuid)
# Validate metrics
assert len(finished_run.data.metrics) == 2
for key, value in finished_run.data.metrics.items():
assert expected_metrics[key] == value
# TODO: use client get_metric_history API here instead once it exists
fs = FileStore(os.path.join(tmpdir.strpath, "mlruns"))
metric_history0 = fs.get_metric_history(run_uuid, "metric-key0")
assert set([(m.value, m.timestamp, m.step)
for m in metric_history0]) == set([
(1.0, t, 0),
])
metric_history1 = fs.get_metric_history(run_uuid, "metric-key1")
assert set([(m.value, m.timestamp, m.step)
for m in metric_history1]) == set([
(4.0, t, 1),
])
# Validate tags (for automatically-set tags)
assert len(finished_run.data.tags
) == len(exact_expected_tags) + len(approx_expected_tags)
for tag_key, tag_value in finished_run.data.tags.items():
if tag_key in approx_expected_tags:
pass
else:
assert exact_expected_tags[tag_key] == tag_value
# Validate params
assert finished_run.data.params == expected_params
def test_log_metric_allows_multiple_values_at_same_ts_and_run_data_uses_max_ts_value(
self):
fs = FileStore(self.test_root)
run_uuid = self._create_run(fs).info.run_uuid
metric_name = "test-metric-1"
timestamp_values_mapping = {
1000: [float(i) for i in range(-20, 20)],
2000: [float(i) for i in range(-10, 10)],
}
logged_values = []
for timestamp, value_range in timestamp_values_mapping.items():
for value in reversed(value_range):
fs.log_metric(run_uuid, Metric(metric_name, value, timestamp))
logged_values.append(value)
six.assertCountEqual(self, [
metric.value
for metric in fs.get_metric_history(run_uuid, metric_name)
], logged_values)
run_metrics = fs.get_run(run_uuid).data.metrics
assert len(run_metrics) == 1
logged_metric_val = run_metrics[metric_name]
max_timestamp = max(timestamp_values_mapping)
assert logged_metric_val == max(
timestamp_values_mapping[max_timestamp])
def test_log_metric_allows_multiple_values_at_same_step_and_run_data_uses_max_step_value(self):
fs = FileStore(self.test_root)
run_id = self._create_run(fs).info.run_id
metric_name = "test-metric-1"
# Check that we get the max of (step, timestamp, value) in that order
tuples_to_log = [
(0, 100, 1000),
(3, 40, 100), # larger step wins even though it has smaller value
(3, 50, 10), # larger timestamp wins even though it has smaller value
(3, 50, 20), # tiebreak by max value
(3, 50, 20), # duplicate metrics with same (step, timestamp, value) are ok
# verify that we can log steps out of order / negative steps
(-3, 900, 900),
(-1, 800, 800),
]
for step, timestamp, value in reversed(tuples_to_log):
fs.log_metric(run_id, Metric(metric_name, value, timestamp, step))
metric_history = fs.get_metric_history(run_id, metric_name)
logged_tuples = [(m.step, m.timestamp, m.value) for m in metric_history]
assert set(logged_tuples) == set(tuples_to_log)
run_data = fs.get_run(run_id).data
run_metrics = run_data.metrics
assert len(run_metrics) == 1
assert run_metrics[metric_name] == 20
metric_obj = run_data._metric_objs[0]
assert metric_obj.key == metric_name
assert metric_obj.step == 3
assert metric_obj.timestamp == 50
assert metric_obj.value == 20
def test_weird_metric_names(self):
WEIRD_METRIC_NAME = "this is/a weird/but valid metric"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_metric(run_id, Metric(WEIRD_METRIC_NAME, 10, 1234, 0))
run = fs.get_run(run_id)
assert run.data.metrics[WEIRD_METRIC_NAME] == 10
history = fs.get_metric_history(run_id, WEIRD_METRIC_NAME)
assert len(history) == 1
metric = history[0]
assert metric.key == WEIRD_METRIC_NAME
assert metric.value == 10
assert metric.timestamp == 1234
def test_get_metric_history(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
runs = self.exp_data[exp_id]["runs"]
for run_uuid in runs:
run_info = self.run_data[run_uuid]
metrics = run_info.pop("metrics")
for metric_name, values in metrics.items():
metric_history = fs.get_metric_history(run_uuid, metric_name)
sorted_values = sorted(values, reverse=True)
for metric in metric_history:
timestamp, metric_value = sorted_values.pop()
self.assertEqual(metric.timestamp, timestamp)
self.assertEqual(metric.key, metric_name)
self.assertEqual(metric.value, metric_value)
def test_log_metrics_params_tags(mlflow_client):
experiment_id = mlflow_client.create_experiment('Oh My')
created_run = mlflow_client.create_run(experiment_id)
run_id = created_run.info.run_uuid
mlflow_client.log_metric(run_id,
key='metric',
value=123.456,
timestamp=789,
step=2)
mlflow_client.log_metric(run_id,
key='stepless-metric',
value=987.654,
timestamp=321)
mlflow_client.log_param(run_id, 'param', 'value')
mlflow_client.set_tag(run_id, 'taggity', 'do-dah')
run = mlflow_client.get_run(run_id)
assert run.data.metrics.get('metric') == 123.456
assert run.data.metrics.get('stepless-metric') == 987.654
assert run.data.params.get('param') == 'value'
assert run.data.tags.get('taggity') == 'do-dah'
# TODO(sid): replace this with mlflow_client.get_metric_history
fs = FileStore(server_root_dir)
metric_history0 = fs.get_metric_history(run_id, "metric")
assert len(metric_history0) == 1
metric0 = metric_history0[0]
assert metric0.key == "metric"
assert metric0.value == 123.456
assert metric0.timestamp == 789
assert metric0.step == 2
metric_history1 = fs.get_metric_history(run_id, "stepless-metric")
assert len(metric_history1) == 1
metric1 = metric_history1[0]
assert metric1.key == "stepless-metric"
assert metric1.value == 987.654
assert metric1.timestamp == 321
assert metric1.step == 0
def test_log_batch(mlflow_client):
experiment_id = mlflow_client.create_experiment('Batch em up')
created_run = mlflow_client.create_run(experiment_id)
run_id = created_run.info.run_uuid
# TODO(sid): pass and assert on step
mlflow_client.log_batch(run_id=run_id,
metrics=[Metric("metric", 123.456, 789, 0)],
params=[Param("param", "value")],
tags=[RunTag("taggity", "do-dah")])
run = mlflow_client.get_run(run_id)
assert run.data.metrics.get('metric') == 123.456
assert run.data.params.get('param') == 'value'
assert run.data.tags.get('taggity') == 'do-dah'
# TODO(sid): replace this with mlflow_client.get_metric_history
fs = FileStore(server_root_dir)
metric_history = fs.get_metric_history(run_id, "metric")
assert len(metric_history) == 1
metric = metric_history[0]
assert metric.key == "metric"
assert metric.value == 123.456
assert metric.timestamp == 789
def test_log_metrics_params_tags(mlflow_client):
experiment_id = mlflow_client.create_experiment('Oh My')
created_run = mlflow_client.create_run(experiment_id)
run_id = created_run.info.run_uuid
# TODO(sid): pass and assert on step
mlflow_client.log_metric(run_id,
key='metric',
value=123.456,
timestamp=789)
mlflow_client.log_param(run_id, 'param', 'value')
mlflow_client.set_tag(run_id, 'taggity', 'do-dah')
run = mlflow_client.get_run(run_id)
assert run.data.metrics.get('metric') == 123.456
assert run.data.params.get('param') == 'value'
assert run.data.tags.get('taggity') == 'do-dah'
# TODO(sid): replace this with mlflow_client.get_metric_history
fs = FileStore(server_root_dir)
metric_history = fs.get_metric_history(run_id, "metric")
assert len(metric_history) == 1
metric = metric_history[0]
assert metric.key == "metric"
assert metric.value == 123.456
assert metric.timestamp == 789
