def test_log_metric(tracking_uri_mock, tmpdir): with start_run() as active_run, mock.patch("time.time") as time_mock: time_mock.side_effect = range(300, 400) run_uuid = active_run.info.run_uuid mlflow.log_metric("name_1", 25) mlflow.log_metric("name_2", -3) mlflow.log_metric("name_1", 30, 5) mlflow.log_metric("name_1", 40, -2) mlflow.log_metric("nested/nested/name", 40) finished_run = tracking.MlflowClient().get_run(run_uuid) # Validate metrics assert len(finished_run.data.metrics) == 3 expected_pairs = {"name_1": 30, "name_2": -3, "nested/nested/name": 40} for key, value in finished_run.data.metrics.items(): assert expected_pairs[key] == value # TODO: use client get_metric_history API here instead once it exists fs = FileStore(os.path.join(tmpdir.strpath, "mlruns")) metric_history_name1 = fs.get_metric_history(run_uuid, "name_1") assert set([(m.value, m.timestamp, m.step) for m in metric_history_name1]) == set([ (25, 300, 0), (30, 302, 5), (40, 303, -2), ]) metric_history_name2 = fs.get_metric_history(run_uuid, "name_2") assert set([(m.value, m.timestamp, m.step) for m in metric_history_name2]) == set([ (-3, 301, 0), ])
def test_log_batch(tracking_uri_mock, tmpdir): expected_metrics = {"metric-key0": 1.0, "metric-key1": 4.0} expected_params = {"param-key0": "param-val0", "param-key1": "param-val1"} exact_expected_tags = {"tag-key0": "tag-val0", "tag-key1": "tag-val1"} approx_expected_tags = set([MLFLOW_SOURCE_NAME, MLFLOW_SOURCE_TYPE]) t = int(time.time()) sorted_expected_metrics = sorted(expected_metrics.items(), key=lambda kv: kv[0]) metrics = [ Metric(key=key, value=value, timestamp=t, step=i) for i, (key, value) in enumerate(sorted_expected_metrics) ] params = [ Param(key=key, value=value) for key, value in expected_params.items() ] tags = [ RunTag(key=key, value=value) for key, value in exact_expected_tags.items() ] with start_run() as active_run: run_uuid = active_run.info.run_uuid mlflow.tracking.MlflowClient().log_batch(run_id=run_uuid, metrics=metrics, params=params, tags=tags) finished_run = tracking.MlflowClient().get_run(run_uuid) # Validate metrics assert len(finished_run.data.metrics) == 2 for key, value in finished_run.data.metrics.items(): assert expected_metrics[key] == value # TODO: use client get_metric_history API here instead once it exists fs = FileStore(os.path.join(tmpdir.strpath, "mlruns")) metric_history0 = fs.get_metric_history(run_uuid, "metric-key0") assert set([(m.value, m.timestamp, m.step) for m in metric_history0]) == set([ (1.0, t, 0), ]) metric_history1 = fs.get_metric_history(run_uuid, "metric-key1") assert set([(m.value, m.timestamp, m.step) for m in metric_history1]) == set([ (4.0, t, 1), ]) # Validate tags (for automatically-set tags) assert len(finished_run.data.tags ) == len(exact_expected_tags) + len(approx_expected_tags) for tag_key, tag_value in finished_run.data.tags.items(): if tag_key in approx_expected_tags: pass else: assert exact_expected_tags[tag_key] == tag_value # Validate params assert finished_run.data.params == expected_params
def test_log_metric_allows_multiple_values_at_same_ts_and_run_data_uses_max_ts_value( self): fs = FileStore(self.test_root) run_uuid = self._create_run(fs).info.run_uuid metric_name = "test-metric-1" timestamp_values_mapping = { 1000: [float(i) for i in range(-20, 20)], 2000: [float(i) for i in range(-10, 10)], } logged_values = [] for timestamp, value_range in timestamp_values_mapping.items(): for value in reversed(value_range): fs.log_metric(run_uuid, Metric(metric_name, value, timestamp)) logged_values.append(value) six.assertCountEqual(self, [ metric.value for metric in fs.get_metric_history(run_uuid, metric_name) ], logged_values) run_metrics = fs.get_run(run_uuid).data.metrics assert len(run_metrics) == 1 logged_metric_val = run_metrics[metric_name] max_timestamp = max(timestamp_values_mapping) assert logged_metric_val == max( timestamp_values_mapping[max_timestamp])
def test_log_metric_allows_multiple_values_at_same_step_and_run_data_uses_max_step_value(self): fs = FileStore(self.test_root) run_id = self._create_run(fs).info.run_id metric_name = "test-metric-1" # Check that we get the max of (step, timestamp, value) in that order tuples_to_log = [ (0, 100, 1000), (3, 40, 100), # larger step wins even though it has smaller value (3, 50, 10), # larger timestamp wins even though it has smaller value (3, 50, 20), # tiebreak by max value (3, 50, 20), # duplicate metrics with same (step, timestamp, value) are ok # verify that we can log steps out of order / negative steps (-3, 900, 900), (-1, 800, 800), ] for step, timestamp, value in reversed(tuples_to_log): fs.log_metric(run_id, Metric(metric_name, value, timestamp, step)) metric_history = fs.get_metric_history(run_id, metric_name) logged_tuples = [(m.step, m.timestamp, m.value) for m in metric_history] assert set(logged_tuples) == set(tuples_to_log) run_data = fs.get_run(run_id).data run_metrics = run_data.metrics assert len(run_metrics) == 1 assert run_metrics[metric_name] == 20 metric_obj = run_data._metric_objs[0] assert metric_obj.key == metric_name assert metric_obj.step == 3 assert metric_obj.timestamp == 50 assert metric_obj.value == 20
def test_weird_metric_names(self): WEIRD_METRIC_NAME = "this is/a weird/but valid metric" fs = FileStore(self.test_root) run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0] fs.log_metric(run_id, Metric(WEIRD_METRIC_NAME, 10, 1234, 0)) run = fs.get_run(run_id) assert run.data.metrics[WEIRD_METRIC_NAME] == 10 history = fs.get_metric_history(run_id, WEIRD_METRIC_NAME) assert len(history) == 1 metric = history[0] assert metric.key == WEIRD_METRIC_NAME assert metric.value == 10 assert metric.timestamp == 1234
def test_get_metric_history(self): fs = FileStore(self.test_root) for exp_id in self.experiments: runs = self.exp_data[exp_id]["runs"] for run_uuid in runs: run_info = self.run_data[run_uuid] metrics = run_info.pop("metrics") for metric_name, values in metrics.items(): metric_history = fs.get_metric_history(run_uuid, metric_name) sorted_values = sorted(values, reverse=True) for metric in metric_history: timestamp, metric_value = sorted_values.pop() self.assertEqual(metric.timestamp, timestamp) self.assertEqual(metric.key, metric_name) self.assertEqual(metric.value, metric_value)
def test_log_metrics_params_tags(mlflow_client): experiment_id = mlflow_client.create_experiment('Oh My') created_run = mlflow_client.create_run(experiment_id) run_id = created_run.info.run_uuid mlflow_client.log_metric(run_id, key='metric', value=123.456, timestamp=789, step=2) mlflow_client.log_metric(run_id, key='stepless-metric', value=987.654, timestamp=321) mlflow_client.log_param(run_id, 'param', 'value') mlflow_client.set_tag(run_id, 'taggity', 'do-dah') run = mlflow_client.get_run(run_id) assert run.data.metrics.get('metric') == 123.456 assert run.data.metrics.get('stepless-metric') == 987.654 assert run.data.params.get('param') == 'value' assert run.data.tags.get('taggity') == 'do-dah' # TODO(sid): replace this with mlflow_client.get_metric_history fs = FileStore(server_root_dir) metric_history0 = fs.get_metric_history(run_id, "metric") assert len(metric_history0) == 1 metric0 = metric_history0[0] assert metric0.key == "metric" assert metric0.value == 123.456 assert metric0.timestamp == 789 assert metric0.step == 2 metric_history1 = fs.get_metric_history(run_id, "stepless-metric") assert len(metric_history1) == 1 metric1 = metric_history1[0] assert metric1.key == "stepless-metric" assert metric1.value == 987.654 assert metric1.timestamp == 321 assert metric1.step == 0
def test_log_batch(mlflow_client): experiment_id = mlflow_client.create_experiment('Batch em up') created_run = mlflow_client.create_run(experiment_id) run_id = created_run.info.run_uuid # TODO(sid): pass and assert on step mlflow_client.log_batch(run_id=run_id, metrics=[Metric("metric", 123.456, 789, 0)], params=[Param("param", "value")], tags=[RunTag("taggity", "do-dah")]) run = mlflow_client.get_run(run_id) assert run.data.metrics.get('metric') == 123.456 assert run.data.params.get('param') == 'value' assert run.data.tags.get('taggity') == 'do-dah' # TODO(sid): replace this with mlflow_client.get_metric_history fs = FileStore(server_root_dir) metric_history = fs.get_metric_history(run_id, "metric") assert len(metric_history) == 1 metric = metric_history[0] assert metric.key == "metric" assert metric.value == 123.456 assert metric.timestamp == 789
def test_log_metrics_params_tags(mlflow_client): experiment_id = mlflow_client.create_experiment('Oh My') created_run = mlflow_client.create_run(experiment_id) run_id = created_run.info.run_uuid # TODO(sid): pass and assert on step mlflow_client.log_metric(run_id, key='metric', value=123.456, timestamp=789) mlflow_client.log_param(run_id, 'param', 'value') mlflow_client.set_tag(run_id, 'taggity', 'do-dah') run = mlflow_client.get_run(run_id) assert run.data.metrics.get('metric') == 123.456 assert run.data.params.get('param') == 'value' assert run.data.tags.get('taggity') == 'do-dah' # TODO(sid): replace this with mlflow_client.get_metric_history fs = FileStore(server_root_dir) metric_history = fs.get_metric_history(run_id, "metric") assert len(metric_history) == 1 metric = metric_history[0] assert metric.key == "metric" assert metric.value == 123.456 assert metric.timestamp == 789