def test_set_tags(self): fs = FileStore(self.test_root) run_uuid = self.exp_data[0]["runs"][0] fs.set_tag(run_uuid, RunTag("tag0", "value0")) fs.set_tag(run_uuid, RunTag("tag1", "value1")) tags = [(t.key, t.value) for t in fs.get_run(run_uuid).data.tags] assert set(tags) == { ("tag0", "value0"), ("tag1", "value1"), } # Can overwrite tags. fs.set_tag(run_uuid, RunTag("tag0", "value2")) tags = [(t.key, t.value) for t in fs.get_run(run_uuid).data.tags] assert set(tags) == { ("tag0", "value2"), ("tag1", "value1"), } # Can set multiline tags. fs.set_tag(run_uuid, RunTag("multiline_tag", "value2\nvalue2\nvalue2")) tags = [(t.key, t.value) for t in fs.get_run(run_uuid).data.tags] assert set(tags) == { ("tag0", "value2"), ("tag1", "value1"), ("multiline_tag", "value2\nvalue2\nvalue2"), }
def test_malformed_run(self): fs = FileStore(self.test_root) exp_0 = fs.get_experiment(Experiment.DEFAULT_EXPERIMENT_ID) all_runs = fs.search_runs([exp_0.experiment_id], [], run_view_type=ViewType.ALL) all_run_ids = self.exp_data[exp_0.experiment_id]["runs"] assert len(all_runs) == len(all_run_ids) # delete metadata file. bad_run_id = self.exp_data[exp_0.experiment_id]['runs'][0] path = os.path.join(self.test_root, str(exp_0.experiment_id), str(bad_run_id), "meta.yaml") os.remove(path) with pytest.raises(MissingConfigException) as e: fs.get_run(bad_run_id) assert e.message.contains("does not exist") valid_runs = fs.search_runs([exp_0.experiment_id], [], run_view_type=ViewType.ALL) assert len(valid_runs) == len(all_runs) - 1 for rid in all_run_ids: if rid != bad_run_id: fs.get_run(rid)
def test_delete_tags(self): fs = FileStore(self.test_root) exp_id = self.experiments[random_int(0, len(self.experiments) - 1)] run_id = self.exp_data[exp_id]['runs'][0] fs.set_tag(run_id, RunTag("tag0", "value0")) fs.set_tag(run_id, RunTag("tag1", "value1")) tags = fs.get_run(run_id).data.tags assert tags["tag0"] == "value0" assert tags["tag1"] == "value1" fs.delete_tag(run_id, "tag0") new_tags = fs.get_run(run_id).data.tags assert "tag0" not in new_tags.keys() # test that you cannot delete tags that don't exist. with pytest.raises(MlflowException): fs.delete_tag(run_id, "fakeTag") # test that you cannot delete tags for nonexistent runs with pytest.raises(MlflowException): fs.delete_tag("random_id", "tag0") fs = FileStore(self.test_root) fs.delete_run(run_id) # test that you cannot delete tags for deleted runs. assert fs.get_run( run_id).info.lifecycle_stage == LifecycleStage.DELETED with pytest.raises(MlflowException): fs.delete_tag(run_id, "tag0")
def test_bad_experiment_id_recorded_for_run(self): fs = FileStore(self.test_root) exp_0 = fs.get_experiment(Experiment.DEFAULT_EXPERIMENT_ID) all_runs = fs.search_runs([exp_0.experiment_id], [], run_view_type=ViewType.ALL) all_run_ids = self.exp_data[exp_0.experiment_id]["runs"] assert len(all_runs) == len(all_run_ids) # change experiment pointer in run bad_run_id = str(self.exp_data[exp_0.experiment_id]['runs'][0]) path = os.path.join(self.test_root, str(exp_0.experiment_id), bad_run_id) experiment_data = read_yaml(path, "meta.yaml") experiment_data["experiment_id"] = 1 write_yaml(path, "meta.yaml", experiment_data, True) with pytest.raises(MlflowException) as e: fs.get_run(bad_run_id) assert e.message.contains("not found") valid_runs = fs.search_runs([exp_0.experiment_id], [], run_view_type=ViewType.ALL) assert len(valid_runs) == len(all_runs) - 1 for rid in all_run_ids: if rid != bad_run_id: fs.get_run(rid)
def test_delete_restore_run(self): fs = FileStore(self.test_root) exp_id = self.experiments[random_int(0, len(self.experiments) - 1)] run_id = self.exp_data[exp_id]['runs'][0] # Should not throw. assert fs.get_run(run_id).info.lifecycle_stage == 'active' fs.delete_run(run_id) assert fs.get_run(run_id).info.lifecycle_stage == 'deleted' fs.restore_run(run_id) assert fs.get_run(run_id).info.lifecycle_stage == 'active'
def test_log_metric_allows_multiple_values_at_same_step_and_run_data_uses_max_step_value(self): fs = FileStore(self.test_root) run_id = self._create_run(fs).info.run_id metric_name = "test-metric-1" # Check that we get the max of (step, timestamp, value) in that order tuples_to_log = [ (0, 100, 1000), (3, 40, 100), # larger step wins even though it has smaller value (3, 50, 10), # larger timestamp wins even though it has smaller value (3, 50, 20), # tiebreak by max value (3, 50, 20), # duplicate metrics with same (step, timestamp, value) are ok # verify that we can log steps out of order / negative steps (-3, 900, 900), (-1, 800, 800), ] for step, timestamp, value in reversed(tuples_to_log): fs.log_metric(run_id, Metric(metric_name, value, timestamp, step)) metric_history = fs.get_metric_history(run_id, metric_name) logged_tuples = [(m.step, m.timestamp, m.value) for m in metric_history] assert set(logged_tuples) == set(tuples_to_log) run_data = fs.get_run(run_id).data run_metrics = run_data.metrics assert len(run_metrics) == 1 assert run_metrics[metric_name] == 20 metric_obj = run_data._metric_objs[0] assert metric_obj.key == metric_name assert metric_obj.step == 3 assert metric_obj.timestamp == 50 assert metric_obj.value == 20
def test_log_metric_allows_multiple_values_at_same_ts_and_run_data_uses_max_ts_value( self): fs = FileStore(self.test_root) run_uuid = self._create_run(fs).info.run_uuid metric_name = "test-metric-1" timestamp_values_mapping = { 1000: [float(i) for i in range(-20, 20)], 2000: [float(i) for i in range(-10, 10)], } logged_values = [] for timestamp, value_range in timestamp_values_mapping.items(): for value in reversed(value_range): fs.log_metric(run_uuid, Metric(metric_name, value, timestamp)) logged_values.append(value) six.assertCountEqual(self, [ metric.value for metric in fs.get_metric_history(run_uuid, metric_name) ], logged_values) run_metrics = fs.get_run(run_uuid).data.metrics assert len(run_metrics) == 1 logged_metric_val = run_metrics[metric_name] max_timestamp = max(timestamp_values_mapping) assert logged_metric_val == max( timestamp_values_mapping[max_timestamp])
def test_log_batch(self): fs = FileStore(self.test_root) run = fs.create_run(experiment_id=Experiment.DEFAULT_EXPERIMENT_ID, user_id='user', run_name=None, source_type='source_type', source_name='source_name', entry_point_name='entry_point_name', start_time=0, source_version=None, tags=[], parent_run_id=None) run_uuid = run.info.run_uuid metric_entities = [ Metric("m1", 0.87, 12345), Metric("m2", 0.49, 12345) ] param_entities = [Param("p1", "p1val"), Param("p2", "p2val")] tag_entities = [RunTag("t1", "t1val"), RunTag("t2", "t2val")] fs.log_batch(run_id=run_uuid, metrics=metric_entities, params=param_entities, tags=tag_entities) run = fs.get_run(run_uuid) tags = [(t.key, t.value) for t in run.data.tags] metrics = [(m.key, m.value, m.timestamp) for m in run.data.metrics] params = [(p.key, p.value) for p in run.data.params] assert set(tags) == set([("t1", "t1val"), ("t2", "t2val")]) assert set(metrics) == set([("m1", 0.87, 12345), ("m2", 0.49, 12345)]) assert set(params) == set([("p1", "p1val"), ("p2", "p2val")])
def test_unicode_tag(self): fs = FileStore(self.test_root) run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0] value = u"𝐼 𝓈𝑜𝓁𝑒𝓂𝓃𝓁𝓎 𝓈𝓌𝑒𝒶𝓇 𝓉𝒽𝒶𝓉 𝐼 𝒶𝓂 𝓊𝓅 𝓉𝑜 𝓃𝑜 𝑔𝑜𝑜𝒹" fs.set_tag(run_id, RunTag("message", value)) tags = fs.get_run(run_id).data.tags assert tags["message"] == value
def test_weird_tag_names(self): WEIRD_TAG_NAME = "this is/a weird/but valid tag" fs = FileStore(self.test_root) run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0] fs.set_tag(run_id, RunTag(WEIRD_TAG_NAME, "Muhahaha!")) run = fs.get_run(run_id) assert run.data.tags[WEIRD_TAG_NAME] == "Muhahaha!"
def test_log_empty_str(self): PARAM_NAME = "new param" fs = FileStore(self.test_root) run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0] fs.log_param(run_id, Param(PARAM_NAME, "")) run = fs.get_run(run_id) assert run.data.params[PARAM_NAME] == ""
def test_weird_param_names(self): WEIRD_PARAM_NAME = "this is/a weird/but valid param" fs = FileStore(self.test_root) run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0] fs.log_param(run_id, Param(WEIRD_PARAM_NAME, "Value")) run = fs.get_run(run_id) assert run.data.params[WEIRD_PARAM_NAME] == "Value"
def test_run(tmpdir, tracking_uri_mock, use_start_run): # pylint: disable=unused-argument submitted_run = mlflow.projects.run( TEST_PROJECT_DIR, entry_point="test_tracking", parameters={"use_start_run": use_start_run}, use_conda=False, experiment_id=0) assert submitted_run.run_id is not None # Blocking runs should be finished when they return validate_exit_status(submitted_run.get_status(), RunStatus.FINISHED) # Test that we can call wait() on a synchronous run & that the run has the correct # status after calling wait(). submitted_run.wait() validate_exit_status(submitted_run.get_status(), RunStatus.FINISHED) # Validate run contents in the FileStore run_uuid = submitted_run.run_id store = FileStore(tmpdir.strpath) run_infos = store.list_run_infos(experiment_id=0, run_view_type=ViewType.ACTIVE_ONLY) assert len(run_infos) == 1 store_run_uuid = run_infos[0].run_uuid assert run_uuid == store_run_uuid run = store.get_run(run_uuid) expected_params = {"use_start_run": use_start_run} assert run.info.status == RunStatus.FINISHED assert len(run.data.params) == len(expected_params) for param in run.data.params: assert param.value == expected_params[param.key] expected_metrics = {"some_key": 3} assert len(run.data.metrics) == len(expected_metrics) for metric in run.data.metrics: assert metric.value == expected_metrics[metric.key]
def test_weird_tag_names(self): WEIRD_TAG_NAME = "this is/a weird/but valid tag" fs = FileStore(self.test_root) run_uuid = self.exp_data[0]["runs"][0] fs.set_tag(run_uuid, RunTag(WEIRD_TAG_NAME, "Muhahaha!")) tag = fs.get_run(run_uuid).data.tags[0] assert tag.key == WEIRD_TAG_NAME assert tag.value == "Muhahaha!"
def test_unicode_tag(self): fs = FileStore(self.test_root) run_uuid = self.exp_data[0]["runs"][0] value = u"𝐼 𝓈𝑜𝓁𝑒𝓂𝓃𝓁𝓎 𝓈𝓌𝑒𝒶𝓇 𝓉𝒽𝒶𝓉 𝐼 𝒶𝓂 𝓊𝓅 𝓉𝑜 𝓃𝑜 𝑔𝑜𝑜𝒹" fs.set_tag(run_uuid, RunTag("message", value)) tag = fs.get_run(run_uuid).data.tags[0] assert tag.key == "message" assert tag.value == value
def test_get_deleted_run(self): """ Getting metrics/tags/params/run info should be allowed on deleted runs. """ fs = FileStore(self.test_root) exp_id = self.experiments[random_int(0, len(self.experiments) - 1)] run_id = self.exp_data[exp_id]['runs'][0] fs.delete_run(run_id) assert fs.get_run(run_id)
def test_create_run_with_parent_id(self): fs = FileStore(self.test_root) exp_id = self.experiments[random_int(0, len(self.experiments) - 1)] run = fs.create_run(exp_id, 'user', 'name', 'source_type', 'source_name', 'entry_point_name', 0, None, [], 'test_parent_run_id') assert fs.get_run( run.info.run_uuid ).data.tags[MLFLOW_PARENT_RUN_ID] == 'test_parent_run_id'
def test_set_tags(self): fs = FileStore(self.test_root) run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0] fs.set_tag(run_id, RunTag("tag0", "value0")) fs.set_tag(run_id, RunTag("tag1", "value1")) tags = fs.get_run(run_id).data.tags assert tags["tag0"] == "value0" assert tags["tag1"] == "value1" # Can overwrite tags. fs.set_tag(run_id, RunTag("tag0", "value2")) tags = fs.get_run(run_id).data.tags assert tags["tag0"] == "value2" assert tags["tag1"] == "value1" # Can set multiline tags. fs.set_tag(run_id, RunTag("multiline_tag", "value2\nvalue2\nvalue2")) tags = fs.get_run(run_id).data.tags assert tags["multiline_tag"] == "value2\nvalue2\nvalue2"
def test_get_run(self): fs = FileStore(self.test_root) for exp_id in self.experiments: runs = self.exp_data[exp_id]["runs"] for run_uuid in runs: run = fs.get_run(run_uuid) run_info = self.run_data[run_uuid] run_info.pop("metrics") run_info.pop("params") self.assertEqual(run_info, dict(run.info))
def test_log_empty_str(self): PARAM_NAME = "new param" fs = FileStore(self.test_root) run_uuid = self.exp_data[0]["runs"][0] fs.log_param(run_uuid, Param(PARAM_NAME, "")) run = fs.get_run(run_uuid) my_params = [p for p in run.data.params if p.key == PARAM_NAME] assert len(my_params) == 1 param = my_params[0] assert param.key == PARAM_NAME assert param.value == ""
def test_weird_param_names(self): WEIRD_PARAM_NAME = "this is/a weird/but valid param" fs = FileStore(self.test_root) run_uuid = self.exp_data[0]["runs"][0] fs.log_param(run_uuid, Param(WEIRD_PARAM_NAME, "Value")) run = fs.get_run(run_uuid) my_params = [p for p in run.data.params if p.key == WEIRD_PARAM_NAME] assert len(my_params) == 1 param = my_params[0] assert param.key == WEIRD_PARAM_NAME assert param.value == "Value"
def test_get_run(self): fs = FileStore(self.test_root) for exp_id in self.experiments: runs = self.exp_data[exp_id]["runs"] for run_uuid in runs: run = fs.get_run(run_uuid) run_info = self.run_data[run_uuid] run_info.pop("metrics") run_info.pop("params") run_info.pop("tags") run_info['lifecycle_stage'] = RunInfo.ACTIVE_LIFECYCLE self.assertEqual(run_info, dict(run.info))
def test_weird_metric_names(self): WEIRD_METRIC_NAME = "this is/a weird/but valid metric" fs = FileStore(self.test_root) run_uuid = self.exp_data[0]["runs"][0] fs.log_metric(run_uuid, Metric(WEIRD_METRIC_NAME, 10, 1234)) run = fs.get_run(run_uuid) my_metrics = [m for m in run.data.metrics if m.key == WEIRD_METRIC_NAME] assert len(my_metrics) == 1 metric = my_metrics[0] assert metric.key == WEIRD_METRIC_NAME assert metric.value == 10 assert metric.timestamp == 1234
def test_weird_metric_names(self): WEIRD_METRIC_NAME = "this is/a weird/but valid metric" fs = FileStore(self.test_root) run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0] fs.log_metric(run_id, Metric(WEIRD_METRIC_NAME, 10, 1234, 0)) run = fs.get_run(run_id) assert run.data.metrics[WEIRD_METRIC_NAME] == 10 history = fs.get_metric_history(run_id, WEIRD_METRIC_NAME) assert len(history) == 1 metric = history[0] assert metric.key == WEIRD_METRIC_NAME assert metric.value == 10 assert metric.timestamp == 1234
def test_run_local_git_repo( tmpdir, local_git_repo, local_git_repo_uri, tracking_uri_mock, # pylint: disable=unused-argument use_start_run, version): if version is not None: uri = local_git_repo_uri + "#" + TEST_PROJECT_NAME else: uri = os.path.join("%s/" % local_git_repo, TEST_PROJECT_NAME) if version == "git-commit": version = _get_version_local_git_repo(local_git_repo) submitted_run = mlflow.projects.run( uri, entry_point="test_tracking", version=version, parameters={"use_start_run": use_start_run}, use_conda=False, experiment_id=0) # Blocking runs should be finished when they return validate_exit_status(submitted_run.get_status(), RunStatus.FINISHED) # Test that we can call wait() on a synchronous run & that the run has the correct # status after calling wait(). submitted_run.wait() validate_exit_status(submitted_run.get_status(), RunStatus.FINISHED) # Validate run contents in the FileStore run_uuid = submitted_run.run_id store = FileStore(tmpdir.strpath) run_infos = store.list_run_infos(experiment_id=0, run_view_type=ViewType.ACTIVE_ONLY) assert "file:" in run_infos[0].source_name assert len(run_infos) == 1 store_run_uuid = run_infos[0].run_uuid assert run_uuid == store_run_uuid run = store.get_run(run_uuid) expected_params = {"use_start_run": use_start_run} assert run.info.status == RunStatus.FINISHED assert len(run.data.params) == len(expected_params) for param in run.data.params: assert param.value == expected_params[param.key] expected_metrics = {"some_key": 3} assert len(run.data.metrics) == len(expected_metrics) for metric in run.data.metrics: assert metric.value == expected_metrics[metric.key] # Validate the branch name tag is logged if version == "master": expected_tags = {"mlflow.gitBranchName": "master"} for tag in run.data.tags: assert tag.value == expected_tags[tag.key]
def test_log_parameters(): """ Test that we log provided parameters when running a project. """ with TempDir() as tmp, mock.patch("mlflow.tracking.get_tracking_uri") as get_tracking_uri_mock: tmp_dir = tmp.path() get_tracking_uri_mock.return_value = tmp_dir mlflow.projects.run( TEST_PROJECT_DIR, entry_point="greeter", parameters={"name": "friend"}, use_conda=False, experiment_id=0) store = FileStore(tmp_dir) run_uuid = store.list_run_infos(experiment_id=0)[0].run_uuid run = store.get_run(run_uuid) expected_params = {"name": "friend"} assert len(run.data.params) == len(expected_params) for param in run.data.params: assert param.value == expected_params[param.key]
def test_set_deleted_run(self): """ Setting metrics/tags/params/updating run info should not be allowed on deleted runs. """ fs = FileStore(self.test_root) exp_id = self.experiments[random_int(0, len(self.experiments) - 1)] run_id = self.exp_data[exp_id]['runs'][0] fs.delete_run(run_id) assert fs.get_run(run_id).info.lifecycle_stage == LifecycleStage.DELETED with pytest.raises(MlflowException): fs.set_tag(run_id, RunTag('a', 'b')) with pytest.raises(MlflowException): fs.log_metric(run_id, Metric('a', 0.0, timestamp=0)) with pytest.raises(MlflowException): fs.log_param(run_id, Param('a', 'b'))
def test_run(): for use_start_run in map(str, [0, 1]): with TempDir() as tmp, mock.patch("mlflow.tracking.get_tracking_uri")\ as get_tracking_uri_mock: tmp_dir = tmp.path() get_tracking_uri_mock.return_value = tmp_dir submitted_run = mlflow.projects.run( TEST_PROJECT_DIR, entry_point="test_tracking", parameters={"use_start_run": use_start_run}, use_conda=False, experiment_id=0) # Blocking runs should be finished when they return validate_exit_status(submitted_run.get_status(), RunStatus.FINISHED) # Test that we can call wait() on a synchronous run & that the run has the correct # status after calling wait(). submitted_run.wait() validate_exit_status(submitted_run.get_status(), RunStatus.FINISHED) # Validate run contents in the FileStore run_uuid = submitted_run.run_id store = FileStore(tmp_dir) run_infos = store.list_run_infos(experiment_id=0) assert len(run_infos) == 1 store_run_uuid = run_infos[0].run_uuid assert run_uuid == store_run_uuid run = store.get_run(run_uuid) expected_params = {"use_start_run": use_start_run} assert run.info.status == RunStatus.FINISHED assert len(run.data.params) == len(expected_params) for param in run.data.params: assert param.value == expected_params[param.key] expected_metrics = {"some_key": 3} for metric in run.data.metrics: assert metric.value == expected_metrics[metric.key]