def test_log_batch_internal_error(self):
# Verify that internal errors during log_batch result in MlflowExceptions
fs = FileStore(self.test_root)
run = self._create_run(fs)
def _raise_exception_fn(*args, **kwargs): # pylint: disable=unused-argument
raise Exception("Some internal error")
with mock.patch(FILESTORE_PACKAGE + ".FileStore.log_metric") as log_metric_mock, \
mock.patch(FILESTORE_PACKAGE + ".FileStore.log_param") as log_param_mock, \
mock.patch(FILESTORE_PACKAGE + ".FileStore.set_tag") as set_tag_mock:
log_metric_mock.side_effect = _raise_exception_fn
log_param_mock.side_effect = _raise_exception_fn
set_tag_mock.side_effect = _raise_exception_fn
for kwargs in [{
"metrics": [Metric("a", 3, 1, 0)]
}, {
"params": [Param("b", "c")]
}, {
"tags": [RunTag("c", "d")]
}]:
log_batch_kwargs = {"metrics": [], "params": [], "tags": []}
log_batch_kwargs.update(kwargs)
print(log_batch_kwargs)
with self.assertRaises(MlflowException) as e:
fs.log_batch(run.info.run_id, **log_batch_kwargs)
self.assertIn(str(e.exception.message), "Some internal error")
assert e.exception.error_code == ErrorCode.Name(INTERNAL_ERROR)
def test_weird_tag_names(self):
WEIRD_TAG_NAME = "this is/a weird/but valid tag"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.set_tag(run_id, RunTag(WEIRD_TAG_NAME, "Muhahaha!"))
run = fs.get_run(run_id)
assert run.data.tags[WEIRD_TAG_NAME] == "Muhahaha!"
def test_log_batch_tags_idempotency(self):
fs = FileStore(self.test_root)
run = self._create_run(fs)
fs.log_batch(run.info.run_id, metrics=[], params=[], tags=[RunTag("t-key", "t-val")])
fs.log_batch(run.info.run_id, metrics=[], params=[], tags=[RunTag("t-key", "t-val")])
self._verify_logged(fs, run.info.run_id, metrics=[], params=[],
tags=[RunTag("t-key", "t-val")])
def test_unicode_tag(self):
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
value = u"πΌ ππππππππ ππππΆπ ππ½πΆπ πΌ πΆπ ππ
ππ ππ ππππΉ"
fs.set_tag(run_id, RunTag("message", value))
tags = fs.get_run(run_id).data.tags
assert tags["message"] == value
def test_log_batch_same_metric_repeated_single_req(self):
fs = FileStore(self.test_root)
run = self._create_run(fs)
metric0 = Metric(key="metric-key", value=1, timestamp=2, step=0)
metric1 = Metric(key="metric-key", value=2, timestamp=3, step=0)
fs.log_batch(run.info.run_id, params=[], metrics=[metric0, metric1], tags=[])
self._verify_logged(fs, run.info.run_id, params=[], metrics=[metric0, metric1], tags=[])
def test_list_experiments(self):
fs = FileStore(self.test_root)
for exp in fs.list_experiments():
exp_id = exp.experiment_id
self.assertTrue(exp_id in self.experiments)
self.assertEqual(exp.name, self.exp_data[exp_id]["name"])
self.assertEqual(exp.artifact_location, self.exp_data[exp_id]["artifact_location"])
def test_log_batch_params_idempotency(self):
fs = FileStore(self.test_root)
run = self._create_run(fs)
params = [Param("p-key", "p-val")]
fs.log_batch(run.info.run_id, metrics=[], params=params, tags=[])
fs.log_batch(run.info.run_id, metrics=[], params=params, tags=[])
self._verify_logged(fs, run.info.run_id, metrics=[], params=params, tags=[])
def test_log_batch_nonexistent_run(self):
fs = FileStore(self.test_root)
nonexistent_uuid = uuid.uuid4().hex
with self.assertRaises(MlflowException) as e:
fs.log_batch(nonexistent_uuid, [], [], [])
assert e.exception.error_code == ErrorCode.Name(RESOURCE_DOES_NOT_EXIST)
assert ("Run '%s' not found" % nonexistent_uuid) in e.exception.message
def test_weird_param_names(self):
WEIRD_PARAM_NAME = "this is/a weird/but valid param"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_param(run_id, Param(WEIRD_PARAM_NAME, "Value"))
run = fs.get_run(run_id)
assert run.data.params[WEIRD_PARAM_NAME] == "Value"
def test_log_empty_str(self):
PARAM_NAME = "new param"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_param(run_id, Param(PARAM_NAME, ""))
run = fs.get_run(run_id)
assert run.data.params[PARAM_NAME] == ""
def test_log_param_with_newline(self):
param_name = "new param"
param_value = "a string\nwith multiple\nlines"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_param(run_id, Param(param_name, param_value))
run = fs.get_run(run_id)
assert run.data.params[param_name] == param_value
def test_get_deleted_runs(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[0]
run_id = self.exp_data[exp_id]["runs"][0]
fs.delete_run(run_id)
deleted_runs = fs._get_deleted_runs()
assert len(deleted_runs) == 1
assert deleted_runs[0] == run_id
def test_search_with_deterministic_max_results(self):
fs = FileStore(self.test_root)
exp = fs.create_experiment("test_search_with_deterministic_max_results")
# Create 10 runs with the same start_time.
# Sort based on run_id
runs = sorted([fs.create_run(exp, "user", 1000, []).info.run_id for r in range(10)])
for n in [0, 1, 2, 4, 8, 10, 20]:
assert runs[: min(10, n)] == self._search(fs, exp, max_results=n)
def test_get_deleted_run(self):
"""
Getting metrics/tags/params/run info should be allowed on deleted runs.
"""
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]['runs'][0]
fs.delete_run(run_id)
assert fs.get_run(run_id)
def test_log_batch_accepts_empty_payload(self):
fs = FileStore(self.test_root)
run = self._create_run(fs)
fs.log_batch(run.info.run_id, metrics=[], params=[], tags=[])
self._verify_logged(fs,
run.info.run_id,
metrics=[],
params=[],
tags=[])
def test_get_experiment(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
self._verify_experiment(fs, exp_id)
# test that fake experiments dont exist.
# look for random experiment ids between 8000, 15000 since created ones are (100, 2000)
for exp_id in set(random_int(8000, 15000) for x in range(20)):
with self.assertRaises(Exception):
fs.get_experiment(exp_id)
def test_log_batch_allows_tag_overwrite_single_req(self):
fs = FileStore(self.test_root)
run = self._create_run(fs)
tags = [RunTag("t-key", "val"), RunTag("t-key", "newval")]
fs.log_batch(run.info.run_id, metrics=[], params=[], tags=tags)
self._verify_logged(fs,
run.info.run_id,
metrics=[],
params=[],
tags=[tags[-1]])
def test_create_experiment(self):
fs = FileStore(self.test_root)
# Error cases
with self.assertRaises(Exception):
fs.create_experiment(None)
with self.assertRaises(Exception):
fs.create_experiment("")
exp_id_ints = (int(exp_id) for exp_id in self.experiments)
next_id = str(max(exp_id_ints) + 1)
name = random_str(25) # since existing experiments are 10 chars long
created_id = fs.create_experiment(name)
# test that newly created experiment matches expected id
self.assertEqual(created_id, next_id)
# get the new experiment (by id) and verify (by name)
exp1 = fs.get_experiment(created_id)
self.assertEqual(exp1.name, name)
self.assertEqual(
exp1.artifact_location,
path_to_local_file_uri(posixpath.join(self.test_root, created_id)))
# get the new experiment (by name) and verify (by id)
exp2 = fs.get_experiment_by_name(name)
self.assertEqual(exp2.experiment_id, created_id)
def test_log_batch(self):
fs = FileStore(self.test_root)
run = fs.create_run(
experiment_id=FileStore.DEFAULT_EXPERIMENT_ID, user_id='user', start_time=0, tags=[])
run_id = run.info.run_id
metric_entities = [Metric("m1", 0.87, 12345, 0), Metric("m2", 0.49, 12345, 0)]
param_entities = [Param("p1", "p1val"), Param("p2", "p2val")]
tag_entities = [RunTag("t1", "t1val"), RunTag("t2", "t2val")]
fs.log_batch(
run_id=run_id, metrics=metric_entities, params=param_entities, tags=tag_entities)
self._verify_logged(fs, run_id, metric_entities, param_entities, tag_entities)
def test_create_run_appends_to_artifact_uri_path_correctly(self):
cases = [
("path/to/local/folder", "path/to/local/folder/{e}/{r}/artifacts"),
("/path/to/local/folder",
"/path/to/local/folder/{e}/{r}/artifacts"),
("#path/to/local/folder?",
"#path/to/local/folder?/{e}/{r}/artifacts"),
("file:path/to/local/folder",
"file:path/to/local/folder/{e}/{r}/artifacts"),
("file:///path/to/local/folder",
"file:///path/to/local/folder/{e}/{r}/artifacts"),
("file:path/to/local/folder?param=value",
"file:path/to/local/folder/{e}/{r}/artifacts?param=value"),
("file:///path/to/local/folder",
"file:///path/to/local/folder/{e}/{r}/artifacts"),
(
"file:///path/to/local/folder?param=value#fragment",
"file:///path/to/local/folder/{e}/{r}/artifacts?param=value#fragment",
),
("s3://bucket/path/to/root",
"s3://bucket/path/to/root/{e}/{r}/artifacts"),
(
"s3://bucket/path/to/root?creds=mycreds",
"s3://bucket/path/to/root/{e}/{r}/artifacts?creds=mycreds",
),
(
"dbscheme+driver://root@host/dbname?creds=mycreds#myfragment",
"dbscheme+driver://root@host/dbname/{e}/{r}/artifacts?creds=mycreds#myfragment",
),
(
"dbscheme+driver://root:[email protected]?creds=mycreds#myfragment",
"dbscheme+driver://root:[email protected]/{e}/{r}/artifacts"
"?creds=mycreds#myfragment",
),
(
"dbscheme+driver://root:[email protected]/mydb?creds=mycreds#myfragment",
"dbscheme+driver://root:[email protected]/mydb/{e}/{r}/artifacts"
"?creds=mycreds#myfragment",
),
]
for artifact_root_uri, expected_artifact_uri_format in cases:
with TempDir() as tmp:
fs = FileStore(tmp.path(), artifact_root_uri)
exp_id = fs.create_experiment("exp")
run = fs.create_run(experiment_id=exp_id,
user_id='user',
start_time=0,
tags=[])
self.assertEqual(
run.info.artifact_uri,
expected_artifact_uri_format.format(e=exp_id,
r=run.info.run_id))
def test_get_all_metrics(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
runs = self.exp_data[exp_id]["runs"]
for run_id in runs:
run_info = self.run_data[run_id]
metrics = fs.get_all_metrics(run_id)
metrics_dict = run_info.pop("metrics")
for metric in metrics:
expected_timestamp, expected_value = max(metrics_dict[metric.key])
self.assertEqual(metric.timestamp, expected_timestamp)
self.assertEqual(metric.value, expected_value)
def test_list_run_infos(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
run_infos = fs.list_run_infos(exp_id, run_view_type=ViewType.ALL)
for run_info in run_infos:
run_id = run_info.run_id
dict_run_info = self.run_data[run_id]
dict_run_info.pop("metrics")
dict_run_info.pop("params")
dict_run_info.pop("tags")
dict_run_info["lifecycle_stage"] = LifecycleStage.ACTIVE
dict_run_info["status"] = RunStatus.to_string(dict_run_info["status"])
self.assertEqual(dict_run_info, dict(run_info))
def test_delete_restore_run(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]['runs'][0]
# Should not throw.
assert fs.get_run(run_id).info.lifecycle_stage == 'active'
fs.delete_run(run_id)
assert fs.get_run(run_id).info.lifecycle_stage == 'deleted'
fs.restore_run(run_id)
assert fs.get_run(run_id).info.lifecycle_stage == 'active'
def test_get_experiment_by_name(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
name = self.exp_data[exp_id]["name"]
exp = fs.get_experiment_by_name(name)
self.assertEqual(exp.experiment_id, exp_id)
self.assertEqual(exp.name, self.exp_data[exp_id]["name"])
self.assertEqual(exp.artifact_location, self.exp_data[exp_id]["artifact_location"])
# test that fake experiments dont exist.
# look up experiments with names of length 15 since created ones are of length 10
for exp_names in set(random_str(15) for x in range(20)):
exp = fs.get_experiment_by_name(exp_names)
self.assertIsNone(exp)
def test_get_metric_history(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
runs = self.exp_data[exp_id]["runs"]
for run_id in runs:
run_info = self.run_data[run_id]
metrics = run_info.pop("metrics")
for metric_name, values in metrics.items():
metric_history = fs.get_metric_history(run_id, metric_name)
sorted_values = sorted(values, reverse=True)
for metric in metric_history:
timestamp, metric_value = sorted_values.pop()
self.assertEqual(metric.timestamp, timestamp)
self.assertEqual(metric.key, metric_name)
self.assertEqual(metric.value, metric_value)
def test_log_param_enforces_value_immutability(self):
param_name = "new param"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_param(run_id, Param(param_name, "value1"))
# Duplicate calls to `log_param` with the same key and value should succeed
fs.log_param(run_id, Param(param_name, "value1"))
with pytest.raises(MlflowException) as exc:
fs.log_param(run_id, Param(param_name, "value2"))
assert exc.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
run = fs.get_run(run_id)
assert run.data.params[param_name] == "value1"
def test_get_host_creds_from_default_store_file_store():
with mock.patch("mlflow.tracking._tracking_service.utils._get_store"
) as get_store_mock:
get_store_mock.return_value = FileStore()
with pytest.raises(MlflowException,
match="Failed to get credentials for DBFS"):
_get_host_creds_from_default_store()
def test_search_with_max_results(self):
fs = FileStore(self.test_root)
exp = fs.create_experiment("search_with_max_results")
runs = [fs.create_run(exp, "user", r, []).info.run_id for r in range(10)]
runs.reverse()
print(runs)
print(self._search(fs, exp))
assert runs[:10] == self._search(fs, exp)
for n in [0, 1, 2, 4, 8, 10, 20, 50, 100, 500, 1000, 1200, 2000]:
assert runs[: min(1200, n)] == self._search(fs, exp, max_results=n)
with self.assertRaises(MlflowException) as e:
self._search(fs, exp, None, max_results=int(1e10))
self.assertIn("Invalid value for request parameter max_results. It ", e.exception.message)
def test_search_runs(self):
# replace with test with code is implemented
fs = FileStore(self.test_root)
# Expect 2 runs for each experiment
assert len(self._search(fs, self.experiments[0], run_view_type=ViewType.ACTIVE_ONLY)) == 2
assert len(self._search(fs, self.experiments[0])) == 2
assert len(self._search(fs, self.experiments[0], run_view_type=ViewType.DELETED_ONLY)) == 0
def test_get_experiment_int_experiment_id_backcompat(self):
fs = FileStore(self.test_root)
exp_id = FileStore.DEFAULT_EXPERIMENT_ID
root_dir = os.path.join(self.test_root, exp_id)
with safe_edit_yaml(root_dir, "meta.yaml",
self._experiment_id_edit_func):
self._verify_experiment(fs, exp_id)
