def test_bad_experiment_id_recorded_for_run(self):
fs = FileStore(self.test_root)
exp_0 = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
all_runs = self._search(fs, exp_0.experiment_id)
all_run_ids = self.exp_data[exp_0.experiment_id]["runs"]
assert len(all_runs) == len(all_run_ids)
# change experiment pointer in run
bad_run_id = str(self.exp_data[exp_0.experiment_id]['runs'][0])
path = os.path.join(self.test_root, str(exp_0.experiment_id),
bad_run_id)
experiment_data = read_yaml(path, "meta.yaml")
experiment_data["experiment_id"] = 1
write_yaml(path, "meta.yaml", experiment_data, True)
with pytest.raises(MlflowException) as e:
fs.get_run(bad_run_id)
assert e.message.contains("not found")
valid_runs = self._search(fs, exp_0.experiment_id)
assert len(valid_runs) == len(all_runs) - 1
for rid in all_run_ids:
if rid != bad_run_id:
fs.get_run(rid)
def test_delete_tags(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]['runs'][0]
fs.set_tag(run_id, RunTag("tag0", "value0"))
fs.set_tag(run_id, RunTag("tag1", "value1"))
tags = fs.get_run(run_id).data.tags
assert tags["tag0"] == "value0"
assert tags["tag1"] == "value1"
fs.delete_tag(run_id, "tag0")
new_tags = fs.get_run(run_id).data.tags
assert "tag0" not in new_tags.keys()
# test that you cannot delete tags that don't exist.
with pytest.raises(MlflowException):
fs.delete_tag(run_id, "fakeTag")
# test that you cannot delete tags for nonexistent runs
with pytest.raises(MlflowException):
fs.delete_tag("random_id", "tag0")
fs = FileStore(self.test_root)
fs.delete_run(run_id)
# test that you cannot delete tags for deleted runs.
assert fs.get_run(
run_id).info.lifecycle_stage == LifecycleStage.DELETED
with pytest.raises(MlflowException):
fs.delete_tag(run_id, "tag0")
def test_delete_restore_run(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]['runs'][0]
# Should not throw.
assert fs.get_run(run_id).info.lifecycle_stage == 'active'
fs.delete_run(run_id)
assert fs.get_run(run_id).info.lifecycle_stage == 'deleted'
fs.restore_run(run_id)
assert fs.get_run(run_id).info.lifecycle_stage == 'active'
def test_hard_delete_run(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]["runs"][0]
fs._hard_delete_run(run_id)
with self.assertRaises(MlflowException):
fs.get_run(run_id)
with self.assertRaises(MlflowException):
fs.get_all_tags(run_id)
with self.assertRaises(MlflowException):
fs.get_all_metrics(run_id)
with self.assertRaises(MlflowException):
fs.get_all_params(run_id)
def test_unicode_tag(self):
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
value = u"πΌ ππππππππ ππππΆπ ππ½πΆπ πΌ πΆπ ππ
ππ ππ ππππΉ"
fs.set_tag(run_id, RunTag("message", value))
tags = fs.get_run(run_id).data.tags
assert tags["message"] == value
def test_weird_tag_names(self):
WEIRD_TAG_NAME = "this is/a weird/but valid tag"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.set_tag(run_id, RunTag(WEIRD_TAG_NAME, "Muhahaha!"))
run = fs.get_run(run_id)
assert run.data.tags[WEIRD_TAG_NAME] == "Muhahaha!"
def test_log_empty_str(self):
PARAM_NAME = "new param"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_param(run_id, Param(PARAM_NAME, ""))
run = fs.get_run(run_id)
assert run.data.params[PARAM_NAME] == ""
def test_weird_param_names(self):
WEIRD_PARAM_NAME = "this is/a weird/but valid param"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_param(run_id, Param(WEIRD_PARAM_NAME, "Value"))
run = fs.get_run(run_id)
assert run.data.params[WEIRD_PARAM_NAME] == "Value"
def test_log_metric_allows_multiple_values_at_same_step_and_run_data_uses_max_step_value(self):
fs = FileStore(self.test_root)
run_id = self._create_run(fs).info.run_id
metric_name = "test-metric-1"
# Check that we get the max of (step, timestamp, value) in that order
tuples_to_log = [
(0, 100, 1000),
(3, 40, 100), # larger step wins even though it has smaller value
(3, 50, 10), # larger timestamp wins even though it has smaller value
(3, 50, 20), # tiebreak by max value
(3, 50, 20), # duplicate metrics with same (step, timestamp, value) are ok
# verify that we can log steps out of order / negative steps
(-3, 900, 900),
(-1, 800, 800),
]
for step, timestamp, value in reversed(tuples_to_log):
fs.log_metric(run_id, Metric(metric_name, value, timestamp, step))
metric_history = fs.get_metric_history(run_id, metric_name)
logged_tuples = [(m.step, m.timestamp, m.value) for m in metric_history]
assert set(logged_tuples) == set(tuples_to_log)
run_data = fs.get_run(run_id).data
run_metrics = run_data.metrics
assert len(run_metrics) == 1
assert run_metrics[metric_name] == 20
metric_obj = run_data._metric_objs[0]
assert metric_obj.key == metric_name
assert metric_obj.step == 3
assert metric_obj.timestamp == 50
assert metric_obj.value == 20
def test_log_param_with_newline(self):
param_name = "new param"
param_value = "a string\nwith multiple\nlines"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_param(run_id, Param(param_name, param_value))
run = fs.get_run(run_id)
assert run.data.params[param_name] == param_value
def test_get_deleted_run(self):
"""
Getting metrics/tags/params/run info should be allowed on deleted runs.
"""
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]['runs'][0]
fs.delete_run(run_id)
assert fs.get_run(run_id)
def test_set_tags(self):
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.set_tag(run_id, RunTag("tag0", "value0"))
fs.set_tag(run_id, RunTag("tag1", "value1"))
tags = fs.get_run(run_id).data.tags
assert tags["tag0"] == "value0"
assert tags["tag1"] == "value1"
# Can overwrite tags.
fs.set_tag(run_id, RunTag("tag0", "value2"))
tags = fs.get_run(run_id).data.tags
assert tags["tag0"] == "value2"
assert tags["tag1"] == "value1"
# Can set multiline tags.
fs.set_tag(run_id, RunTag("multiline_tag", "value2\nvalue2\nvalue2"))
tags = fs.get_run(run_id).data.tags
assert tags["multiline_tag"] == "value2\nvalue2\nvalue2"
def test_log_param_enforces_value_immutability(self):
param_name = "new param"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_param(run_id, Param(param_name, "value1"))
# Duplicate calls to `log_param` with the same key and value should succeed
fs.log_param(run_id, Param(param_name, "value1"))
with pytest.raises(MlflowException) as exc:
fs.log_param(run_id, Param(param_name, "value2"))
assert exc.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
run = fs.get_run(run_id)
assert run.data.params[param_name] == "value1"
def test_weird_metric_names(self):
WEIRD_METRIC_NAME = "this is/a weird/but valid metric"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_metric(run_id, Metric(WEIRD_METRIC_NAME, 10, 1234, 0))
run = fs.get_run(run_id)
assert run.data.metrics[WEIRD_METRIC_NAME] == 10
history = fs.get_metric_history(run_id, WEIRD_METRIC_NAME)
assert len(history) == 1
metric = history[0]
assert metric.key == WEIRD_METRIC_NAME
assert metric.value == 10
assert metric.timestamp == 1234
def test_malformed_run(self):
fs = FileStore(self.test_root)
exp_0 = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
all_runs = self._search(fs, exp_0.experiment_id)
all_run_ids = self.exp_data[exp_0.experiment_id]["runs"]
assert len(all_runs) == len(all_run_ids)
# delete metadata file.
bad_run_id = self.exp_data[exp_0.experiment_id]["runs"][0]
path = os.path.join(self.test_root, str(exp_0.experiment_id), str(bad_run_id), "meta.yaml")
os.remove(path)
with pytest.raises(MissingConfigException) as e:
fs.get_run(bad_run_id)
assert e.message.contains("does not exist")
valid_runs = self._search(fs, exp_0.experiment_id)
assert len(valid_runs) == len(all_runs) - 1
for rid in all_run_ids:
if rid != bad_run_id:
fs.get_run(rid)
def test_set_deleted_run(self):
"""
Setting metrics/tags/params/updating run info should not be allowed on deleted runs.
"""
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]["runs"][0]
fs.delete_run(run_id)
assert fs.get_run(run_id).info.lifecycle_stage == LifecycleStage.DELETED
with pytest.raises(MlflowException):
fs.set_tag(run_id, RunTag("a", "b"))
with pytest.raises(MlflowException):
fs.log_metric(run_id, Metric("a", 0.0, timestamp=0, step=0))
with pytest.raises(MlflowException):
fs.log_param(run_id, Param("a", "b"))
