def test_log_metric_allows_multiple_values_at_same_ts_and_run_data_uses_max_ts_value(
self):
fs = FileStore(self.test_root)
run_uuid = self._create_run(fs).info.run_uuid
metric_name = "test-metric-1"
timestamp_values_mapping = {
1000: [float(i) for i in range(-20, 20)],
2000: [float(i) for i in range(-10, 10)],
}
logged_values = []
for timestamp, value_range in timestamp_values_mapping.items():
for value in reversed(value_range):
fs.log_metric(run_uuid, Metric(metric_name, value, timestamp))
logged_values.append(value)
six.assertCountEqual(self, [
metric.value
for metric in fs.get_metric_history(run_uuid, metric_name)
], logged_values)
run_metrics = fs.get_run(run_uuid).data.metrics
assert len(run_metrics) == 1
logged_metric_val = run_metrics[metric_name]
max_timestamp = max(timestamp_values_mapping)
assert logged_metric_val == max(
timestamp_values_mapping[max_timestamp])
def test_log_metric_allows_multiple_values_at_same_step_and_run_data_uses_max_step_value(self):
fs = FileStore(self.test_root)
run_id = self._create_run(fs).info.run_id
metric_name = "test-metric-1"
# Check that we get the max of (step, timestamp, value) in that order
tuples_to_log = [
(0, 100, 1000),
(3, 40, 100), # larger step wins even though it has smaller value
(3, 50, 10), # larger timestamp wins even though it has smaller value
(3, 50, 20), # tiebreak by max value
(3, 50, 20), # duplicate metrics with same (step, timestamp, value) are ok
# verify that we can log steps out of order / negative steps
(-3, 900, 900),
(-1, 800, 800),
]
for step, timestamp, value in reversed(tuples_to_log):
fs.log_metric(run_id, Metric(metric_name, value, timestamp, step))
metric_history = fs.get_metric_history(run_id, metric_name)
logged_tuples = [(m.step, m.timestamp, m.value) for m in metric_history]
assert set(logged_tuples) == set(tuples_to_log)
run_data = fs.get_run(run_id).data
run_metrics = run_data.metrics
assert len(run_metrics) == 1
assert run_metrics[metric_name] == 20
metric_obj = run_data._metric_objs[0]
assert metric_obj.key == metric_name
assert metric_obj.step == 3
assert metric_obj.timestamp == 50
assert metric_obj.value == 20
def test_weird_metric_names(self):
WEIRD_METRIC_NAME = "this is/a weird/but valid metric"
fs = FileStore(self.test_root)
run_uuid = self.exp_data[0]["runs"][0]
fs.log_metric(run_uuid, Metric(WEIRD_METRIC_NAME, 10, 1234))
metric = fs.get_metric(run_uuid, WEIRD_METRIC_NAME)
assert metric.key == WEIRD_METRIC_NAME
assert metric.value == 10
assert metric.timestamp == 1234
def test_weird_metric_names(self):
WEIRD_METRIC_NAME = "this is/a weird/but valid metric"
fs = FileStore(self.test_root)
run_uuid = self.exp_data[0]["runs"][0]
fs.log_metric(run_uuid, Metric(WEIRD_METRIC_NAME, 10, 1234))
run = fs.get_run(run_uuid)
my_metrics = [m for m in run.data.metrics if m.key == WEIRD_METRIC_NAME]
assert len(my_metrics) == 1
metric = my_metrics[0]
assert metric.key == WEIRD_METRIC_NAME
assert metric.value == 10
assert metric.timestamp == 1234
def test_weird_metric_names(self):
WEIRD_METRIC_NAME = "this is/a weird/but valid metric"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_metric(run_id, Metric(WEIRD_METRIC_NAME, 10, 1234, 0))
run = fs.get_run(run_id)
assert run.data.metrics[WEIRD_METRIC_NAME] == 10
history = fs.get_metric_history(run_id, WEIRD_METRIC_NAME)
assert len(history) == 1
metric = history[0]
assert metric.key == WEIRD_METRIC_NAME
assert metric.value == 10
assert metric.timestamp == 1234
def test_set_deleted_run(self):
"""
Setting metrics/tags/params/updating run info should not be allowed on deleted runs.
"""
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]['runs'][0]
fs.delete_run(run_id)
assert fs.get_run(run_id).info.lifecycle_stage == LifecycleStage.DELETED
with pytest.raises(MlflowException):
fs.set_tag(run_id, RunTag('a', 'b'))
with pytest.raises(MlflowException):
fs.log_metric(run_id, Metric('a', 0.0, timestamp=0))
with pytest.raises(MlflowException):
fs.log_param(run_id, Param('a', 'b'))
