def test_log_metric_allows_multiple_values_at_same_step_and_run_data_uses_max_step_value(self): fs = FileStore(self.test_root) run_id = self._create_run(fs).info.run_id metric_name = "test-metric-1" # Check that we get the max of (step, timestamp, value) in that order tuples_to_log = [ (0, 100, 1000), (3, 40, 100), # larger step wins even though it has smaller value (3, 50, 10), # larger timestamp wins even though it has smaller value (3, 50, 20), # tiebreak by max value (3, 50, 20), # duplicate metrics with same (step, timestamp, value) are ok # verify that we can log steps out of order / negative steps (-3, 900, 900), (-1, 800, 800), ] for step, timestamp, value in reversed(tuples_to_log): fs.log_metric(run_id, Metric(metric_name, value, timestamp, step)) metric_history = fs.get_metric_history(run_id, metric_name) logged_tuples = [(m.step, m.timestamp, m.value) for m in metric_history] assert set(logged_tuples) == set(tuples_to_log) run_data = fs.get_run(run_id).data run_metrics = run_data.metrics assert len(run_metrics) == 1 assert run_metrics[metric_name] == 20 metric_obj = run_data._metric_objs[0] assert metric_obj.key == metric_name assert metric_obj.step == 3 assert metric_obj.timestamp == 50 assert metric_obj.value == 20
def test_weird_metric_names(self): WEIRD_METRIC_NAME = "this is/a weird/but valid metric" fs = FileStore(self.test_root) run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0] fs.log_metric(run_id, Metric(WEIRD_METRIC_NAME, 10, 1234, 0)) run = fs.get_run(run_id) assert run.data.metrics[WEIRD_METRIC_NAME] == 10 history = fs.get_metric_history(run_id, WEIRD_METRIC_NAME) assert len(history) == 1 metric = history[0] assert metric.key == WEIRD_METRIC_NAME assert metric.value == 10 assert metric.timestamp == 1234
def test_get_metric_history(self): fs = FileStore(self.test_root) for exp_id in self.experiments: runs = self.exp_data[exp_id]["runs"] for run_id in runs: run_info = self.run_data[run_id] metrics = run_info.pop("metrics") for metric_name, values in metrics.items(): metric_history = fs.get_metric_history(run_id, metric_name) sorted_values = sorted(values, reverse=True) for metric in metric_history: timestamp, metric_value = sorted_values.pop() self.assertEqual(metric.timestamp, timestamp) self.assertEqual(metric.key, metric_name) self.assertEqual(metric.value, metric_value)