def _run_factory(self, name='test', experiment_id=None, config=None): m1 = models.SqlMetric(key='accuracy', value=0.89) m2 = models.SqlMetric(key='recal', value=0.89) p1 = models.SqlParam(key='loss', value='test param') p2 = models.SqlParam(key='blue', value='test param') if not experiment_id: experiment = self._experiment_factory('test exp') experiment_id = experiment.experiment_id config = { 'experiment_id': experiment_id, 'name': name, 'user_id': 'Anderson', 'run_uuid': uuid.uuid4().hex, 'status': entities.RunStatus.to_string(entities.RunStatus.SCHEDULED), 'source_type': entities.SourceType.to_string(entities.SourceType.NOTEBOOK), 'source_name': 'Python application', 'entry_point_name': 'main.py', 'start_time': int(time.time()), 'end_time': int(time.time()), 'source_version': mlflow.__version__, 'lifecycle_stage': entities.RunInfo.ACTIVE_LIFECYCLE, 'artifact_uri': '//' } run = models.SqlRun(**config) run.params.append(p1) run.params.append(p2) run.metrics.append(m1) run.metrics.append(m2) self.session.add(run) return run
def test_get_metric_history(self): run = self._run_factory() self.session.commit() key = 'test' expected = [ models.SqlMetric(key=key, value=0.6, timestamp=1).to_mlflow_entity(), models.SqlMetric(key=key, value=0.7, timestamp=2).to_mlflow_entity() ] for metric in expected: self.store.log_metric(run.run_uuid, metric) actual = self.store.get_metric_history(run.run_uuid, key) self.assertSequenceEqual([(m.key, m.value, m.timestamp) for m in expected], [(m.key, m.value, m.timestamp) for m in actual])
def test_get_metric_history(self): run = self._run_factory('test') self.session.commit() key = 'test' expected = [ models.SqlMetric(key=key, value=0.6, timestamp=1).to_mlflow_entity(), models.SqlMetric(key=key, value=0.7, timestamp=2).to_mlflow_entity() ] for metric in expected: self.store.log_metric(run.run_uuid, metric) actual = self.store.get_metric_history(run.run_uuid, key) self.assertEqual(len(expected), len(actual))
def test_metric_model(self): run_data = models.SqlMetric(run_uuid='testuid', key='accuracy', value=0.89) self.session.add(run_data) self.session.commit() metrics = self.session.query(models.SqlMetric).all() self.assertEqual(len(metrics), 1) actual = metrics[0].to_mlflow_entity() self.assertEqual(actual.value, run_data.value) self.assertEqual(actual.key, run_data.key)
def test_run_data_model(self): m1 = models.SqlMetric(key='accuracy', value=0.89) m2 = models.SqlMetric(key='recal', value=0.89) p1 = models.SqlParam(key='loss', value='test param') p2 = models.SqlParam(key='blue', value='test param') self.session.add_all([m1, m2, p1, p2]) run_data = models.SqlRun(run_uuid=uuid.uuid4().hex) run_data.params.append(p1) run_data.params.append(p2) run_data.metrics.append(m1) run_data.metrics.append(m2) self.session.add(run_data) self.session.commit() run_datums = self.session.query(models.SqlRun).all() actual = run_datums[0] self.assertEqual(len(run_datums), 1) self.assertEqual(len(actual.params), 2) self.assertEqual(len(actual.metrics), 2)
def test_metric_model(self): # Create a run whose UUID we can reference when creating metric models. # `run_uuid` is a foreign key in the tags table; therefore, in order # to insert a metric with a given run UUID, the UUID must be present in # the runs table run = self._run_factory() with self.store.ManagedSessionMaker() as session: new_metric = models.SqlMetric(run_uuid=run.info.run_uuid, key='accuracy', value=0.89) session.add(new_metric) session.commit() metrics = session.query(models.SqlMetric).all() self.assertEqual(len(metrics), 1) added_metric = metrics[0].to_mlflow_entity() self.assertEqual(added_metric.value, new_metric.value) self.assertEqual(added_metric.key, new_metric.key)