def test_correct_filtering(filter_string, matching_runs): runs = [ Run(run_info=RunInfo( run_uuid="hi", run_id="hi", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData( metrics=[Metric("key1", 121, 1, 0)], params=[Param("my_param", "A")], tags=[])), Run(run_info=RunInfo( run_uuid="hi2", run_id="hi2", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FINISHED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData( metrics=[Metric("key1", 123, 1, 0)], params=[Param("my_param", "A")], tags=[RunTag("tag1", "C")])), Run(run_info=RunInfo( run_uuid="hi3", run_id="hi3", experiment_id=1, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData( metrics=[Metric("key1", 125, 1, 0)], params=[Param("my_param", "B")], tags=[RunTag("tag1", "D")])), ] filtered_runs = SearchUtils.filter(runs, filter_string) assert set(filtered_runs) == set([runs[i] for i in matching_runs])
def test_pagination(page_token, max_results, matching_runs, expected_next_page_token): runs = [ Run(run_info=RunInfo( run_uuid="0", run_id="0", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData([], [], [])), Run(run_info=RunInfo( run_uuid="1", run_id="1", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData([], [], [])), Run(run_info=RunInfo( run_uuid="2", run_id="2", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData([], [], [])) ] encoded_page_token = None if page_token: encoded_page_token = base64.b64encode(json.dumps(page_token).encode("utf-8")) paginated_runs, next_page_token = SearchUtils.paginate(runs, encoded_page_token, max_results) paginated_run_indices = [] for run in paginated_runs: for i, r in enumerate(runs): if r == run: paginated_run_indices.append(i) break assert paginated_run_indices == matching_runs decoded_next_page_token = None if next_page_token: decoded_next_page_token = json.loads(base64.b64decode(next_page_token)) assert decoded_next_page_token == expected_next_page_token
def test_correct_sorting(order_bys, matching_runs): runs = [ Run(run_info=RunInfo( run_uuid="9", run_id="9", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData( metrics=[Metric("key1", 121, 1, 0)], params=[Param("my_param", "A")], tags=[])), Run(run_info=RunInfo( run_uuid="8", run_id="8", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FINISHED), start_time=1, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData( metrics=[Metric("key1", 123, 1, 0)], params=[Param("my_param", "A")], tags=[RunTag("tag1", "C")])), Run(run_info=RunInfo( run_uuid="7", run_id="7", experiment_id=1, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=1, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData( metrics=[Metric("key1", 125, 1, 0)], params=[Param("my_param", "B")], tags=[RunTag("tag1", "D")])), ] sorted_runs = SearchUtils.sort(runs, order_bys) sorted_run_indices = [] for run in sorted_runs: for i, r in enumerate(runs): if r == run: sorted_run_indices.append(i) break assert sorted_run_indices == matching_runs
def test_creation_and_hydration(self): rd1, metrics, params = self._create() self._check(rd1, metrics, params) as_dict = {"metrics": metrics, "params": params} self.assertEqual(dict(rd1), as_dict) proto = rd1.to_proto() rd2 = RunData.from_proto(proto) self._check(rd2, metrics, params) rd3 = RunData.from_dictionary(as_dict) self._check(rd3, metrics, params)
def _create(): metrics = [Metric(random_str(10), random_int(0, 1000), int(time.time()) + random_int(-1e4, 1e4))] params = [Param(random_str(10), random_str(random_int(10, 35))) for _ in range(10)] # noqa tags = [RunTag(random_str(10), random_str(random_int(10, 35))) for _ in range(10)] # noqa rd = RunData(metrics=metrics, params=params, tags=tags) return rd, metrics, params, tags
def to_mlflow_entity(self): """ Convert DB model to corresponding MLflow entity. :return: :py:class:`mlflow.entities.Run`. """ run_info = RunInfo( run_uuid=self.run_uuid, run_id=self.run_uuid, experiment_id=str(self.experiment_id), user_id=self.user_id, status=self.status, start_time=self.start_time, end_time=self.end_time, lifecycle_stage=self.lifecycle_stage, artifact_uri=self.artifact_uri, ) run_data = RunData( metrics=[m.to_mlflow_entity() for m in self.latest_metrics], params=[p.to_mlflow_entity() for p in self.params], tags=[t.to_mlflow_entity() for t in self.tags], ) return Run(run_info=run_info, run_data=run_data)
def test_string_repr(self): run_info = RunInfo(run_uuid="hi", run_id="hi", experiment_id=0, name="name", source_type=SourceType.PROJECT, source_name="source-name", entry_point_name="entry-point-name", user_id="user-id", status=RunStatus.FAILED, start_time=0, end_time=1, source_version="version", lifecycle_stage=LifecycleStage.ACTIVE) metrics = [ Metric(key="key-%s" % i, value=i, timestamp=0, step=i) for i in range(3) ] run_data = RunData(metrics=metrics, params=[], tags=[]) run1 = Run(run_info, run_data) expected = ( "<Run: data=<RunData: metrics={'key-0': 0, 'key-1': 1, 'key-2': 2}, " "params={}, tags={}>, info=<RunInfo: artifact_uri=None, end_time=1, " "entry_point_name='entry-point-name', experiment_id=0, " "lifecycle_stage='active', name='name', run_id='hi', run_uuid='hi', " "source_name='source-name', source_type=3, source_version='version', " "start_time=0, status=4, user_id='user-id'>>") assert str(run1) == expected
def mlflow_run( name=RUN_NAME, status="RUNNING", end_time=None, lifecycle_stage=LifecycleStage.ACTIVE, name_tag=RunTag(MLFLOW_RUN_NAME, RUN_NAME), parent_run_id_tag=RunTag(MLFLOW_PARENT_RUN_ID, PARENT_RUN_UUID_HEX_STR), ): tags = [MLFLOW_TAG] if name_tag is not None: tags.append(name_tag) if parent_run_id_tag is not None: tags.append(parent_run_id_tag) data = RunData(params=[MLFLOW_PARAM], metrics=[MLFLOW_METRIC], tags=tags) info = RunInfo( run_uuid=RUN_UUID_HEX_STR, experiment_id=str(EXPERIMENT_ID), user_id="", status=status, start_time=RUN_STARTED_AT_MILLISECONDS, end_time=end_time, lifecycle_stage=lifecycle_stage, artifact_uri=ARTIFACT_LOCATION, run_id=RUN_UUID_HEX_STR, ) return Run(info, data)
def to_mlflow_entity(self): """ Convert DB model to corresponding MLflow entity. :return: :py:class:`mlflow.entities.Run`. """ run_info = RunInfo(run_uuid=self.run_uuid, run_id=self.run_uuid, experiment_id=str(self.experiment_id), user_id=self.user_id, status=self.status, start_time=self.start_time, end_time=self.end_time, lifecycle_stage=self.lifecycle_stage, artifact_uri=self.artifact_uri) # only get latest recorded metrics per key all_metrics = [m.to_mlflow_entity() for m in self.metrics] metrics = {} for m in all_metrics: existing_metric = metrics.get(m.key) if (existing_metric is None)\ or ((m.step, m.timestamp, m.value) >= (existing_metric.step, existing_metric.timestamp, existing_metric.value)): metrics[m.key] = m run_data = RunData(metrics=list(metrics.values()), params=[p.to_mlflow_entity() for p in self.params], tags=[t.to_mlflow_entity() for t in self.tags]) return Run(run_info=run_info, run_data=run_data)
def test_order_by_metric_with_nans_infs_nones(): metric_vals_str = ["nan", "inf", "-inf", "-1000", "0", "1000", "None"] runs = [ Run( run_info=RunInfo( run_id=x, run_uuid=x, experiment_id=0, user_id="user", status=RunStatus.to_string(RunStatus.FINISHED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE, ), run_data=RunData( metrics=[Metric("x", None if x == "None" else float(x), 1, 0) ]), ) for x in metric_vals_str ] sorted_runs_asc = [ x.info.run_id for x in SearchUtils.sort(runs, ["metrics.x asc"]) ] sorted_runs_desc = [ x.info.run_id for x in SearchUtils.sort(runs, ["metrics.x desc"]) ] # asc assert ["-inf", "-1000", "0", "1000", "inf", "nan", "None"] == sorted_runs_asc # desc assert ["inf", "1000", "0", "-1000", "-inf", "nan", "None"] == sorted_runs_desc
def create_run( run_id="", exp_id="", uid="", start=0, end=0, metrics=None, params=None, tags=None, status=RunStatus.FINISHED, a_uri=None, ): return Run( RunInfo( run_uuid=run_id, run_id=run_id, experiment_id=exp_id, user_id=uid, status=status, start_time=start, end_time=end, lifecycle_stage=LifecycleStage.ACTIVE, artifact_uri=a_uri, ), RunData(metrics=metrics, params=params, tags=tags), )
def test_get_run(init_store): expected_run_info = RunInfo(run_uuid="7b2e71956f3d4c08b042624a8d83700d", experiment_id="hTb553MBNoOYfhXjnnQh", user_id="1", status="RUNNING", start_time=1597324762662, end_time=None, lifecycle_stage="active", artifact_uri="artifact_path/7b2e71956f3d4c08b042624a8d83700d" "/artifacts", run_id="7b2e71956f3d4c08b042624a8d83700d") expected_metrics = [Metric(key="metric0", value=20.0, timestamp=1597324762778, step=2), Metric(key="metric1", value=7.0, timestamp=1597324762890, step=2)] expected_params = [Param(key="param0", value="val2"), Param(key="param1", value="Val1"), Param(key="param2", value="Val1"), Param(key="param3", value="valeur4")] expected_tags = [RunTag(key="tag0", value="val2"), RunTag(key="tag1", value="test3"), RunTag(key="tag2", value="val2"), RunTag(key="tag3", value="test3")] expected_run_data = RunData(metrics=expected_metrics, params=expected_params, tags=expected_tags) run = init_store.get_run(expected_run_info._run_id) assert run._info == expected_run_info for i, metric in enumerate(run._data._metric_objs): assert metric.__dict__ == expected_run_data._metric_objs[i].__dict__ assert run._data._params == expected_run_data._params assert run._data._tags == expected_run_data._tags
def get_run(self, run_uuid): _validate_run_id(run_uuid) run_dir = self._find_run_root(run_uuid) if run_dir is None: raise Exception("Run '%s' not found" % run_uuid) run_info = self.get_run_info(run_dir) metrics = self.get_all_metrics(run_uuid) params = self.get_all_params(run_uuid) return Run(run_info, RunData(metrics, params))
def test_find_or_download_params(self): run_id: str = "my_test_run_id" sut: MlflowRepoDownloader = self.create_downloader(run_id) params = [Param(key="test_param", value="test_param_value")] run_data: RunData = RunData(metrics=None, params=params, tags=None) with patch.object(sut, '_MlflowRepoDownloader__get_run_data', return_value=run_data) as method: run_params: Dict[str, Any] = sut.find_or_download_run_params() self.assertEqual("test_param_value", run_params["test_param"])
def get_run(self, run_uuid): """ Will get both active and deleted runs. """ _validate_run_id(run_uuid) run_info = self._get_run_info(run_uuid) metrics = self.get_all_metrics(run_uuid) params = self.get_all_params(run_uuid) tags = self.get_all_tags(run_uuid) return Run(run_info, RunData(metrics, params, tags))
def faculty_run_to_mlflow_run(faculty_run): lifecycle_stage = ( LifecycleStage.ACTIVE if faculty_run.deleted_at is None else LifecycleStage.DELETED ) start_time = _datetime_to_mlflow_timestamp(faculty_run.started_at) end_time = ( _datetime_to_mlflow_timestamp(faculty_run.ended_at) if faculty_run.ended_at is not None else None ) tag_dict = {tag.key: tag.value for tag in faculty_run.tags} extra_mlflow_tags = [] # Set run name tag if set as attribute but not already a tag if MLFLOW_RUN_NAME not in tag_dict and faculty_run.name: extra_mlflow_tags.append(RunTag(MLFLOW_RUN_NAME, faculty_run.name)) # Set parent run ID tag if set as attribute but not already a tag if ( MLFLOW_PARENT_RUN_ID not in tag_dict and faculty_run.parent_run_id is not None ): extra_mlflow_tags.append( RunTag(MLFLOW_PARENT_RUN_ID, faculty_run.parent_run_id.hex) ) run_info = RunInfo( run_uuid=faculty_run.id.hex, experiment_id=str(faculty_run.experiment_id), user_id="", status=_FACULTY_TO_MLFLOW_RUN_STATUS_MAP[faculty_run.status], start_time=start_time, end_time=end_time, lifecycle_stage=lifecycle_stage, artifact_uri=faculty_run.artifact_location, run_id=faculty_run.id.hex, ) run_data = RunData( params=[ faculty_param_to_mlflow_param(param) for param in faculty_run.params ], metrics=[ faculty_metric_to_mlflow_metric(metric) for metric in faculty_run.metrics ], tags=[faculty_tag_to_mlflow_tag(tag) for tag in faculty_run.tags] + extra_mlflow_tags, ) run = Run(run_info, run_data) return run
def test_creation_and_hydration(self): rd1, metrics, params, tags = self._create() self._check(rd1, metrics, params, tags) as_dict = {"metrics": {m.key: m.value for m in metrics}, "params": {p.key: p.value for p in params}, "tags": {t.key: t.value for t in tags}} self.assertEqual(dict(rd1), as_dict) proto = rd1.to_proto() rd2 = RunData.from_proto(proto) self._check(rd2, metrics, params, tags)
def test_bad_comparators(entity_type, bad_comparators, key, entity_value): run = Run(run_info=RunInfo( run_uuid="hi", run_id="hi", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData(metrics=[], params=[], tags=[]) ) for bad_comparator in bad_comparators: bad_filter = "{entity_type}.{key} {comparator} {value}".format( entity_type=entity_type, key=key, comparator=bad_comparator, value=entity_value) with pytest.raises(MlflowException) as e: SearchUtils.filter([run], bad_filter) assert "Invalid comparator" in str(e.value.message)
def _create(): metrics = [ Metric( key=random_str(10), value=random_int(0, 1000), timestamp=int(time.time()) + random_int(-1e4, 1e4), step=random_int(), ) ] params = [Param(random_str(10), random_str(random_int(10, 35))) for _ in range(10)] tags = [RunTag(random_str(10), random_str(random_int(10, 35))) for _ in range(10)] rd = RunData(metrics=metrics, params=params, tags=tags) return rd, metrics, params, tags
def get_run(self, run_id): """ Note: Will get both active and deleted runs. """ _validate_run_id(run_id) run_info = self._get_run_info(run_id) if run_info is None: raise MlflowException("Run '%s' metadata is in invalid state." % run_id, databricks_pb2.INVALID_STATE) metrics = self.get_all_metrics(run_id) params = self.get_all_params(run_id) tags = self.get_all_tags(run_id) return Run(run_info, RunData(metrics, params, tags))
def to_mlflow_entity(self) -> Run: run_info = RunInfo(run_uuid=self.meta.id, run_id=self.meta.id, experiment_id=str(self.experiment_id), user_id=self.user_id, status=self.status, start_time=self.start_time, end_time=self.end_time, lifecycle_stage=self.lifecycle_stage, artifact_uri=self.artifact_uri) run_data = RunData( metrics=[m.to_mlflow_entity() for m in self.latest_metrics], params=[p.to_mlflow_entity() for p in self.params], tags=[t.to_mlflow_entity() for t in self.tags]) return Run(run_info=run_info, run_data=run_data)
def test_filter_runs_by_start_time(): runs = [ Run( run_info=RunInfo( run_uuid=run_id, run_id=run_id, experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FINISHED), start_time=idx, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE, ), run_data=RunData(), ) for idx, run_id in enumerate(["a", "b", "c"]) ] assert SearchUtils.filter(runs, "attribute.start_time >= 0") == runs assert SearchUtils.filter(runs, "attribute.start_time > 1") == runs[2:] assert SearchUtils.filter(runs, "attribute.start_time = 2") == runs[2:]
def _create(): metrics = [ Metric(random_str(10), random_int(0, 1000), int(time.time() + random_int(-1e4, 1e4))) for _ in range(100) ] params = [ Param(random_str(10), random_str(random_int(10, 35))) for _ in range(10) ] # noqa tags = [ RunTag(random_str(10), random_str(random_int(10, 35))) for _ in range(10) ] # noqa rd = RunData() for p in params: rd._add_param(p) for m in metrics: rd._add_metric(m) for t in tags: rd._add_tag(t) return rd, metrics, params, tags
def _generate_run(self, i, runs_dict): """ Generate a run object and save to runs_dict keyed by run_id. Most of data just depends on i, and some data are hard-coded for simplicityGenerate n number of runs. Most of data just depends on n, and some data are hard-coded for simplicity. """ key = f"key{i}" value = f"value{i}" start_time = 123456 * i end_time = start_time + (1000 * i) run_id = f"run_id_{i}" metrics = [Metric(key, value, start_time, "stage")] params = [Param(key, value)] tags = [RunTag(key, value)] run_info = RunInfo(run_id, "experiment_id", "user_id", "status", start_time, end_time, "lifecycle_stage") run = Run(run_info=run_info, run_data=RunData(metrics=metrics, params=params, tags=tags)) runs_dict[run_id] = run return run
def _hit_to_mlflow_run_data( self, hit: Any, columns_to_whitelist_key_dict: dict) -> RunData: metrics = [ self._hit_to_mlflow_metric(m) for m in ( hit.latest_metrics if hasattr(hit, 'latest_metrics') else []) if (columns_to_whitelist_key_dict is None or m.key in columns_to_whitelist_key_dict["metrics"]) ] params = [ self._hit_to_mlflow_param(p) for p in (hit.params if hasattr(hit, 'params') else []) if (columns_to_whitelist_key_dict is None or p.key in columns_to_whitelist_key_dict["params"]) ] tags = [ self._hit_to_mlflow_tag(t) for t in (hit.tags if hasattr(hit, 'tags') else []) if (columns_to_whitelist_key_dict is None or t.key in columns_to_whitelist_key_dict["tags"]) ] return RunData(metrics=metrics, params=params, tags=tags)
def test_string_repr(self): run_info = RunInfo(run_uuid="hi", experiment_id=0, name="name", source_type=SourceType.PROJECT, source_name="source-name", entry_point_name="entry-point-name", user_id="user-id", status=RunStatus.FAILED, start_time=0, end_time=1, source_version="version") metrics = [Metric("key", i, 0) for i in range(5)] run_data = RunData(metrics=metrics, params=[], tags=[]) run1 = Run(run_info, run_data) expected = "<Run: info=<RunInfo: run_uuid='hi', experiment_id=0, name='name', " \ "source_type=3, source_name='source-name', " \ "entry_point_name='entry-point-name', user_id='user-id', status=4, " \ "start_time=0, end_time=1, source_version='version', artifact_uri=None>, " \ "data=<RunData: metrics=[<Metric: key='key', value=0, timestamp=0>, " \ "<Metric: key='key', value=1, timestamp=0>, ...], params=[], tags=[]>>" assert str(run1) == expected
def get_run(self, run_id): """ Fetch the run from backend store. The resulting :py:class:`Run <mlflow.entities.Run>` contains a collection of run metadata - :py:class:`RunInfo <mlflow.entities.RunInfo>`, as well as a collection of run parameters, tags, and metrics - :py:class`RunData <mlflow.entities.RunData>`. In the case where multiple metrics with the same key are logged for the run, the :py:class:`RunData <mlflow.entities.RunData>` contains the value at the latest timestamp for each metric. If there are multiple values with the latest timestamp for a given metric, the maximum of these values is returned. :param run_id: Unique identifier for the run. :return: A single :py:class:`mlflow.entities.Run` object, if the run exists. Otherwise, raises an exception. """ _validate_run_id(run_id) run_info = self._get_run_info(run_id) metrics = self.get_all_metrics(run_id) params = self.get_all_params(run_id) tags = self.get_all_tags(run_id) return Run(run_info, RunData(metrics, params, tags))
def test_string_repr(self): run_info = RunInfo(run_uuid="hi", run_id="hi", experiment_id=0, user_id="user-id", status=RunStatus.FAILED, start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE) metrics = [ Metric(key="key-%s" % i, value=i, timestamp=0, step=i) for i in range(3) ] run_data = RunData(metrics=metrics, params=[], tags=[]) run1 = Run(run_info, run_data) expected = ( "<Run: data=<RunData: metrics={'key-0': 0, 'key-1': 1, 'key-2': 2}, " "params={}, tags={}>, info=<RunInfo: artifact_uri=None, end_time=1, " "experiment_id=0, " "lifecycle_stage='active', run_id='hi', run_uuid='hi', " "start_time=0, status=4, user_id='user-id'>>") assert str(run1) == expected
def test_bad_comparators(entity_type, bad_comparators, entity_value): run = Run(run_info=RunInfo(run_uuid="hi", experiment_id=0, name="name", source_type=SourceType.PROJECT, source_name="source-name", entry_point_name="entry-point-name", user_id="user-id", status=RunStatus.FAILED, start_time=0, end_time=1, source_version="version", lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData(metrics=[], params=[], tags=[])) for bad_comparator in bad_comparators: bad_filter = "{entity_type}.abc {comparator} {value}".format( entity_type=entity_type, comparator=bad_comparator, value=entity_value) sf = SearchFilter(filter_string=bad_filter) with pytest.raises(MlflowException) as e: sf.filter(run) assert "Invalid comparator" in str(e.value.message)
def _search_runs( self, experiment_ids, filter_string, run_view_type, max_results, order_by, page_token, ): if max_results > SEARCH_MAX_RESULTS_THRESHOLD: raise MlflowException( "Invalid value for request parameter max_results. It must be at " "most {}, but got value {}".format( SEARCH_MAX_RESULTS_THRESHOLD, max_results), INVALID_PARAMETER_VALUE, ) runs = [] for experiment_id in experiment_ids: run_ids = self._list_runs_ids(experiment_id, run_view_type) run_infos = [ _dict_to_run_info(r) for r in self._get_run_list(run_ids) ] for run_info in run_infos: # Load the metrics, params and tags for the run run_id = run_info.run_id metrics = self.get_all_metrics(run_id) params = self.get_all_params(run_id) tags = self.get_all_tags(run_id) run = Run(run_info, RunData(metrics, params, tags)) runs.append(run) filtered = SearchUtils.filter(runs, filter_string) sorted_runs = SearchUtils.sort(filtered, order_by) runs, next_page_token = SearchUtils.paginate(sorted_runs, page_token, max_results) return runs, next_page_token