def create_run(run_id="", exp_id="", uid="", start=0, end=0, metrics=None, params=None, tags=None, status=RunStatus.FINISHED, a_uri=None): return Run( RunInfo(run_uuid=run_id, run_id=run_id, experiment_id=exp_id, user_id=uid, status=status, start_time=start, end_time=end, lifecycle_stage=LifecycleStage.ACTIVE, artifact_uri=a_uri), RunData(metrics=metrics, params=params, tags=tags))
def _generate_run(self, i, runs_dict): """ Generate a run object and save to runs_dict keyed by run_id. Most of data just depends on i, and some data are hard-coded for simplicityGenerate n number of runs. Most of data just depends on n, and some data are hard-coded for simplicity. """ key = f"key{i}" value = f"value{i}" start_time = 123456 * i end_time = start_time + (1000 * i) run_id = f"run_id_{i}" metrics = [Metric(key, value, start_time, "stage")] params = [Param(key, value)] tags = [RunTag(key, value)] run_info = RunInfo(run_id, "experiment_id", "user_id", "status", start_time, end_time, "lifecycle_stage") run = Run(run_info=run_info, run_data=RunData(metrics=metrics, params=params, tags=tags)) runs_dict[run_id] = run return run
def _search_runs(self, experiment_ids, filter_string, run_view_type, max_results, order_by, page_token): experiment_ids = [ str(experiment_id) for experiment_id in experiment_ids ] sr = SearchRuns( experiment_ids=experiment_ids, filter=filter_string, run_view_type=ViewType.to_proto(run_view_type), max_results=max_results, order_by=order_by, page_token=page_token, ) req_body = message_to_json(sr) response_proto = self._call_endpoint(SearchRuns, req_body) runs = [Run.from_proto(proto_run) for proto_run in response_proto.runs] # If next_page_token is not set, we will see it as "". We need to convert this to None. next_page_token = None if response_proto.next_page_token: next_page_token = response_proto.next_page_token return runs, next_page_token
def search_runs(self, experiment_ids, search_filter, run_view_type): """ Returns runs that match the given list of search expressions within the experiments. Given multiple search expressions, all these expressions are ANDed together for search. :param experiment_ids: List of experiment ids to scope the search :param search_filter: :py:class`mlflow.utils.search_utils.SearchFilter` object to encode search expression or filter string. :param run_view_type: ACTIVE, DELETED, or ALL runs. :return: A list of Run objects that satisfy the search expressions """ sr = SearchRuns( experiment_ids=experiment_ids, anded_expressions=search_filter.search_expressions if search_filter else [], filter=search_filter.filter_string if search_filter else None, run_view_type=ViewType.to_proto(run_view_type)) req_body = message_to_json(sr) response_proto = self._call_endpoint(SearchRuns, req_body) return [Run.from_proto(proto_run) for proto_run in response_proto.runs]
def test_string_repr(self): run_info = RunInfo(run_uuid="hi", run_id="hi", experiment_id=0, user_id="user-id", status=RunStatus.FAILED, start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE) metrics = [ Metric(key="key-%s" % i, value=i, timestamp=0, step=i) for i in range(3) ] run_data = RunData(metrics=metrics, params=[], tags=[]) run1 = Run(run_info, run_data) expected = ( "<Run: data=<RunData: metrics={'key-0': 0, 'key-1': 1, 'key-2': 2}, " "params={}, tags={}>, info=<RunInfo: artifact_uri=None, end_time=1, " "experiment_id=0, " "lifecycle_stage='active', run_id='hi', run_uuid='hi', " "start_time=0, status=4, user_id='user-id'>>") assert str(run1) == expected
def create_run(self, experiment_id, user_id, start_time, tags): """ Create a run under the specified experiment ID, setting the run's status to "RUNNING" and the start time to the current time. :param experiment_id: ID of the experiment for this run :param user_id: ID of the user launching this run :param source_type: Enum (integer) describing the source of the run :return: The created Run object """ tag_protos = [tag.to_proto() for tag in tags] req_body = message_to_json( CreateRun( experiment_id=str(experiment_id), user_id=user_id, start_time=start_time, tags=tag_protos, )) response_proto = self._call_endpoint(CreateRun, req_body) run = Run.from_proto(response_proto.run) return run
def create_run(self, experiment_id, user_id, run_name, source_type, source_name, entry_point_name, start_time, source_version, tags, parent_run_id): """ Creates a run with the specified attributes. """ experiment = self.get_experiment(experiment_id) if experiment is None: raise MlflowException( "Could not create run under experiment with ID %s - no such experiment " "exists." % experiment_id, databricks_pb2.RESOURCE_DOES_NOT_EXIST) if experiment.lifecycle_stage != Experiment.ACTIVE_LIFECYCLE: raise MlflowException( "Could not create run under non-active experiment with ID " "%s." % experiment_id, databricks_pb2.INVALID_STATE) run_uuid = uuid.uuid4().hex artifact_uri = self._get_artifact_dir(experiment_id, run_uuid) run_info = RunInfo(run_uuid=run_uuid, experiment_id=experiment_id, name="", artifact_uri=artifact_uri, source_type=source_type, source_name=source_name, entry_point_name=entry_point_name, user_id=user_id, status=RunStatus.RUNNING, start_time=start_time, end_time=None, source_version=source_version, lifecycle_stage=RunInfo.ACTIVE_LIFECYCLE) # Persist run metadata and create directories for logging metrics, parameters, artifacts run_dir = self._get_run_dir(run_info.experiment_id, run_info.run_uuid) mkdir(run_dir) write_yaml(run_dir, FileStore.META_DATA_FILE_NAME, _make_persisted_run_info_dict(run_info)) mkdir(run_dir, FileStore.METRICS_FOLDER_NAME) mkdir(run_dir, FileStore.PARAMS_FOLDER_NAME) mkdir(run_dir, FileStore.ARTIFACTS_FOLDER_NAME) for tag in tags: self.set_tag(run_uuid, tag) if parent_run_id: self.set_tag(run_uuid, RunTag(key=MLFLOW_PARENT_RUN_ID, value=parent_run_id)) if run_name: self.set_tag(run_uuid, RunTag(key=MLFLOW_RUN_NAME, value=run_name)) return Run(run_info=run_info, run_data=None)
def to_mlflow_entity(self): """ Convert DB model to corresponding MLflow entity. :return: :py:class:`mlflow.entities.Run`. """ run_info = RunInfo(run_uuid=self.run_uuid, run_id=self.run_uuid, experiment_id=str(self.experiment_id), user_id=self.user_id, status=self.status, start_time=self.start_time, end_time=self.end_time, lifecycle_stage=self.lifecycle_stage, artifact_uri=self.artifact_uri) run_data = RunData( metrics=[m.to_mlflow_entity() for m in self.latest_metrics], params=[p.to_mlflow_entity() for p in self.params], tags=[t.to_mlflow_entity() for t in self.tags]) return Run(run_info=run_info, run_data=run_data)
def test_string_repr(self): run_info = RunInfo(run_uuid="hi", experiment_id=0, name="name", source_type=SourceType.PROJECT, source_name="source-name", entry_point_name="entry-point-name", user_id="user-id", status=RunStatus.FAILED, start_time=0, end_time=1, source_version="version") metrics = [Metric("key", i, 0) for i in range(5)] run_data = RunData(metrics=metrics, params=[], tags=[]) run1 = Run(run_info, run_data) expected = "<Run: info=<RunInfo: run_uuid='hi', experiment_id=0, name='name', " \ "source_type=3, source_name='source-name', " \ "entry_point_name='entry-point-name', user_id='user-id', status=4, " \ "start_time=0, end_time=1, source_version='version', artifact_uri=None>, " \ "data=<RunData: metrics=[<Metric: key='key', value=0, timestamp=0>, " \ "<Metric: key='key', value=1, timestamp=0>, ...], params=[], tags=[]>>" assert str(run1) == expected
def create_run(self, experiment_id, user_id, run_name, source_type, source_name, entry_point_name, start_time, source_version, tags, parent_run_id): """ Creates a run under the specified experiment ID, setting the run's status to "RUNNING" and the start time to the current time. :param experiment_id: ID of the experiment for this run :param user_id: ID of the user launching this run :param source_type: Enum (integer) describing the source of the run :return: The created Run object """ tag_protos = [tag.to_proto() for tag in tags] req_body = message_to_json(CreateRun( experiment_id=experiment_id, user_id=user_id, run_name="", source_type=source_type, source_name=source_name, entry_point_name=entry_point_name, start_time=start_time, source_version=source_version, tags=tag_protos, parent_run_id=parent_run_id)) response_proto = self._call_endpoint(CreateRun, req_body) run = Run.from_proto(response_proto.run) if run_name: self.set_tag(run.info.run_uuid, RunTag(key=MLFLOW_RUN_NAME, value=run_name)) return run
def to_mlflow_entity(self, session): """ Convert DB model to corresponding MLflow entity. :return: :py:class:`mlflow.entities.Run`. """ run_info = RunInfo(run_uuid=self.run_uuid, run_id=self.run_uuid, experiment_id=str(self.experiment_id), user_id=self.user_id, status=self.status, start_time=self.start_time, end_time=self.end_time, lifecycle_stage=self.lifecycle_stage, artifact_uri=self.artifact_uri) # only get latest recorded metrics per key last_metrics = self.get_last_recorded_metrics(session) all_metrics = [ Metric(key=m[1], value=m[4] if not m[5] else float("nan"), timestamp=m[3], step=m[2]) for m in last_metrics ] metrics = {} for m in all_metrics: existing_metric = metrics.get(m.key) if (existing_metric is None)\ or ((m.step, m.timestamp, m.value) >= (existing_metric.step, existing_metric.timestamp, existing_metric.value)): metrics[m.key] = m run_data = RunData(metrics=list(metrics.values()), params=[p.to_mlflow_entity() for p in self.params], tags=[t.to_mlflow_entity() for t in self.tags]) return Run(run_info=run_info, run_data=run_data)
def get_run(self, run_id): """ Fetch the run from backend store. The resulting :py:class:`Run <mlflow.entities.Run>` contains a collection of run metadata - :py:class:`RunInfo <mlflow.entities.RunInfo>`, as well as a collection of run parameters, tags, and metrics - :py:class`RunData <mlflow.entities.RunData>`. In the case where multiple metrics with the same key are logged for the run, the :py:class:`RunData <mlflow.entities.RunData>` contains the value at the latest timestamp for each metric. If there are multiple values with the latest timestamp for a given metric, the maximum of these values is returned. :param run_id: Unique identifier for the run. :return: A single :py:class:`mlflow.entities.Run` object, if the run exists. Otherwise, raises an exception. """ _validate_run_id(run_id) run_info = self._get_run_info(run_id) metrics = self.get_all_metrics(run_id) params = self.get_all_params(run_id) tags = self.get_all_tags(run_id) return Run(run_info, RunData(metrics, params, tags))
def test_create_model_version_run_link_in_notebook_with_default_profile(mock_registry_store): experiment_id = 'test-exp-id' hostname = 'https://workspace.databricks.com/' workspace_id = '10002' run_id = 'runid' workspace_url = construct_run_url(hostname, experiment_id, run_id, workspace_id) get_run_mock = mock.MagicMock() get_run_mock.return_value = Run(RunInfo(run_id, experiment_id, 'userid', 'status', 0, 1, None), None) with mock.patch('mlflow.tracking.client.is_in_databricks_notebook', return_value=True), \ mock.patch('mlflow.tracking.client.get_workspace_info_from_dbutils', return_value=(hostname, workspace_id)): client = MlflowClient(tracking_uri='databricks', registry_uri='otherplace') client.get_run = get_run_mock mock_registry_store.create_model_version.return_value = \ ModelVersion('name', 1, 0, 1, source='source', run_id=run_id, run_link=workspace_url) model_version = client.create_model_version('name', 'source', 'runid') assert(model_version.run_link == workspace_url) # verify that the client generated the right URL mock_registry_store.create_model_version.assert_called_once_with( "name", 'source', 'runid', [], workspace_url)
def to_mlflow_entity(self): """ Convert DB model to corresponding MLflow entity. :return: :py:class:`mlflow.entities.Run`. """ run_info = RunInfo(run_uuid=self.run_uuid, run_id=self.run_uuid, experiment_id=str(self.experiment_id), name=self.name, source_type=SourceType.from_string( self.source_type), source_name=self.source_name, entry_point_name=self.entry_point_name, user_id=self.user_id, status=RunStatus.from_string(self.status), start_time=self.start_time, end_time=self.end_time, source_version=self.source_version, lifecycle_stage=self.lifecycle_stage, artifact_uri=self.artifact_uri) # only get latest recorded metrics per key all_metrics = [m.to_mlflow_entity() for m in self.metrics] metrics = {} for m in all_metrics: existing_metric = metrics.get(m.key) if (existing_metric is None)\ or ((m.step, m.timestamp, m.value) >= (existing_metric.step, existing_metric.timestamp, existing_metric.value)): metrics[m.key] = m run_data = RunData(metrics=list(metrics.values()), params=[p.to_mlflow_entity() for p in self.params], tags=[t.to_mlflow_entity() for t in self.tags]) return Run(run_info=run_info, run_data=run_data)
def test_bad_comparators(entity_type, bad_comparators, key, entity_value): run = Run( run_info=RunInfo( run_uuid="hi", run_id="hi", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE, ), run_data=RunData(metrics=[], params=[], tags=[]), ) for bad_comparator in bad_comparators: bad_filter = "{entity_type}.{key} {comparator} {value}".format( entity_type=entity_type, key=key, comparator=bad_comparator, value=entity_value) with pytest.raises(MlflowException) as e: SearchUtils.filter([run], bad_filter) assert "Invalid comparator" in str(e.value.message)
def test_bad_comparators(entity_type, bad_comparators, entity_value): run = Run(run_info=RunInfo(run_uuid="hi", experiment_id=0, name="name", source_type=SourceType.PROJECT, source_name="source-name", entry_point_name="entry-point-name", user_id="user-id", status=RunStatus.FAILED, start_time=0, end_time=1, source_version="version", lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData(metrics=[], params=[], tags=[])) for bad_comparator in bad_comparators: bad_filter = "{entity_type}.abc {comparator} {value}".format( entity_type=entity_type, comparator=bad_comparator, value=entity_value) sf = SearchFilter(filter_string=bad_filter) with pytest.raises(MlflowException) as e: sf.filter(run) assert "Invalid comparator" in str(e.value.message)
def create_run(self, experiment_id, user_id, run_name, source_type, source_name, entry_point_name, start_time, source_version, tags): """ Creates a run with the specified attributes. """ if self.get_experiment(experiment_id) is None: raise Exception( "Could not create run under experiment with ID %s - no such experiment " "exists." % experiment_id) run_uuid = uuid.uuid4().hex artifact_uri = self._get_artifact_dir(experiment_id, run_uuid) run_info = RunInfo(run_uuid=run_uuid, experiment_id=experiment_id, name="", artifact_uri=artifact_uri, source_type=source_type, source_name=source_name, entry_point_name=entry_point_name, user_id=user_id, status=RunStatus.RUNNING, start_time=start_time, end_time=None, source_version=source_version) # Persist run metadata and create directories for logging metrics, parameters, artifacts run_dir = self._get_run_dir(run_info.experiment_id, run_info.run_uuid) mkdir(run_dir) write_yaml(run_dir, FileStore.META_DATA_FILE_NAME, self._make_run_info_dict(run_info)) mkdir(run_dir, FileStore.METRICS_FOLDER_NAME) mkdir(run_dir, FileStore.PARAMS_FOLDER_NAME) mkdir(run_dir, FileStore.ARTIFACTS_FOLDER_NAME) for tag in tags: self.set_tag(run_uuid, tag) if run_name: self.set_tag(run_uuid, RunTag(key=MLFLOW_RUN_NAME, value=run_name)) return Run(run_info=run_info, run_data=None)
def create_run(self, experiment_id, user_id, start_time, tags): """ Creates a run with the specified attributes. """ experiment_id = FileStore.DEFAULT_EXPERIMENT_ID if experiment_id is None else experiment_id experiment = self.get_experiment(experiment_id) if experiment is None: raise MlflowException( "Could not create run under experiment with ID %s - no such experiment " "exists." % experiment_id, databricks_pb2.RESOURCE_DOES_NOT_EXIST) if experiment.lifecycle_stage != LifecycleStage.ACTIVE: raise MlflowException( "Could not create run under non-active experiment with ID " "%s." % experiment_id, databricks_pb2.INVALID_STATE) run_uuid = uuid.uuid4().hex artifact_uri = self._get_artifact_dir(experiment_id, run_uuid) run_info = RunInfo(run_uuid=run_uuid, run_id=run_uuid, experiment_id=experiment_id, artifact_uri=artifact_uri, user_id=user_id, status=RunStatus.RUNNING, start_time=start_time, end_time=None, lifecycle_stage=LifecycleStage.ACTIVE) # Persist run metadata and create directories for logging metrics, parameters, artifacts run_dir = self._get_run_dir(run_info.experiment_id, run_info.run_id) mkdir(run_dir) run_info_dict = _make_persisted_run_info_dict(run_info) write_yaml(run_dir, FileStore.META_DATA_FILE_NAME, run_info_dict) mkdir(run_dir, FileStore.METRICS_FOLDER_NAME) mkdir(run_dir, FileStore.PARAMS_FOLDER_NAME) mkdir(run_dir, FileStore.ARTIFACTS_FOLDER_NAME) for tag in tags: self.set_tag(run_uuid, tag) return Run(run_info=run_info, run_data=None)
def _search_runs( self, experiment_ids, filter_string, run_view_type, max_results, order_by, page_token, ): if max_results > SEARCH_MAX_RESULTS_THRESHOLD: raise MlflowException( "Invalid value for request parameter max_results. It must be at " "most {}, but got value {}".format( SEARCH_MAX_RESULTS_THRESHOLD, max_results), INVALID_PARAMETER_VALUE, ) runs = [] for experiment_id in experiment_ids: run_ids = self._list_runs_ids(experiment_id, run_view_type) run_infos = [ _dict_to_run_info(r) for r in self._get_run_list(run_ids) ] for run_info in run_infos: # Load the metrics, params and tags for the run run_id = run_info.run_id metrics = self.get_all_metrics(run_id) params = self.get_all_params(run_id) tags = self.get_all_tags(run_id) run = Run(run_info, RunData(metrics, params, tags)) runs.append(run) filtered = SearchUtils.filter(runs, filter_string) sorted_runs = SearchUtils.sort(filtered, order_by) runs, next_page_token = SearchUtils.paginate(sorted_runs, page_token, max_results) return runs, next_page_token
def test_order_by_metric_with_nans_and_infs(): metric_vals_str = ["nan", "inf", "-inf", "-1000", "0", "1000"] runs = [ Run(run_info=RunInfo(run_id=x, run_uuid=x, experiment_id=0, user_id="user", status=RunStatus.to_string(RunStatus.FINISHED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData(metrics=[Metric("x", float(x), 1, 0)])) for x in metric_vals_str ] sorted_runs_asc = [ x.info.run_id for x in SearchUtils.sort(runs, ["metrics.x asc"]) ] sorted_runs_desc = [ x.info.run_id for x in SearchUtils.sort(runs, ["metrics.x desc"]) ] # asc assert ["-inf", "-1000", "0", "1000", "inf", "nan"] == sorted_runs_asc # desc assert ["inf", "1000", "0", "-1000", "-inf", "nan"] == sorted_runs_desc
def test_creation_and_hydration(self): run_data, metrics, params, tags = TestRunData._create() (run_info, run_uuid, experiment_id, name, source_type, source_name, entry_point_name, user_id, status, start_time, end_time, source_version, lifecycle_stage, artifact_uri) = TestRunInfo._create() run1 = Run(run_info, run_data) self._check_run(run1, run_info, run_data) as_dict = { "info": { "run_uuid": run_uuid, "experiment_id": experiment_id, "name": name, "source_type": source_type, "source_name": source_name, "entry_point_name": entry_point_name, "user_id": user_id, "status": status, "start_time": start_time, "end_time": end_time, "source_version": source_version, "lifecycle_stage": lifecycle_stage, "artifact_uri": artifact_uri, }, "data": { "metrics": [dict(m) for m in metrics], "params": [dict(p) for p in params], "tags": [dict(t) for t in tags] } } self.assertEqual(run1.to_dictionary(), as_dict) proto = run1.to_proto() run2 = Run.from_proto(proto) self._check_run(run2, run_info, run_data) run3 = Run.from_dictionary(as_dict) self._check_run(run3, run_info, run_data)
def _get_run_from_info(self, run_info): metrics = self._get_all_metrics(run_info) params = self._get_all_params(run_info) tags = self._get_all_tags(run_info) return Run(run_info, RunData(metrics, params, tags))
experiment = Experiment(experiment_id="1", name="experiment_name", artifact_location="artifact_location", lifecycle_stage=LifecycleStage.ACTIVE, tags=[]) run_info = RunInfo(run_uuid="1", run_id="1", experiment_id="experiment_id", user_id="unknown", status=RunStatus.to_string(RunStatus.RUNNING), start_time=1, end_time=None, lifecycle_stage=LifecycleStage.ACTIVE, artifact_uri="artifact_uri") run_data = RunData(metrics=[], params=[], tags=[]) run = Run(run_info=run_info, run_data=run_data) metric = Metric(key="metric1", value=1, timestamp=1, step=1) param = Param(key="param1", value="val1") tag = RunTag(key="tag1", value="val1") experiment_tag = ExperimentTag(key="tag1", value="val1") @mock.patch( "mlflow_elasticsearchstore.elasticsearch_store.ElasticsearchStore.list_experiments" ) @pytest.mark.usefixtures('create_mlflow_client') def test_list_experiments(list_experiments_mock, create_mlflow_client):
def to_mlflow_entity(self): # run has diff parameter names in __init__ than in properties_ so we do this manually info = _create_entity(RunInfo, self) data = _create_entity(RunData, self) return Run(run_info=info, run_data=data)
def list_runs(self, experiment_id): """:return: list of :py:class:`mlflow.entities.Run` (with only RunInfo filled)""" run_infos = self.store.list_run_infos(experiment_id) return [Run(run_info.run_uuid, run_info) for run_info in run_infos]
def test_creating_run_with_absent_info_throws_exception(self): run_data = TestRunData._create()[0] with pytest.raises(MlflowException) as no_info_exc: Run(None, run_data) assert "run_info cannot be None" in str(no_info_exc)
def test_pagination(page_token, max_results, matching_runs, expected_next_page_token): runs = [ Run( run_info=RunInfo( run_uuid="0", run_id="0", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE, ), run_data=RunData([], [], []), ), Run( run_info=RunInfo( run_uuid="1", run_id="1", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE, ), run_data=RunData([], [], []), ), Run( run_info=RunInfo( run_uuid="2", run_id="2", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE, ), run_data=RunData([], [], []), ), ] encoded_page_token = None if page_token: encoded_page_token = base64.b64encode( json.dumps(page_token).encode("utf-8")) paginated_runs, next_page_token = SearchUtils.paginate( runs, encoded_page_token, max_results) paginated_run_indices = [] for run in paginated_runs: for i, r in enumerate(runs): if r == run: paginated_run_indices.append(i) break assert paginated_run_indices == matching_runs decoded_next_page_token = None if next_page_token: decoded_next_page_token = json.loads(base64.b64decode(next_page_token)) assert decoded_next_page_token == expected_next_page_token
def __init__(self, run): Run.__init__(self, run.info, run.data)
def _hit_to_mlflow_run(self, hit: Any, columns_to_whitelist_key_dict: dict = None) -> Run: return Run(run_info=self._hit_to_mlflow_run_info(hit), run_data=self._hit_to_mlflow_run_data( hit, columns_to_whitelist_key_dict))