def _create_experiment(): request_message = _get_request_message(CreateExperiment()) tags = [ExperimentTag(tag.key, tag.value) for tag in request_message.tags] experiment_id = _get_tracking_store().create_experiment( request_message.name, request_message.artifact_location, tags ) response_message = CreateExperiment.Response() response_message.experiment_id = experiment_id response = Response(mimetype="application/json") response.set_data(message_to_json(response_message)) return response
def set_experiment_tag(self, experiment_id, key, value): """ Set a tag on the experiment with the specified ID. Value is converted to a string. :param experiment_id: String ID of the experiment. :param key: Name of the tag. :param value: Tag value (converted to a string). """ _validate_tag_name(key) tag = ExperimentTag(key, str(value)) self.store.set_experiment_tag(experiment_id, tag)
def _dict_to_experiment(experiment_dict): dict_copy = experiment_dict.copy() # 'experiment_id' was changed from int to string, so we must cast to string # when reading legacy experiments if isinstance(dict_copy["experiment_id"], int): dict_copy["experiment_id"] = str(dict_copy["experiment_id"]) # Turn the key/value tags into list of experiment tags if "tags" in dict_copy: dict_copy["tags"] = [ ExperimentTag(kv[0], kv[1]) for kv in dict_copy["tags"].items() ] return Experiment.from_dictionary(dict_copy)
def set_experiment_tag(self, experiment_id, key, value): """ Set a tag on the experiment with the specified ID. Value is converted to a string. :param experiment_id: String ID of the experiment. :param key: Name of the tag. :param value: Tag value (converted to a string). """ if not self.ranger_can_authorize_experiment_id(experiment_id, 'update'): print("Access denied.") raise _validate_tag_name(key) tag = ExperimentTag(key, str(value)) self.store.set_experiment_tag(experiment_id, tag)
def create_experiment(self, name, artifact_location=None, tags=None): """Create an experiment. :param name: The experiment name. Must be unique. :param artifact_location: The location to store run artifacts. If not provided, the server picks an appropriate default. :param tags: A dictionary of key-value pairs that are converted into :py:class:`mlflow.entities.ExperimentTag` objects. :return: Integer ID of the created experiment. """ _validate_experiment_artifact_location(artifact_location) return self.store.create_experiment( name=name, artifact_location=artifact_location, tags=[ExperimentTag(key, value) for (key, value) in tags.items()] if tags else [], )
def test_set_experiment_tags(self): fs = self._get_store() fs.set_experiment_tag(TestDynamodbStore.DEFAULT_EXPERIMENT_ID, ExperimentTag("tag0", "value0")) fs.set_experiment_tag(TestDynamodbStore.DEFAULT_EXPERIMENT_ID, ExperimentTag("tag1", "value1")) experiment = fs.get_experiment(TestDynamodbStore.DEFAULT_EXPERIMENT_ID) assert len(experiment.tags) == 2 assert experiment.tags["tag0"] == "value0" assert experiment.tags["tag1"] == "value1" # test that updating a tag works fs.set_experiment_tag(TestDynamodbStore.DEFAULT_EXPERIMENT_ID, ExperimentTag("tag0", "value00000")) experiment = fs.get_experiment(TestDynamodbStore.DEFAULT_EXPERIMENT_ID) assert experiment.tags["tag0"] == "value00000" assert experiment.tags["tag1"] == "value1" # test that setting a tag on 1 experiment does not impact another experiment. exp_id = None for exp in self.experiments: if exp != TestDynamodbStore.DEFAULT_EXPERIMENT_ID: exp_id = exp break experiment = fs.get_experiment(exp_id) assert len(experiment.tags) == 0 # setting a tag on different experiments maintains different values across experiments fs.set_experiment_tag(exp_id, ExperimentTag("tag1", "value11111")) experiment = fs.get_experiment(exp_id) assert len(experiment.tags) == 1 assert experiment.tags["tag1"] == "value11111" experiment = fs.get_experiment(TestDynamodbStore.DEFAULT_EXPERIMENT_ID) assert experiment.tags["tag0"] == "value00000" assert experiment.tags["tag1"] == "value1" # test can set multi-line tags fs.set_experiment_tag( exp_id, ExperimentTag("multiline_tag", "value2\nvalue2\nvalue2")) experiment = fs.get_experiment(exp_id) assert experiment.tags["multiline_tag"] == "value2\nvalue2\nvalue2" # test cannot set tags on deleted experiments fs.delete_experiment(exp_id) with pytest.raises(MlflowException): fs.set_experiment_tag(exp_id, ExperimentTag("should", "notset"))
def _get_experiment_tag_from_file(parent_path, tag_name): _validate_tag_name(tag_name) tag_data = read_file(parent_path, tag_name) return ExperimentTag(tag_name, tag_data)
user_id="unknown", status=RunStatus.to_string(RunStatus.RUNNING), start_time=1, end_time=None, lifecycle_stage=LifecycleStage.ACTIVE, artifact_uri="artifact_uri") run_data = RunData(metrics=[], params=[], tags=[]) run = Run(run_info=run_info, run_data=run_data) metric = Metric(key="metric1", value=1, timestamp=1, step=1) param = Param(key="param1", value="val1") tag = RunTag(key="tag1", value="val1") experiment_tag = ExperimentTag(key="tag1", value="val1") @mock.patch( "mlflow_elasticsearchstore.elasticsearch_store.ElasticsearchStore.list_experiments" ) @pytest.mark.usefixtures('create_mlflow_client') def test_list_experiments(list_experiments_mock, create_mlflow_client): create_mlflow_client.list_experiments(ViewType.ACTIVE_ONLY) list_experiments_mock.assert_called_once_with( view_type=ViewType.ACTIVE_ONLY) @mock.patch( "mlflow_elasticsearchstore.elasticsearch_store.ElasticsearchStore.create_experiment" )
def test_requestor(self, request): response = mock.MagicMock response.status_code = 200 response.text = '{}' request.return_value = response creds = MlflowHostCreds('https://hello') store = RestStore(lambda: creds) user_name = "mock user" source_name = "rest test" source_name_patch = mock.patch( "mlflow.tracking.context.default_context._get_source_name", return_value=source_name ) source_type_patch = mock.patch( "mlflow.tracking.context.default_context._get_source_type", return_value=SourceType.LOCAL ) with mock.patch('mlflow.store.rest_store.http_request') as mock_http, \ mock.patch('mlflow.tracking.utils._get_store', return_value=store), \ mock.patch('mlflow.tracking.context.default_context._get_user', return_value=user_name), \ mock.patch('time.time', return_value=13579), \ source_name_patch, source_type_patch: with mlflow.start_run(experiment_id="43"): cr_body = message_to_json(CreateRun(experiment_id="43", user_id=user_name, start_time=13579000, tags=[ProtoRunTag(key='mlflow.source.name', value=source_name), ProtoRunTag(key='mlflow.source.type', value='LOCAL'), ProtoRunTag(key='mlflow.user', value=user_name)])) expected_kwargs = self._args(creds, "runs/create", "POST", cr_body) assert mock_http.call_count == 1 actual_kwargs = mock_http.call_args[1] # Test the passed tag values separately from the rest of the request # Tag order is inconsistent on Python 2 and 3, but the order does not matter expected_tags = expected_kwargs['json'].pop('tags') actual_tags = actual_kwargs['json'].pop('tags') assert ( sorted(expected_tags, key=lambda t: t['key']) == sorted(actual_tags, key=lambda t: t['key']) ) assert expected_kwargs == actual_kwargs with mock.patch('mlflow.store.rest_store.http_request') as mock_http: store.log_param("some_uuid", Param("k1", "v1")) body = message_to_json(LogParam( run_uuid="some_uuid", run_id="some_uuid", key="k1", value="v1")) self._verify_requests(mock_http, creds, "runs/log-parameter", "POST", body) with mock.patch('mlflow.store.rest_store.http_request') as mock_http: store.set_experiment_tag("some_id", ExperimentTag("t1", "abcd"*1000)) body = message_to_json(SetExperimentTag( experiment_id="some_id", key="t1", value="abcd"*1000)) self._verify_requests(mock_http, creds, "experiments/set-experiment-tag", "POST", body) with mock.patch('mlflow.store.rest_store.http_request') as mock_http: store.set_tag("some_uuid", RunTag("t1", "abcd"*1000)) body = message_to_json(SetTag( run_uuid="some_uuid", run_id="some_uuid", key="t1", value="abcd"*1000)) self._verify_requests(mock_http, creds, "runs/set-tag", "POST", body) with mock.patch('mlflow.store.rest_store.http_request') as mock_http: store.delete_tag("some_uuid", "t1") body = message_to_json(DeleteTag(run_id="some_uuid", key="t1")) self._verify_requests(mock_http, creds, "runs/delete-tag", "POST", body) with mock.patch('mlflow.store.rest_store.http_request') as mock_http: store.log_metric("u2", Metric("m1", 0.87, 12345, 3)) body = message_to_json(LogMetric( run_uuid="u2", run_id="u2", key="m1", value=0.87, timestamp=12345, step=3)) self._verify_requests(mock_http, creds, "runs/log-metric", "POST", body) with mock.patch('mlflow.store.rest_store.http_request') as mock_http: metrics = [Metric("m1", 0.87, 12345, 0), Metric("m2", 0.49, 12345, -1), Metric("m3", 0.58, 12345, 2)] params = [Param("p1", "p1val"), Param("p2", "p2val")] tags = [RunTag("t1", "t1val"), RunTag("t2", "t2val")] store.log_batch(run_id="u2", metrics=metrics, params=params, tags=tags) metric_protos = [metric.to_proto() for metric in metrics] param_protos = [param.to_proto() for param in params] tag_protos = [tag.to_proto() for tag in tags] body = message_to_json(LogBatch(run_id="u2", metrics=metric_protos, params=param_protos, tags=tag_protos)) self._verify_requests(mock_http, creds, "runs/log-batch", "POST", body) with mock.patch('mlflow.store.rest_store.http_request') as mock_http: store.delete_run("u25") self._verify_requests(mock_http, creds, "runs/delete", "POST", message_to_json(DeleteRun(run_id="u25"))) with mock.patch('mlflow.store.rest_store.http_request') as mock_http: store.restore_run("u76") self._verify_requests(mock_http, creds, "runs/restore", "POST", message_to_json(RestoreRun(run_id="u76"))) with mock.patch('mlflow.store.rest_store.http_request') as mock_http: store.delete_experiment("0") self._verify_requests(mock_http, creds, "experiments/delete", "POST", message_to_json(DeleteExperiment(experiment_id="0"))) with mock.patch('mlflow.store.rest_store.http_request') as mock_http: store.restore_experiment("0") self._verify_requests(mock_http, creds, "experiments/restore", "POST", message_to_json(RestoreExperiment(experiment_id="0"))) with mock.patch('mlflow.store.rest_store.http_request') as mock_http: response = mock.MagicMock response.text = '{"runs": ["1a", "2b", "3c"], "next_page_token": "67890fghij"}' mock_http.return_value = response result = store.search_runs(["0", "1"], "params.p1 = 'a'", ViewType.ACTIVE_ONLY, max_results=10, order_by=["a"], page_token="12345abcde") expected_message = SearchRuns(experiment_ids=["0", "1"], filter="params.p1 = 'a'", run_view_type=ViewType.to_proto(ViewType.ACTIVE_ONLY), max_results=10, order_by=["a"], page_token="12345abcde") self._verify_requests(mock_http, creds, "runs/search", "POST", message_to_json(expected_message)) assert result.token == "67890fghij"
def to_mlflow_entity(self) -> ExperimentTag: return ExperimentTag(key=self.key, value=self.value)
def test_experiment_set_tag(init_store): new_experiment_tag = ExperimentTag(key="new_tag", value="new_value") expected_elastic_tags = [ElasticExperimentTag(key="new_tag", value="new_value")] init_store.set_experiment_tag("hTb553MBNoOYfhXjnnQh", new_experiment_tag) actual_experiment = init_store._get_experiment("hTb553MBNoOYfhXjnnQh") assert actual_experiment.tags == expected_elastic_tags
def test_set_experiment_tag_of_non_active_experiment(init_store): new_experiment_tag = ExperimentTag(key="new_tag", value="new_value") with pytest.raises(MlflowException) as excinfo: init_store.set_experiment_tag("hzb553MBNoOYfhXjsXRa", new_experiment_tag) assert "must be in the 'active' state" in str(excinfo.value)