def _search_runs():
request_message = _get_request_message(SearchRuns())
response_message = SearchRuns.Response()
run_entities = _get_store().search_runs(request_message.experiment_ids,
request_message.anded_expressions)
response_message.runs.extend([r.to_proto() for r in run_entities])
response = Response(mimetype='application/json')
response.set_data(message_to_json(response_message))
return response
def _search_runs():
request_message = _get_request_message(SearchRuns(), from_get=True)
response_message = SearchRuns.Response()
run_entities = _get_store().search_runs(request_message.experiment_ids,
request_message.anded_expressions)
response_message.runs.extend([r.to_proto() for r in run_entities])
response = Response(mimetype='application/json')
response.set_data(
MessageToJson(response_message, preserving_proto_field_name=True))
return response
def _search_runs():
request_message = _get_request_message(SearchRuns())
response_message = SearchRuns.Response()
run_view_type = ViewType.ACTIVE_ONLY
if request_message.HasField('run_view_type'):
run_view_type = ViewType.from_proto(request_message.run_view_type)
run_entities = _get_store().search_runs(request_message.experiment_ids,
SearchFilter(request_message),
run_view_type)
response_message.runs.extend([r.to_proto() for r in run_entities])
response = Response(mimetype='application/json')
response.set_data(message_to_json(response_message))
return response
def _search_runs():
request_message = _get_request_message(SearchRuns())
response_message = SearchRuns.Response()
run_view_type = ViewType.ACTIVE_ONLY
if request_message.HasField('run_view_type'):
run_view_type = ViewType.from_proto(request_message.run_view_type)
sf = SearchFilter(anded_expressions=request_message.anded_expressions,
filter_string=request_message.filter)
max_results = request_message.max_results
experiment_ids = request_message.experiment_ids
run_entities = _get_store().search_runs(experiment_ids, sf, run_view_type, max_results)
response_message.runs.extend([r.to_proto() for r in run_entities])
response = Response(mimetype='application/json')
response.set_data(message_to_json(response_message))
return response
def search_runs(self,
experiment_ids,
search_filter,
run_view_type,
max_results=SEARCH_MAX_RESULTS_THRESHOLD):
"""
Return runs that match the given list of search expressions within the experiments.
Given multiple search expressions, all these expressions are ANDed together for search.
:param experiment_ids: List of experiment ids to scope the search
:param search_filter: :py:class`mlflow.utils.search_utils.SearchFilter` object to encode
search expression or filter string.
:param run_view_type: ACTIVE, DELETED, or ALL runs.
:param max_results: Maximum number of runs desired.
:return: A list of Run objects that satisfy the search expressions
"""
experiment_ids = [
str(experiment_id) for experiment_id in experiment_ids
]
sr = SearchRuns(
experiment_ids=experiment_ids,
filter=search_filter.filter_string if search_filter else None,
run_view_type=ViewType.to_proto(run_view_type),
max_results=max_results)
req_body = message_to_json(sr)
response_proto = self._call_endpoint(SearchRuns, req_body)
return [Run.from_proto(proto_run) for proto_run in response_proto.runs]
def test_search_runs_default_view_type(mock_get_request_message, mock_store):
"""
Search Runs default view type is filled in as ViewType.ACTIVE_ONLY
"""
mock_get_request_message.return_value = SearchRuns(experiment_ids=[0], anded_expressions=[])
_search_runs()
args, _ = mock_store.search_runs.call_args
assert args[2] == ViewType.ACTIVE_ONLY
def _search(self, experiment_id, metrics_expressions=None, param_expressions=None,
run_view_type=ViewType.ALL):
search_runs = SearchRuns()
search_runs.anded_expressions.extend(metrics_expressions or [])
search_runs.anded_expressions.extend(param_expressions or [])
search_filter = SearchFilter(search_runs)
return [r.info.run_uuid
for r in self.store.search_runs([experiment_id], search_filter, run_view_type)]
def test_search_filter_basics():
search_filter = "This is a filter string"
anded_expressions = [SearchExpression(), SearchExpression()]
# only anded_expressions
SearchFilter(SearchRuns(anded_expressions=anded_expressions))
# only search filter
SearchFilter(SearchRuns(filter=search_filter))
# both
with pytest.raises(MlflowException) as e:
SearchFilter(
SearchRuns(anded_expressions=anded_expressions,
filter=search_filter))
assert e.message.contains(
"Can specify only one of 'filter' or 'search_expression'")
def _search_runs():
request_message = _get_request_message(SearchRuns())
response_message = SearchRuns.Response()
run_view_type = ViewType.ACTIVE_ONLY
if request_message.HasField('run_view_type'):
run_view_type = ViewType.from_proto(request_message.run_view_type)
filter_string = request_message.filter
max_results = request_message.max_results
experiment_ids = request_message.experiment_ids
order_by = request_message.order_by
page_token = request_message.page_token
run_entities = _get_store().search_runs(experiment_ids, filter_string,
run_view_type, max_results,
order_by, page_token)
response_message.runs.extend([r.to_proto() for r in run_entities])
response = Response(mimetype='application/json')
response.set_data(message_to_json(response_message))
return response
def test_search_runs_default_view_type(mock_get_request_message, mock_store):
"""
Search Runs default view type is filled in as ViewType.ACTIVE_ONLY
"""
mock_get_request_message.return_value = SearchRuns(experiment_ids=["0"])
mock_store.search_runs.return_value = PagedList([], None)
_search_runs()
args, _ = mock_store.search_runs.call_args
assert args[2] == ViewType.ACTIVE_ONLY
def test_anded_expression():
se = SearchExpression(metric=MetricSearchExpression(
key="accuracy", double=DoubleClause(comparator=">=", value=.94)))
sf = SearchFilter(SearchRuns(anded_expressions=[se]))
assert sf._parse() == [{
"type": "metric",
"key": "accuracy",
"comparator": ">=",
"value": 0.94
}]
def test_anded_expression_2():
m1 = MetricSearchExpression(key="accuracy",
double=DoubleClause(comparator=">=",
value=.94))
m2 = MetricSearchExpression(key="error",
double=DoubleClause(comparator="<", value=.01))
m3 = MetricSearchExpression(key="mse",
float=FloatClause(comparator=">=", value=5))
p1 = ParameterSearchExpression(key="a",
string=StringClause(comparator="=",
value="0"))
p2 = ParameterSearchExpression(key="b",
string=StringClause(comparator="!=",
value="blah"))
sf = SearchFilter(
SearchRuns(anded_expressions=[
SearchExpression(metric=m1),
SearchExpression(metric=m2),
SearchExpression(metric=m3),
SearchExpression(parameter=p1),
SearchExpression(parameter=p2)
]))
assert sf._parse() == [{
'comparator': '>=',
'key': 'accuracy',
'type': 'metric',
'value': 0.94
}, {
'comparator': '<',
'key': 'error',
'type': 'metric',
'value': 0.01
}, {
'comparator': '>=',
'key': 'mse',
'type': 'metric',
'value': 5
}, {
'comparator': '=',
'key': 'a',
'type': 'parameter',
'value': '0'
}, {
'comparator': '!=',
'key': 'b',
'type': 'parameter',
'value': 'blah'
}]
def search_runs(self, experiment_ids, search_expressions):
"""
Returns runs that match the given list of search expressions within the experiments.
Given multiple search expressions, all these expressions are ANDed together for search.
:param experiment_ids: List of experiment ids to scope the search
:param search_expression: list of search expressions
:return: A list of Run objects that satisfy the search expressions
"""
search_expressions_protos = [expr.to_proto() for expr in search_expressions]
req_body = _message_to_json(SearchRuns(experiment_ids=experiment_ids,
anded_expressions=search_expressions_protos))
response_proto = self._call_endpoint(SearchRuns, req_body)
return [Run.from_proto(proto_run) for proto_run in response_proto.runs]
def _search_runs(self, experiment_ids, filter_string, run_view_type,
max_results, order_by, page_token):
experiment_ids = [
str(experiment_id) for experiment_id in experiment_ids
]
sr = SearchRuns(experiment_ids=experiment_ids,
filter=filter_string,
run_view_type=ViewType.to_proto(run_view_type),
max_results=max_results,
order_by=order_by,
page_token=page_token)
req_body = message_to_json(sr)
response_proto = self._call_endpoint(SearchRuns, req_body)
runs = [Run.from_proto(proto_run) for proto_run in response_proto.runs]
return runs, response_proto.next_page_token
def search_runs(self,
experiment_ids,
filter_string,
run_view_type,
max_results=SEARCH_MAX_RESULTS_THRESHOLD,
order_by=None):
experiment_ids = [
str(experiment_id) for experiment_id in experiment_ids
]
sr = SearchRuns(experiment_ids=experiment_ids,
filter=filter_string,
run_view_type=ViewType.to_proto(run_view_type),
max_results=max_results,
order_by=order_by)
req_body = message_to_json(sr)
response_proto = self._call_endpoint(SearchRuns, req_body)
return [Run.from_proto(proto_run) for proto_run in response_proto.runs]
def _search_runs(self, experiment_ids, filter_string, run_view_type, max_results, order_by,
page_token):
experiment_ids = [str(experiment_id) for experiment_id in experiment_ids]
sr = SearchRuns(experiment_ids=experiment_ids,
filter=filter_string,
run_view_type=ViewType.to_proto(run_view_type),
max_results=max_results,
order_by=order_by,
page_token=page_token)
req_body = message_to_json(sr)
response_proto = self._call_endpoint(SearchRuns, req_body)
runs = [Run.from_proto(proto_run) for proto_run in response_proto.runs]
# If next_page_token is not set, we will see it as "". We need to convert this to None.
next_page_token = None
if response_proto.next_page_token:
next_page_token = response_proto.next_page_token
return runs, next_page_token
def search_runs(self, experiment_ids, search_filter, run_view_type):
"""
Returns runs that match the given list of search expressions within the experiments.
Given multiple search expressions, all these expressions are ANDed together for search.
:param experiment_ids: List of experiment ids to scope the search
:param search_filter: :py:class`mlflow.utils.search_utils.SearchFilter` object to encode
search expression or filter string.
:param run_view_type: ACTIVE, DELETED, or ALL runs.
:return: A list of Run objects that satisfy the search expressions
"""
sr = SearchRuns(experiment_ids=experiment_ids,
anded_expressions=search_filter.search_expressions if search_filter else [],
filter=search_filter.filter_string if search_filter else None,
run_view_type=ViewType.to_proto(run_view_type))
req_body = message_to_json(sr)
response_proto = self._call_endpoint(SearchRuns, req_body)
return [Run.from_proto(proto_run) for proto_run in response_proto.runs]
def test_requestor(self, request):
response = mock.MagicMock
response.status_code = 200
response.text = '{}'
request.return_value = response
creds = MlflowHostCreds('https://hello')
store = RestStore(lambda: creds)
user_name = "mock user"
source_name = "rest test"
source_name_patch = mock.patch(
"mlflow.tracking.context.default_context._get_source_name",
return_value=source_name)
source_type_patch = mock.patch(
"mlflow.tracking.context.default_context._get_source_type",
return_value=SourceType.LOCAL)
with mock.patch('mlflow.store.rest_store.http_request') as mock_http, \
mock.patch('mlflow.tracking.utils._get_store', return_value=store), \
mock.patch('mlflow.tracking.context.default_context._get_user',
return_value=user_name), \
mock.patch('time.time', return_value=13579), \
source_name_patch, source_type_patch:
with mlflow.start_run(experiment_id="43"):
cr_body = message_to_json(
CreateRun(experiment_id="43",
user_id=user_name,
start_time=13579000,
tags=[
ProtoRunTag(key='mlflow.source.name',
value=source_name),
ProtoRunTag(key='mlflow.source.type',
value='LOCAL'),
ProtoRunTag(key='mlflow.user',
value=user_name)
]))
expected_kwargs = self._args(creds, "runs/create", "POST",
cr_body)
assert mock_http.call_count == 1
actual_kwargs = mock_http.call_args[1]
# Test the passed tag values separately from the rest of the request
# Tag order is inconsistent on Python 2 and 3, but the order does not matter
expected_tags = expected_kwargs['json'].pop('tags')
actual_tags = actual_kwargs['json'].pop('tags')
assert (sorted(expected_tags,
key=lambda t: t['key']) == sorted(
actual_tags, key=lambda t: t['key']))
assert expected_kwargs == actual_kwargs
with mock.patch('mlflow.store.rest_store.http_request') as mock_http:
store.log_param("some_uuid", Param("k1", "v1"))
body = message_to_json(
LogParam(run_uuid="some_uuid",
run_id="some_uuid",
key="k1",
value="v1"))
self._verify_requests(mock_http, creds, "runs/log-parameter",
"POST", body)
with mock.patch('mlflow.store.rest_store.http_request') as mock_http:
store.set_tag("some_uuid", RunTag("t1", "abcd" * 1000))
body = message_to_json(
SetTag(run_uuid="some_uuid",
run_id="some_uuid",
key="t1",
value="abcd" * 1000))
self._verify_requests(mock_http, creds, "runs/set-tag", "POST",
body)
with mock.patch('mlflow.store.rest_store.http_request') as mock_http:
store.delete_tag("some_uuid", "t1")
body = message_to_json(DeleteTag(run_id="some_uuid", key="t1"))
self._verify_requests(mock_http, creds, "runs/delete-tag", "POST",
body)
with mock.patch('mlflow.store.rest_store.http_request') as mock_http:
store.log_metric("u2", Metric("m1", 0.87, 12345, 3))
body = message_to_json(
LogMetric(run_uuid="u2",
run_id="u2",
key="m1",
value=0.87,
timestamp=12345,
step=3))
self._verify_requests(mock_http, creds, "runs/log-metric", "POST",
body)
with mock.patch('mlflow.store.rest_store.http_request') as mock_http:
metrics = [
Metric("m1", 0.87, 12345, 0),
Metric("m2", 0.49, 12345, -1),
Metric("m3", 0.58, 12345, 2)
]
params = [Param("p1", "p1val"), Param("p2", "p2val")]
tags = [RunTag("t1", "t1val"), RunTag("t2", "t2val")]
store.log_batch(run_id="u2",
metrics=metrics,
params=params,
tags=tags)
metric_protos = [metric.to_proto() for metric in metrics]
param_protos = [param.to_proto() for param in params]
tag_protos = [tag.to_proto() for tag in tags]
body = message_to_json(
LogBatch(run_id="u2",
metrics=metric_protos,
params=param_protos,
tags=tag_protos))
self._verify_requests(mock_http, creds, "runs/log-batch", "POST",
body)
with mock.patch('mlflow.store.rest_store.http_request') as mock_http:
store.delete_run("u25")
self._verify_requests(mock_http, creds, "runs/delete", "POST",
message_to_json(DeleteRun(run_id="u25")))
with mock.patch('mlflow.store.rest_store.http_request') as mock_http:
store.restore_run("u76")
self._verify_requests(mock_http, creds, "runs/restore", "POST",
message_to_json(RestoreRun(run_id="u76")))
with mock.patch('mlflow.store.rest_store.http_request') as mock_http:
store.delete_experiment("0")
self._verify_requests(
mock_http, creds, "experiments/delete", "POST",
message_to_json(DeleteExperiment(experiment_id="0")))
with mock.patch('mlflow.store.rest_store.http_request') as mock_http:
store.restore_experiment("0")
self._verify_requests(
mock_http, creds, "experiments/restore", "POST",
message_to_json(RestoreExperiment(experiment_id="0")))
with mock.patch('mlflow.store.rest_store.http_request') as mock_http:
response = mock.MagicMock
response.text = '{"runs": ["1a", "2b", "3c"], "next_page_token": "67890fghij"}'
mock_http.return_value = response
result = store.search_runs(["0", "1"],
"params.p1 = 'a'",
ViewType.ACTIVE_ONLY,
max_results=10,
order_by=["a"],
page_token="12345abcde")
expected_message = SearchRuns(experiment_ids=["0", "1"],
filter="params.p1 = 'a'",
run_view_type=ViewType.to_proto(
ViewType.ACTIVE_ONLY),
max_results=10,
order_by=["a"],
page_token="12345abcde")
self._verify_requests(mock_http, creds, "runs/search", "POST",
message_to_json(expected_message))
assert result.token == "67890fghij"
def test_correct_quote_trimming(filter_string, parsed_filter):
assert SearchFilter(
SearchRuns(filter=filter_string))._parse() == parsed_filter
def test_filter(filter_string, parsed_filter):
assert SearchFilter(
SearchRuns(filter=filter_string))._parse() == parsed_filter
def test_error_filter(filter_string, error_message):
with pytest.raises(MlflowException) as e:
SearchFilter(SearchRuns(filter=filter_string))._parse()
assert error_message in e.value.message
