def test_anded_expression(): se = SearchExpression(metric=MetricSearchExpression( key="accuracy", double=DoubleClause(comparator=">=", value=.94))) sf = SearchFilter(anded_expressions=[se]) assert sf._parse() == [{ "type": "metric", "key": "accuracy", "comparator": ">=", "value": 0.94 }]
def test_search_filter_basics(): search_filter = "This is a filter string" anded_expressions = [SearchExpression(), SearchExpression()] # only anded_expressions SearchFilter(anded_expressions=anded_expressions) # only search filter SearchFilter(filter_string=search_filter) # both with pytest.raises(MlflowException) as e: SearchFilter(anded_expressions=anded_expressions, filter_string=search_filter) assert e.message.contains("Can specify only one of 'filter' or 'search_expression'")
def test_bad_comparators(entity_type, bad_comparators, key, entity_value): run = Run(run_info=RunInfo( run_uuid="hi", run_id="hi", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData(metrics=[], params=[], tags=[]) ) for bad_comparator in bad_comparators: bad_filter = "{entity_type}.{key} {comparator} {value}".format( entity_type=entity_type, key=key, comparator=bad_comparator, value=entity_value) sf = SearchFilter(filter_string=bad_filter) with pytest.raises(MlflowException) as e: sf.filter(run) assert "Invalid comparator" in str(e.value.message)
def test_anded_expression_2(): m1 = MetricSearchExpression(key="accuracy", double=DoubleClause(comparator=">=", value=.94)) m2 = MetricSearchExpression(key="error", double=DoubleClause(comparator="<", value=.01)) m3 = MetricSearchExpression(key="mse", float=FloatClause(comparator=">=", value=5)) p1 = ParameterSearchExpression(key="a", string=StringClause(comparator="=", value="0")) p2 = ParameterSearchExpression(key="b", string=StringClause(comparator="!=", value="blah")) sf = SearchFilter( SearchRuns(anded_expressions=[ SearchExpression(metric=m1), SearchExpression(metric=m2), SearchExpression(metric=m3), SearchExpression(parameter=p1), SearchExpression(parameter=p2) ])) assert sf._parse() == [{ 'comparator': '>=', 'key': 'accuracy', 'type': 'metric', 'value': 0.94 }, { 'comparator': '<', 'key': 'error', 'type': 'metric', 'value': 0.01 }, { 'comparator': '>=', 'key': 'mse', 'type': 'metric', 'value': 5 }, { 'comparator': '=', 'key': 'a', 'type': 'parameter', 'value': '0' }, { 'comparator': '!=', 'key': 'b', 'type': 'parameter', 'value': 'blah' }]
def _search(self, experiment_id, metrics_expressions=None, param_expressions=None, run_view_type=ViewType.ALL): search_runs = SearchRuns() search_runs.anded_expressions.extend(metrics_expressions or []) search_runs.anded_expressions.extend(param_expressions or []) search_filter = SearchFilter(search_runs) return [r.info.run_uuid for r in self.store.search_runs([experiment_id], search_filter, run_view_type)]
def _search(self, fs, experiment_id, filter_str=None, run_view_type=ViewType.ALL, max_results=SEARCH_MAX_RESULTS_DEFAULT): search_filter = SearchFilter( filter_string=filter_str) if filter_str else None return [ r.info.run_id for r in fs.search_runs( [experiment_id], search_filter, run_view_type, max_results) ]
def search_runs(self, experiment_ids, filter_string, run_view_type=ViewType.ACTIVE_ONLY): """ Search experiments that fit the search criteria. :param experiment_ids: List of experiment IDs :param filter_string: Filter query string. :param run_view_type: one of enum values ACTIVE_ONLY, DELETED_ONLY, or ALL runs defined in :py:class:`mlflow.entities.ViewType`. :return: """ return self.store.search_runs(experiment_ids=experiment_ids, search_filter=SearchFilter(filter_string=filter_string), run_view_type=run_view_type)
def _search_runs(): request_message = _get_request_message(SearchRuns()) response_message = SearchRuns.Response() run_view_type = ViewType.ACTIVE_ONLY if request_message.HasField('run_view_type'): run_view_type = ViewType.from_proto(request_message.run_view_type) run_entities = _get_store().search_runs(request_message.experiment_ids, SearchFilter(request_message), run_view_type) response_message.runs.extend([r.to_proto() for r in run_entities]) response = Response(mimetype='application/json') response.set_data(message_to_json(response_message)) return response
def test_bad_comparators(entity_type, bad_comparators, entity_value): run = Run(run_info=RunInfo(run_uuid="hi", experiment_id=0, name="name", source_type=SourceType.PROJECT, source_name="source-name", entry_point_name="entry-point-name", user_id="user-id", status=RunStatus.FAILED, start_time=0, end_time=1, source_version="version", lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData(metrics=[], params=[], tags=[])) for bad_comparator in bad_comparators: bad_filter = "{entity_type}.abc {comparator} {value}".format( entity_type=entity_type, comparator=bad_comparator, value=entity_value) sf = SearchFilter(filter_string=bad_filter) with pytest.raises(MlflowException) as e: sf.filter(run) assert "Invalid comparator" in str(e.value.message)
def _search_runs(): request_message = _get_request_message(SearchRuns()) response_message = SearchRuns.Response() run_view_type = ViewType.ACTIVE_ONLY if request_message.HasField('run_view_type'): run_view_type = ViewType.from_proto(request_message.run_view_type) sf = SearchFilter(anded_expressions=request_message.anded_expressions, filter_string=request_message.filter) max_results = request_message.max_results experiment_ids = request_message.experiment_ids run_entities = _get_store().search_runs(experiment_ids, sf, run_view_type, max_results) response_message.runs.extend([r.to_proto() for r in run_entities]) response = Response(mimetype='application/json') response.set_data(message_to_json(response_message)) return response
def search_runs(self, experiment_ids, filter_string, run_view_type=ViewType.ACTIVE_ONLY, max_results=SEARCH_MAX_RESULTS_DEFAULT): """ Search experiments that fit the search criteria. :param experiment_ids: List of experiment IDs :param filter_string: Filter query string. :param run_view_type: one of enum values ACTIVE_ONLY, DELETED_ONLY, or ALL runs defined in :py:class:`mlflow.entities.ViewType`. :param max_results: Maximum number of runs desired. :return: A list of :py:class:`mlflow.entities.Run` objects that satisfy the search expressions """ return self.store.search_runs(experiment_ids=experiment_ids, search_filter=SearchFilter(filter_string=filter_string), run_view_type=run_view_type, max_results=max_results)
def test_invalid_clauses(filter_string, error_message): with pytest.raises(MlflowException) as e: SearchFilter(filter_string=filter_string)._parse() assert error_message in e.value.message
def test_filter(filter_string, parsed_filter): assert SearchFilter( SearchRuns(filter=filter_string))._parse() == parsed_filter
def test_correct_quote_trimming(filter_string, parsed_filter): assert SearchFilter(filter_string=filter_string)._parse() == parsed_filter
def test_filter(filter_string, parsed_filter): assert SearchFilter(filter_string=filter_string)._parse() == parsed_filter
def test_error_filter(filter_string, error_message): with pytest.raises(MlflowException) as e: SearchFilter(SearchRuns(filter=filter_string))._parse() assert error_message in e.value.message
def _search_with_filter_string(self, fs, experiment_id, filter_str, run_view_type=ViewType.ALL): search_filter = SearchFilter(filter_string=filter_str) return [r.info.run_uuid for r in fs.search_runs([experiment_id], search_filter, run_view_type)]