def test_quotes_filtered_on_raw(self): # Enclose the full raw query? Strip it. assert parse_search_query('thinger:unknown "what is this?"') == [ SearchFilter( key=SearchKey(name='thinger'), operator='=', value=SearchValue(raw_value='unknown'), ), SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='what is this?'), ), ] # Enclose the full query? Strip it and the whole query is raw. assert parse_search_query('"thinger:unknown what is this?"') == [ SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='thinger:unknown what is this?'), ), ] # Allow a single quotation at end assert parse_search_query('end"') == [ SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='end"'), ), ] # Allow a single quotation at beginning assert parse_search_query('"beginning') == [ SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='"beginning'), ), ] # Allow a single quotation assert parse_search_query('"') == [ SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='"'), ), ] # Empty quotations become a dropped term assert parse_search_query('""') == [] # Allow a search for space assert parse_search_query('" "') == [ SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value=' '), ), ] # Strip in a balanced manner assert parse_search_query('""woof"') == [ SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='"woof'), ), ] # Don't try this at home kids assert parse_search_query('"""""""""') == [ SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='"'), ), ]
def test_raw_search_anywhere(self): assert parse_search_query( 'hello what user.email:[email protected] where release:1.2.1 when' ) == [ SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='hello what'), ), SearchFilter( key=SearchKey(name='user.email'), operator="=", value=SearchValue(raw_value='*****@*****.**'), ), SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='where'), ), SearchFilter( key=SearchKey(name='release'), operator="=", value=SearchValue(raw_value='1.2.1'), ), SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='when'), ), ] assert parse_search_query('hello') == [ SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='hello'), ), ] assert parse_search_query(' hello ') == [ SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='hello'), ), ] assert parse_search_query(' hello there') == [ SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='hello there'), ), ] assert parse_search_query(' hello there:bye') == [ SearchFilter( key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='hello'), ), SearchFilter( key=SearchKey(name='there'), operator='=', value=SearchValue(raw_value='bye'), ), ]
def test_tab_within_quote(self): assert parse_search_query('release:"a\trelease"') == [ SearchFilter(key=SearchKey(name='release'), operator='=', value=SearchValue(raw_value='a\trelease')), ]
def test_no_converter(self): search_val = SearchValue("me") filters = [SearchFilter(SearchKey("something"), "=", search_val)] filters = convert_query_values(filters, [self.project], self.user, None) assert filters[0].value.raw_value == search_val.raw_value
def _build_search_filter(self, key_name, operator, value): return SearchFilter( key=SearchKey(name=key_name), operator=operator, value=SearchValue(raw_value=value), )
def test_no_converter(self): search_val = SearchValue('me') filters = [SearchFilter(SearchKey('something'), '=', search_val)] filters = convert_query_values(filters, [self.project], self.user) assert filters[0].value.raw_value == search_val.raw_value
def test_is_query_inbox(self): assert parse_search_query("is:inbox") == [ SearchFilter(key=SearchKey(name="inbox"), operator="=", value=SearchValue(True)) ]
def test_simple_in(self): assert parse_search_query("user.email:[[email protected]] test:[hello]") == [ SearchFilter( key=SearchKey(name="user.email"), operator="IN", value=SearchValue(raw_value=["*****@*****.**"]), ), SearchFilter( key=SearchKey(name="test"), operator="IN", value=SearchValue(raw_value=["hello"]), ), ] assert parse_search_query( "user.email:[[email protected],[email protected],[email protected]] test:[hello]" ) == [ SearchFilter( key=SearchKey(name="user.email"), operator="IN", value=SearchValue(raw_value=["*****@*****.**", "*****@*****.**", "*****@*****.**"]), ), SearchFilter( key=SearchKey(name="test"), operator="IN", value=SearchValue(raw_value=["hello"]), ), ] assert parse_search_query( "!user.email:[[email protected], [email protected], [email protected]] test:[hello]" ) == [ SearchFilter( key=SearchKey(name="user.email"), operator="NOT IN", value=SearchValue(raw_value=["*****@*****.**", "*****@*****.**", "*****@*****.**"]), ), SearchFilter( key=SearchKey(name="test"), operator="IN", value=SearchValue(raw_value=["hello"]), ), ] # Make sure brackets still work in normal values assert parse_search_query("test:h[e]llo]") == [ SearchFilter( key=SearchKey(name="test"), operator="=", value=SearchValue(raw_value="h[e]llo]"), ), ] assert parse_search_query("test:[h[e]llo") == [ SearchFilter( key=SearchKey(name="test"), operator="=", value=SearchValue(raw_value="[h[e]llo"), ), ] assert parse_search_query('test:"[h]"') == [ SearchFilter( key=SearchKey(name="test"), operator="=", value=SearchValue(raw_value="[h]"), ), ] assert parse_search_query("test:[h]*") == [ SearchFilter( key=SearchKey(name="test"), operator="=", value=SearchValue(raw_value="[h]*"), ), ] assert parse_search_query("test:[h e]") == [ SearchFilter( key=SearchKey(name="test"), operator="=", value=SearchValue(raw_value="[h"), ), SearchFilter( key=SearchKey(name="message"), operator="=", value=SearchValue(raw_value="e]"), ), ] assert parse_search_query("test:[]") == [ SearchFilter( key=SearchKey(name="test"), operator="=", value=SearchValue(raw_value="[]"), ), ] assert parse_search_query('user.email:[[email protected], "hi", 1]') == [ SearchFilter( key=SearchKey(name="user.email"), operator="IN", value=SearchValue(raw_value=["*****@*****.**", "hi", "1"]), ) ] assert parse_search_query('user.email:[[email protected], "hi", 1.0]') == [ SearchFilter( key=SearchKey(name="user.email"), operator="IN", value=SearchValue(raw_value=["*****@*****.**", "hi", "1.0"]), ) ] assert parse_search_query("test:[[h]]") == [ SearchFilter( key=SearchKey(name="test"), operator="IN", value=SearchValue(raw_value=["[h]"]), ), ] assert parse_search_query("test:[a, [h]]") == [ SearchFilter( key=SearchKey(name="test"), operator="IN", value=SearchValue(raw_value=["a", "[h]"]), ), ] assert parse_search_query("user.email:[[email protected]]user.email:[email protected]") == [ SearchFilter( key=SearchKey(name="user.email"), operator="=", value=SearchValue(raw_value="[[email protected]]user.email:[email protected]"), ), ]
def test_negated_duration_on_non_duration_field(self): assert parse_search_query("!user.id:500s") == [ SearchFilter( key=SearchKey(name="user.id"), operator="!=", value=SearchValue(raw_value="500s") ) ]
def format_search_filter(term, params): projects_to_filter = [ ] # Used to avoid doing multiple conditions on project ID conditions = [] group_ids = None name = term.key.name value = term.value.value if name in (PROJECT_ALIAS, PROJECT_NAME_ALIAS): if term.operator == "=" and value == "": raise InvalidSearchQuery( "Invalid query for 'has' search: 'project' cannot be empty.") slugs = to_list(value) projects = { p.slug: p.id for p in Project.objects.filter( id__in=params.get("project_id", []), slug__in=slugs) } missing = [slug for slug in slugs if slug not in projects] if missing and term.operator in EQUALITY_OPERATORS: raise InvalidSearchQuery( f"Invalid query. Project(s) {', '.join(missing)} do not exist or are not actively selected." ) project_ids = list(sorted(projects.values())) if project_ids: # Create a new search filter with the correct values term = SearchFilter( SearchKey("project_id"), term.operator, SearchValue( project_ids if term.is_in_filter else project_ids[0]), ) converted_filter = convert_search_filter_to_snuba_query(term) if converted_filter: if term.operator in EQUALITY_OPERATORS: projects_to_filter = project_ids conditions.append(converted_filter) elif name == ISSUE_ID_ALIAS and value != "": # A blank term value means that this is a has filter group_ids = to_list(value) elif name == ISSUE_ALIAS: operator = term.operator value = to_list(value) # `unknown` is a special value for when there is no issue associated with the event group_short_ids = [v for v in value if v and v != "unknown"] filter_values = ["" for v in value if not v or v == "unknown"] if group_short_ids and params and "organization_id" in params: try: groups = Group.objects.by_qualified_short_id_bulk( params["organization_id"], group_short_ids, ) except Exception: raise InvalidSearchQuery( f"Invalid value '{group_short_ids}' for 'issue:' filter") else: filter_values.extend(sorted([g.id for g in groups])) term = SearchFilter( SearchKey("issue.id"), operator, SearchValue( filter_values if term.is_in_filter else filter_values[0]), ) converted_filter = convert_search_filter_to_snuba_query(term) conditions.append(converted_filter) elif (name == RELEASE_ALIAS and params and (value == "latest" or term.is_in_filter and any(v == "latest" for v in value))): value = [ parse_release( v, params["project_id"], params.get("environment_objects"), params.get("organization_id"), ) for v in to_list(value) ] converted_filter = convert_search_filter_to_snuba_query( SearchFilter( term.key, term.operator, SearchValue(value if term.is_in_filter else value[0]), )) if converted_filter: conditions.append(converted_filter) else: converted_filter = convert_search_filter_to_snuba_query(term, params=params) if converted_filter: conditions.append(converted_filter) return conditions, projects_to_filter, group_ids
def test_weird_values(self): # quotes within quotes assert parse_search_query('release:"a"thing""') == [ SearchFilter( key=SearchKey(name='release'), operator='=', value=SearchValue(raw_value='a"thing"'), ), ] # newline within quote assert parse_search_query('release:"a\nrelease"') == [ SearchFilter(key=SearchKey(name='release'), operator='=', value=SearchValue(raw_value='a\nrelease')), ] # newline outside quote with self.assertRaises(IncompleteParseError): parse_search_query('release:a\nrelease') # tab within quote assert parse_search_query('release:"a\trelease"') == [ SearchFilter(key=SearchKey(name='release'), operator='=', value=SearchValue(raw_value='a\trelease')), ] # tab outside quote assert parse_search_query('release:a\trelease') == [ SearchFilter( key=SearchKey(name='release'), operator='=', value=SearchValue(raw_value='a'), ), SearchFilter(key=SearchKey(name='message'), operator='=', value=SearchValue(raw_value='\trelease')), ] # escaped quotes assert parse_search_query('release:"a\"thing\""') == [ SearchFilter(key=SearchKey(name='release'), operator='=', value=SearchValue(raw_value='a"thing"')), ] assert parse_search_query('release:"a\"\"release"') == [ SearchFilter(key=SearchKey(name='release'), operator='=', value=SearchValue(raw_value='a""release')), ] # poorly escaped quotes assert parse_search_query('release:"a release\"') == [ SearchFilter(key=SearchKey(name='release'), operator='=', value=SearchValue(raw_value='a release')), ] assert parse_search_query('release:\"a release "') == [ SearchFilter(key=SearchKey(name='release'), operator='=', value=SearchValue(raw_value='a release ')), ]
def get_filter(query=None, params=None): """ Returns an eventstore filter given the search text provided by the user and URL params """ # NOTE: this function assumes project permissions check already happened parsed_terms = [] if query is not None: try: parsed_terms = parse_search_query(query, allow_boolean=True, params=params) except ParseError as e: raise InvalidSearchQuery( f"Parse error: {e.expr.name} (column {e.column():d})") kwargs = { "start": None, "end": None, "conditions": [], "having": [], "user_id": None, "organization_id": None, "project_ids": [], "group_ids": [], "condition_aggregates": [], "aliases": params.get("aliases", {}) if params is not None else {}, } projects_to_filter = [] if any( isinstance(term, ParenExpression) or SearchBoolean.is_operator(term) for term in parsed_terms): ( condition, having, found_projects_to_filter, group_ids, ) = convert_search_boolean_to_snuba_query(parsed_terms, params) if condition: and_conditions = flatten_condition_tree(condition, SNUBA_AND) for func in and_conditions: kwargs["conditions"].append( convert_function_to_condition(func)) if having: kwargs["condition_aggregates"] = [ term.key.name for term in parsed_terms if isinstance(term, AggregateFilter) ] and_having = flatten_condition_tree(having, SNUBA_AND) for func in and_having: kwargs["having"].append(convert_function_to_condition(func)) if found_projects_to_filter: projects_to_filter = list(set(found_projects_to_filter)) if group_ids is not None: kwargs["group_ids"].extend(list(set(group_ids))) else: projects_to_filter = set() for term in parsed_terms: if isinstance(term, SearchFilter): conditions, found_projects_to_filter, group_ids = format_search_filter( term, params) if len(conditions) > 0: kwargs["conditions"].extend(conditions) if found_projects_to_filter: projects_to_filter.update(found_projects_to_filter) if group_ids is not None: kwargs["group_ids"].extend(group_ids) elif isinstance(term, AggregateFilter): converted_filter = convert_aggregate_filter_to_snuba_query( term, params) kwargs["condition_aggregates"].append(term.key.name) if converted_filter: kwargs["having"].append(converted_filter) projects_to_filter = list(projects_to_filter) # Keys included as url params take precedent if same key is included in search # They are also considered safe and to have had access rules applied unlike conditions # from the query string. if params: for key in ("start", "end"): kwargs[key] = params.get(key, None) # OrganizationEndpoint.get_filter() uses project_id, but eventstore.Filter uses project_ids if "user_id" in params: kwargs["user_id"] = params["user_id"] if "organization_id" in params: kwargs["organization_id"] = params["organization_id"] if "project_id" in params: if projects_to_filter: kwargs["project_ids"] = projects_to_filter else: kwargs["project_ids"] = params["project_id"] if "environment" in params: term = SearchFilter(SearchKey("environment"), "=", SearchValue(params["environment"])) kwargs["conditions"].append( convert_search_filter_to_snuba_query(term)) if "group_ids" in params: kwargs["group_ids"] = to_list(params["group_ids"]) # Deprecated alias, use `group_ids` instead if ISSUE_ID_ALIAS in params: kwargs["group_ids"] = to_list(params["issue.id"]) return eventstore.Filter(**kwargs)
def test_timestamp(self): # test date format assert parse_search_query('timestamp>2015-05-18') == [ SearchFilter( key=SearchKey(name='timestamp'), operator=">", value=SearchValue( raw_value=datetime.datetime( 2015, 5, 18, 0, 0, tzinfo=timezone.utc), ), ), ] # test date time format assert parse_search_query('timestamp>2015-05-18T10:15:01') == [ SearchFilter( key=SearchKey(name='timestamp'), operator=">", value=SearchValue( raw_value=datetime.datetime( 2015, 5, 18, 10, 15, 1, tzinfo=timezone.utc), ), ), ] # test date time format w microseconds assert parse_search_query('timestamp>2015-05-18T10:15:01.103') == [ SearchFilter( key=SearchKey(name='timestamp'), operator=">", value=SearchValue( raw_value=datetime.datetime( 2015, 5, 18, 10, 15, 1, 103000, tzinfo=timezone.utc), ), ), ] # test date time format w microseconds and utc marker assert parse_search_query('timestamp:>2015-05-18T10:15:01.103Z') == [ SearchFilter( key=SearchKey(name='timestamp'), operator=">", value=SearchValue( raw_value=datetime.datetime( 2015, 5, 18, 10, 15, 1, 103000, tzinfo=timezone.utc), ), ), ]
def node_visitor(token): if token["type"] == "spaces": return None if token["type"] == "filter": # Filters with an invalid reason raises to signal to the test # runner that we should expect this exception if token.get("invalid"): raise InvalidSearchQuery(token["invalid"]["reason"]) # Transform the operator to match for list values if token["value"]["type"] in ["valueTextList", "valueNumberList"]: operator = "NOT IN" if token["negated"] else "IN" else: # Negate the operator if the filter is negated to match operator = token["operator"] or "=" operator = f"!{operator}" if token["negated"] else operator key = node_visitor(token["key"]) value = node_visitor(token["value"]) if token["filter"] == "boolean" and token["negated"]: operator = "=" value = SearchValue(raw_value=1 if value.raw_value == 0 else 0) return SearchFilter(key, operator, value) if token["type"] == "keySimple": return SearchKey(name=token["value"]) if token["type"] == "keyExplicitTag": return SearchKey(name=f"tags[{token['key']['value']}]") if token["type"] == "keyAggregate": name = node_visitor(token["name"]).name # Consistent join aggregate function parameters args = ", ".join(arg["value"]["value"] for arg in token["args"]["args"]) return AggregateKey(name=f"{name}({args})") if token["type"] == "valueText": # Noramlize values by removing the escaped quotes value = token["value"].replace('\\"', '"') return SearchValue(raw_value=value) if token["type"] == "valueNumber": return SearchValue( raw_value=parse_numeric_value(token["value"], token["unit"])) if token["type"] == "valueTextList": return SearchValue( raw_value=[item["value"]["value"] for item in token["items"]]) if token["type"] == "valueNumberList": return SearchValue(raw_value=[ item["value"]["rawValue"] for item in token["items"] ]) if token["type"] == "valueIso8601Date": return SearchValue(raw_value=parse_datetime_string(token["value"])) if token["type"] == "valueDuration": return SearchValue( raw_value=parse_duration(token["value"], token["unit"])) if token["type"] == "valueRelativeDate": return SearchValue( raw_value=parse_duration(token["value"], token["unit"])) if token["type"] == "valueBoolean": return SearchValue(raw_value=int(token["value"])) if token["type"] == "freeText": if token["quoted"]: # Normalize quotes value = token["value"].replace('\\"', '"') else: # Normalize spacing value = token["value"].strip(" ") if value == "": return None return SearchFilter( key=SearchKey(name="message"), operator="=", value=SearchValue(raw_value=value), )
def test_invalid(self): filters = [SearchFilter(SearchKey("status"), "=", SearchValue("wrong"))] with self.assertRaises(InvalidSearchQuery, expected_regex="invalid status value"): convert_query_values(filters, [self.project], self.user, None)
def test_multiple_statements(self): assert parse_search_query( 'user.email:[email protected] OR user.email:[email protected] OR user.email:[email protected]' ) == [ SearchBoolean(left_term=self.term1, operator='OR', right_term=SearchBoolean(left_term=self.term2, operator='OR', right_term=self.term3)) ] assert parse_search_query( 'user.email:[email protected] AND user.email:[email protected] AND user.email:[email protected]' ) == [ SearchBoolean(left_term=self.term1, operator='AND', right_term=SearchBoolean(left_term=self.term2, operator='AND', right_term=self.term3)) ] term4 = SearchFilter( key=SearchKey(name='user.email'), operator="=", value=SearchValue(raw_value='*****@*****.**'), ) # longer even number of terms assert parse_search_query( 'user.email:[email protected] AND user.email:[email protected] OR user.email:[email protected] AND user.email:[email protected]' ) == [ SearchBoolean(left_term=SearchBoolean(left_term=self.term1, operator='AND', right_term=self.term2), operator='OR', right_term=SearchBoolean(left_term=self.term3, operator='AND', right_term=term4)) ] term5 = SearchFilter( key=SearchKey(name='user.email'), operator="=", value=SearchValue(raw_value='*****@*****.**'), ) # longer odd number of terms assert parse_search_query( 'user.email:[email protected] AND user.email:[email protected] OR user.email:[email protected] AND user.email:[email protected] AND user.email:[email protected]' ) == [ SearchBoolean(left_term=SearchBoolean(left_term=self.term1, operator='AND', right_term=self.term2), operator='OR', right_term=SearchBoolean(left_term=self.term3, operator='AND', right_term=SearchBoolean( left_term=term4, operator='AND', right_term=term5))) ] # absurdly long assert parse_search_query( 'user.email:[email protected] AND user.email:[email protected] OR user.email:[email protected] AND user.email:[email protected] AND user.email:[email protected] OR user.email:[email protected] AND user.email:[email protected] OR user.email:[email protected] AND user.email:[email protected] AND user.email:[email protected]' ) == [ SearchBoolean( left_term=SearchBoolean(left_term=self.term1, operator='AND', right_term=self.term2), operator='OR', right_term=SearchBoolean( left_term=SearchBoolean(left_term=self.term3, operator='AND', right_term=SearchBoolean( left_term=term4, operator='AND', right_term=term5)), operator='OR', right_term=SearchBoolean( left_term=SearchBoolean(left_term=self.term1, operator='AND', right_term=self.term2), operator='OR', right_term=SearchBoolean(left_term=self.term3, operator='AND', right_term=SearchBoolean( left_term=term4, operator='AND', right_term=term5))))) ]