def test_quotes_filtered_on_raw(self):
# Enclose the full raw query? Strip it.
assert parse_search_query('thinger:unknown "what is this?"') == [
SearchFilter(
key=SearchKey(name='thinger'),
operator='=',
value=SearchValue(raw_value='unknown'),
),
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='what is this?'),
),
]
# Enclose the full query? Strip it and the whole query is raw.
assert parse_search_query('"thinger:unknown what is this?"') == [
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='thinger:unknown what is this?'),
),
]
# Allow a single quotation at end
assert parse_search_query('end"') == [
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='end"'),
),
]
# Allow a single quotation at beginning
assert parse_search_query('"beginning') == [
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='"beginning'),
),
]
# Allow a single quotation
assert parse_search_query('"') == [
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='"'),
),
]
# Empty quotations become a dropped term
assert parse_search_query('""') == []
# Allow a search for space
assert parse_search_query('" "') == [
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value=' '),
),
]
# Strip in a balanced manner
assert parse_search_query('""woof"') == [
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='"woof'),
),
]
# Don't try this at home kids
assert parse_search_query('"""""""""') == [
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='"'),
),
]
def test_raw_search_anywhere(self):
assert parse_search_query(
'hello what user.email:[email protected] where release:1.2.1 when'
) == [
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='hello what'),
),
SearchFilter(
key=SearchKey(name='user.email'),
operator="=",
value=SearchValue(raw_value='*****@*****.**'),
),
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='where'),
),
SearchFilter(
key=SearchKey(name='release'),
operator="=",
value=SearchValue(raw_value='1.2.1'),
),
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='when'),
),
]
assert parse_search_query('hello') == [
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='hello'),
),
]
assert parse_search_query(' hello ') == [
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='hello'),
),
]
assert parse_search_query(' hello there') == [
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='hello there'),
),
]
assert parse_search_query(' hello there:bye') == [
SearchFilter(
key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='hello'),
),
SearchFilter(
key=SearchKey(name='there'),
operator='=',
value=SearchValue(raw_value='bye'),
),
]
def test_tab_within_quote(self):
assert parse_search_query('release:"a\trelease"') == [
SearchFilter(key=SearchKey(name='release'),
operator='=',
value=SearchValue(raw_value='a\trelease')),
]
def test_no_converter(self):
search_val = SearchValue("me")
filters = [SearchFilter(SearchKey("something"), "=", search_val)]
filters = convert_query_values(filters, [self.project], self.user,
None)
assert filters[0].value.raw_value == search_val.raw_value
def _build_search_filter(self, key_name, operator, value):
return SearchFilter(
key=SearchKey(name=key_name),
operator=operator,
value=SearchValue(raw_value=value),
)
def test_no_converter(self):
search_val = SearchValue('me')
filters = [SearchFilter(SearchKey('something'), '=', search_val)]
filters = convert_query_values(filters, [self.project], self.user)
assert filters[0].value.raw_value == search_val.raw_value
def test_is_query_inbox(self):
assert parse_search_query("is:inbox") == [
SearchFilter(key=SearchKey(name="inbox"),
operator="=",
value=SearchValue(True))
]
def test_simple_in(self):
assert parse_search_query("user.email:[[email protected]] test:[hello]") == [
SearchFilter(
key=SearchKey(name="user.email"),
operator="IN",
value=SearchValue(raw_value=["*****@*****.**"]),
),
SearchFilter(
key=SearchKey(name="test"),
operator="IN",
value=SearchValue(raw_value=["hello"]),
),
]
assert parse_search_query(
"user.email:[[email protected],[email protected],[email protected]] test:[hello]"
) == [
SearchFilter(
key=SearchKey(name="user.email"),
operator="IN",
value=SearchValue(raw_value=["*****@*****.**", "*****@*****.**", "*****@*****.**"]),
),
SearchFilter(
key=SearchKey(name="test"),
operator="IN",
value=SearchValue(raw_value=["hello"]),
),
]
assert parse_search_query(
"!user.email:[[email protected], [email protected], [email protected]] test:[hello]"
) == [
SearchFilter(
key=SearchKey(name="user.email"),
operator="NOT IN",
value=SearchValue(raw_value=["*****@*****.**", "*****@*****.**", "*****@*****.**"]),
),
SearchFilter(
key=SearchKey(name="test"),
operator="IN",
value=SearchValue(raw_value=["hello"]),
),
]
# Make sure brackets still work in normal values
assert parse_search_query("test:h[e]llo]") == [
SearchFilter(
key=SearchKey(name="test"),
operator="=",
value=SearchValue(raw_value="h[e]llo]"),
),
]
assert parse_search_query("test:[h[e]llo") == [
SearchFilter(
key=SearchKey(name="test"),
operator="=",
value=SearchValue(raw_value="[h[e]llo"),
),
]
assert parse_search_query('test:"[h]"') == [
SearchFilter(
key=SearchKey(name="test"),
operator="=",
value=SearchValue(raw_value="[h]"),
),
]
assert parse_search_query("test:[h]*") == [
SearchFilter(
key=SearchKey(name="test"),
operator="=",
value=SearchValue(raw_value="[h]*"),
),
]
assert parse_search_query("test:[h e]") == [
SearchFilter(
key=SearchKey(name="test"),
operator="=",
value=SearchValue(raw_value="[h"),
),
SearchFilter(
key=SearchKey(name="message"),
operator="=",
value=SearchValue(raw_value="e]"),
),
]
assert parse_search_query("test:[]") == [
SearchFilter(
key=SearchKey(name="test"),
operator="=",
value=SearchValue(raw_value="[]"),
),
]
assert parse_search_query('user.email:[[email protected], "hi", 1]') == [
SearchFilter(
key=SearchKey(name="user.email"),
operator="IN",
value=SearchValue(raw_value=["*****@*****.**", "hi", "1"]),
)
]
assert parse_search_query('user.email:[[email protected], "hi", 1.0]') == [
SearchFilter(
key=SearchKey(name="user.email"),
operator="IN",
value=SearchValue(raw_value=["*****@*****.**", "hi", "1.0"]),
)
]
assert parse_search_query("test:[[h]]") == [
SearchFilter(
key=SearchKey(name="test"),
operator="IN",
value=SearchValue(raw_value=["[h]"]),
),
]
assert parse_search_query("test:[a, [h]]") == [
SearchFilter(
key=SearchKey(name="test"),
operator="IN",
value=SearchValue(raw_value=["a", "[h]"]),
),
]
assert parse_search_query("user.email:[[email protected]]user.email:[email protected]") == [
SearchFilter(
key=SearchKey(name="user.email"),
operator="=",
value=SearchValue(raw_value="[[email protected]]user.email:[email protected]"),
),
]
def test_negated_duration_on_non_duration_field(self):
assert parse_search_query("!user.id:500s") == [
SearchFilter(
key=SearchKey(name="user.id"), operator="!=", value=SearchValue(raw_value="500s")
)
]
def format_search_filter(term, params):
projects_to_filter = [
] # Used to avoid doing multiple conditions on project ID
conditions = []
group_ids = None
name = term.key.name
value = term.value.value
if name in (PROJECT_ALIAS, PROJECT_NAME_ALIAS):
if term.operator == "=" and value == "":
raise InvalidSearchQuery(
"Invalid query for 'has' search: 'project' cannot be empty.")
slugs = to_list(value)
projects = {
p.slug: p.id
for p in Project.objects.filter(
id__in=params.get("project_id", []), slug__in=slugs)
}
missing = [slug for slug in slugs if slug not in projects]
if missing and term.operator in EQUALITY_OPERATORS:
raise InvalidSearchQuery(
f"Invalid query. Project(s) {', '.join(missing)} do not exist or are not actively selected."
)
project_ids = list(sorted(projects.values()))
if project_ids:
# Create a new search filter with the correct values
term = SearchFilter(
SearchKey("project_id"),
term.operator,
SearchValue(
project_ids if term.is_in_filter else project_ids[0]),
)
converted_filter = convert_search_filter_to_snuba_query(term)
if converted_filter:
if term.operator in EQUALITY_OPERATORS:
projects_to_filter = project_ids
conditions.append(converted_filter)
elif name == ISSUE_ID_ALIAS and value != "":
# A blank term value means that this is a has filter
group_ids = to_list(value)
elif name == ISSUE_ALIAS:
operator = term.operator
value = to_list(value)
# `unknown` is a special value for when there is no issue associated with the event
group_short_ids = [v for v in value if v and v != "unknown"]
filter_values = ["" for v in value if not v or v == "unknown"]
if group_short_ids and params and "organization_id" in params:
try:
groups = Group.objects.by_qualified_short_id_bulk(
params["organization_id"],
group_short_ids,
)
except Exception:
raise InvalidSearchQuery(
f"Invalid value '{group_short_ids}' for 'issue:' filter")
else:
filter_values.extend(sorted([g.id for g in groups]))
term = SearchFilter(
SearchKey("issue.id"),
operator,
SearchValue(
filter_values if term.is_in_filter else filter_values[0]),
)
converted_filter = convert_search_filter_to_snuba_query(term)
conditions.append(converted_filter)
elif (name == RELEASE_ALIAS and params and
(value == "latest" or term.is_in_filter and any(v == "latest"
for v in value))):
value = [
parse_release(
v,
params["project_id"],
params.get("environment_objects"),
params.get("organization_id"),
) for v in to_list(value)
]
converted_filter = convert_search_filter_to_snuba_query(
SearchFilter(
term.key,
term.operator,
SearchValue(value if term.is_in_filter else value[0]),
))
if converted_filter:
conditions.append(converted_filter)
else:
converted_filter = convert_search_filter_to_snuba_query(term,
params=params)
if converted_filter:
conditions.append(converted_filter)
return conditions, projects_to_filter, group_ids
def test_weird_values(self):
# quotes within quotes
assert parse_search_query('release:"a"thing""') == [
SearchFilter(
key=SearchKey(name='release'),
operator='=',
value=SearchValue(raw_value='a"thing"'),
),
]
# newline within quote
assert parse_search_query('release:"a\nrelease"') == [
SearchFilter(key=SearchKey(name='release'),
operator='=',
value=SearchValue(raw_value='a\nrelease')),
]
# newline outside quote
with self.assertRaises(IncompleteParseError):
parse_search_query('release:a\nrelease')
# tab within quote
assert parse_search_query('release:"a\trelease"') == [
SearchFilter(key=SearchKey(name='release'),
operator='=',
value=SearchValue(raw_value='a\trelease')),
]
# tab outside quote
assert parse_search_query('release:a\trelease') == [
SearchFilter(
key=SearchKey(name='release'),
operator='=',
value=SearchValue(raw_value='a'),
),
SearchFilter(key=SearchKey(name='message'),
operator='=',
value=SearchValue(raw_value='\trelease')),
]
# escaped quotes
assert parse_search_query('release:"a\"thing\""') == [
SearchFilter(key=SearchKey(name='release'),
operator='=',
value=SearchValue(raw_value='a"thing"')),
]
assert parse_search_query('release:"a\"\"release"') == [
SearchFilter(key=SearchKey(name='release'),
operator='=',
value=SearchValue(raw_value='a""release')),
]
# poorly escaped quotes
assert parse_search_query('release:"a release\"') == [
SearchFilter(key=SearchKey(name='release'),
operator='=',
value=SearchValue(raw_value='a release')),
]
assert parse_search_query('release:\"a release "') == [
SearchFilter(key=SearchKey(name='release'),
operator='=',
value=SearchValue(raw_value='a release ')),
]
def get_filter(query=None, params=None):
"""
Returns an eventstore filter given the search text provided by the user and
URL params
"""
# NOTE: this function assumes project permissions check already happened
parsed_terms = []
if query is not None:
try:
parsed_terms = parse_search_query(query,
allow_boolean=True,
params=params)
except ParseError as e:
raise InvalidSearchQuery(
f"Parse error: {e.expr.name} (column {e.column():d})")
kwargs = {
"start": None,
"end": None,
"conditions": [],
"having": [],
"user_id": None,
"organization_id": None,
"project_ids": [],
"group_ids": [],
"condition_aggregates": [],
"aliases": params.get("aliases", {}) if params is not None else {},
}
projects_to_filter = []
if any(
isinstance(term, ParenExpression)
or SearchBoolean.is_operator(term) for term in parsed_terms):
(
condition,
having,
found_projects_to_filter,
group_ids,
) = convert_search_boolean_to_snuba_query(parsed_terms, params)
if condition:
and_conditions = flatten_condition_tree(condition, SNUBA_AND)
for func in and_conditions:
kwargs["conditions"].append(
convert_function_to_condition(func))
if having:
kwargs["condition_aggregates"] = [
term.key.name for term in parsed_terms
if isinstance(term, AggregateFilter)
]
and_having = flatten_condition_tree(having, SNUBA_AND)
for func in and_having:
kwargs["having"].append(convert_function_to_condition(func))
if found_projects_to_filter:
projects_to_filter = list(set(found_projects_to_filter))
if group_ids is not None:
kwargs["group_ids"].extend(list(set(group_ids)))
else:
projects_to_filter = set()
for term in parsed_terms:
if isinstance(term, SearchFilter):
conditions, found_projects_to_filter, group_ids = format_search_filter(
term, params)
if len(conditions) > 0:
kwargs["conditions"].extend(conditions)
if found_projects_to_filter:
projects_to_filter.update(found_projects_to_filter)
if group_ids is not None:
kwargs["group_ids"].extend(group_ids)
elif isinstance(term, AggregateFilter):
converted_filter = convert_aggregate_filter_to_snuba_query(
term, params)
kwargs["condition_aggregates"].append(term.key.name)
if converted_filter:
kwargs["having"].append(converted_filter)
projects_to_filter = list(projects_to_filter)
# Keys included as url params take precedent if same key is included in search
# They are also considered safe and to have had access rules applied unlike conditions
# from the query string.
if params:
for key in ("start", "end"):
kwargs[key] = params.get(key, None)
# OrganizationEndpoint.get_filter() uses project_id, but eventstore.Filter uses project_ids
if "user_id" in params:
kwargs["user_id"] = params["user_id"]
if "organization_id" in params:
kwargs["organization_id"] = params["organization_id"]
if "project_id" in params:
if projects_to_filter:
kwargs["project_ids"] = projects_to_filter
else:
kwargs["project_ids"] = params["project_id"]
if "environment" in params:
term = SearchFilter(SearchKey("environment"), "=",
SearchValue(params["environment"]))
kwargs["conditions"].append(
convert_search_filter_to_snuba_query(term))
if "group_ids" in params:
kwargs["group_ids"] = to_list(params["group_ids"])
# Deprecated alias, use `group_ids` instead
if ISSUE_ID_ALIAS in params:
kwargs["group_ids"] = to_list(params["issue.id"])
return eventstore.Filter(**kwargs)
def test_timestamp(self):
# test date format
assert parse_search_query('timestamp>2015-05-18') == [
SearchFilter(
key=SearchKey(name='timestamp'),
operator=">",
value=SearchValue(
raw_value=datetime.datetime(
2015,
5,
18,
0,
0,
tzinfo=timezone.utc),
),
),
]
# test date time format
assert parse_search_query('timestamp>2015-05-18T10:15:01') == [
SearchFilter(
key=SearchKey(name='timestamp'),
operator=">",
value=SearchValue(
raw_value=datetime.datetime(
2015,
5,
18,
10,
15,
1,
tzinfo=timezone.utc),
),
),
]
# test date time format w microseconds
assert parse_search_query('timestamp>2015-05-18T10:15:01.103') == [
SearchFilter(
key=SearchKey(name='timestamp'),
operator=">",
value=SearchValue(
raw_value=datetime.datetime(
2015,
5,
18,
10,
15,
1,
103000,
tzinfo=timezone.utc),
),
),
]
# test date time format w microseconds and utc marker
assert parse_search_query('timestamp:>2015-05-18T10:15:01.103Z') == [
SearchFilter(
key=SearchKey(name='timestamp'),
operator=">",
value=SearchValue(
raw_value=datetime.datetime(
2015,
5,
18,
10,
15,
1,
103000,
tzinfo=timezone.utc),
),
),
]
def node_visitor(token):
if token["type"] == "spaces":
return None
if token["type"] == "filter":
# Filters with an invalid reason raises to signal to the test
# runner that we should expect this exception
if token.get("invalid"):
raise InvalidSearchQuery(token["invalid"]["reason"])
# Transform the operator to match for list values
if token["value"]["type"] in ["valueTextList", "valueNumberList"]:
operator = "NOT IN" if token["negated"] else "IN"
else:
# Negate the operator if the filter is negated to match
operator = token["operator"] or "="
operator = f"!{operator}" if token["negated"] else operator
key = node_visitor(token["key"])
value = node_visitor(token["value"])
if token["filter"] == "boolean" and token["negated"]:
operator = "="
value = SearchValue(raw_value=1 if value.raw_value == 0 else 0)
return SearchFilter(key, operator, value)
if token["type"] == "keySimple":
return SearchKey(name=token["value"])
if token["type"] == "keyExplicitTag":
return SearchKey(name=f"tags[{token['key']['value']}]")
if token["type"] == "keyAggregate":
name = node_visitor(token["name"]).name
# Consistent join aggregate function parameters
args = ", ".join(arg["value"]["value"]
for arg in token["args"]["args"])
return AggregateKey(name=f"{name}({args})")
if token["type"] == "valueText":
# Noramlize values by removing the escaped quotes
value = token["value"].replace('\\"', '"')
return SearchValue(raw_value=value)
if token["type"] == "valueNumber":
return SearchValue(
raw_value=parse_numeric_value(token["value"], token["unit"]))
if token["type"] == "valueTextList":
return SearchValue(
raw_value=[item["value"]["value"] for item in token["items"]])
if token["type"] == "valueNumberList":
return SearchValue(raw_value=[
item["value"]["rawValue"] for item in token["items"]
])
if token["type"] == "valueIso8601Date":
return SearchValue(raw_value=parse_datetime_string(token["value"]))
if token["type"] == "valueDuration":
return SearchValue(
raw_value=parse_duration(token["value"], token["unit"]))
if token["type"] == "valueRelativeDate":
return SearchValue(
raw_value=parse_duration(token["value"], token["unit"]))
if token["type"] == "valueBoolean":
return SearchValue(raw_value=int(token["value"]))
if token["type"] == "freeText":
if token["quoted"]:
# Normalize quotes
value = token["value"].replace('\\"', '"')
else:
# Normalize spacing
value = token["value"].strip(" ")
if value == "":
return None
return SearchFilter(
key=SearchKey(name="message"),
operator="=",
value=SearchValue(raw_value=value),
)
def test_invalid(self):
filters = [SearchFilter(SearchKey("status"), "=", SearchValue("wrong"))]
with self.assertRaises(InvalidSearchQuery, expected_regex="invalid status value"):
convert_query_values(filters, [self.project], self.user, None)
def test_multiple_statements(self):
assert parse_search_query(
'user.email:[email protected] OR user.email:[email protected] OR user.email:[email protected]'
) == [
SearchBoolean(left_term=self.term1,
operator='OR',
right_term=SearchBoolean(left_term=self.term2,
operator='OR',
right_term=self.term3))
]
assert parse_search_query(
'user.email:[email protected] AND user.email:[email protected] AND user.email:[email protected]'
) == [
SearchBoolean(left_term=self.term1,
operator='AND',
right_term=SearchBoolean(left_term=self.term2,
operator='AND',
right_term=self.term3))
]
term4 = SearchFilter(
key=SearchKey(name='user.email'),
operator="=",
value=SearchValue(raw_value='*****@*****.**'),
)
# longer even number of terms
assert parse_search_query(
'user.email:[email protected] AND user.email:[email protected] OR user.email:[email protected] AND user.email:[email protected]'
) == [
SearchBoolean(left_term=SearchBoolean(left_term=self.term1,
operator='AND',
right_term=self.term2),
operator='OR',
right_term=SearchBoolean(left_term=self.term3,
operator='AND',
right_term=term4))
]
term5 = SearchFilter(
key=SearchKey(name='user.email'),
operator="=",
value=SearchValue(raw_value='*****@*****.**'),
)
# longer odd number of terms
assert parse_search_query(
'user.email:[email protected] AND user.email:[email protected] OR user.email:[email protected] AND user.email:[email protected] AND user.email:[email protected]'
) == [
SearchBoolean(left_term=SearchBoolean(left_term=self.term1,
operator='AND',
right_term=self.term2),
operator='OR',
right_term=SearchBoolean(left_term=self.term3,
operator='AND',
right_term=SearchBoolean(
left_term=term4,
operator='AND',
right_term=term5)))
]
# absurdly long
assert parse_search_query(
'user.email:[email protected] AND user.email:[email protected] OR user.email:[email protected] AND user.email:[email protected] AND user.email:[email protected] OR user.email:[email protected] AND user.email:[email protected] OR user.email:[email protected] AND user.email:[email protected] AND user.email:[email protected]'
) == [
SearchBoolean(
left_term=SearchBoolean(left_term=self.term1,
operator='AND',
right_term=self.term2),
operator='OR',
right_term=SearchBoolean(
left_term=SearchBoolean(left_term=self.term3,
operator='AND',
right_term=SearchBoolean(
left_term=term4,
operator='AND',
right_term=term5)),
operator='OR',
right_term=SearchBoolean(
left_term=SearchBoolean(left_term=self.term1,
operator='AND',
right_term=self.term2),
operator='OR',
right_term=SearchBoolean(left_term=self.term3,
operator='AND',
right_term=SearchBoolean(
left_term=term4,
operator='AND',
right_term=term5)))))
]
