def test_edit_query():
query = Query({
"selected_columns": ["c1", "c2", "c3"],
"conditions": [["c1", "=", "a"]],
"groupby": ["project_id"],
"aggregations": [["count()", "", "count"]],
"orderby": "event_id",
"sample": 10,
"limit": 100,
"offset": 50,
})
query.set_selected_columns(["c4"])
assert query.get_selected_columns() == ["c4"]
query.set_aggregations([["different_agg()", "", "something"]])
assert query.get_aggregations() == [["different_agg()", "", "something"]]
query.add_groupby(["more", "more2"])
assert query.get_groupby() == ["project_id", "more", "more2"]
query.add_conditions([["c5", "=", "9"]])
assert query.get_conditions() == [
["c1", "=", "a"],
["c5", "=", "9"],
]
query.set_conditions([["c6", "=", "10"]])
assert query.get_conditions() == [
["c6", "=", "10"],
]
def test_edit_query():
query = Query(
{
"selected_columns": ["c1", "c2", "c3"],
"conditions": [["c1", "=", "a"]],
"arrayjoin": "tags",
"having": [["c4", "=", "c"]],
"groupby": ["project_id"],
"aggregations": [["count()", "", "count"]],
"orderby": "event_id",
"limitby": (100, "environment"),
"sample": 10,
"limit": 100,
"offset": 50,
"totals": True,
},
TableSource("my_table", ColumnSet([])),
)
query.set_selected_columns(["c4"])
assert query.get_selected_columns() == ["c4"]
query.set_aggregations([["different_agg()", "", "something"]])
assert query.get_aggregations() == [["different_agg()", "", "something"]]
query.add_groupby(["more", "more2"])
assert query.get_groupby() == ["project_id", "more", "more2"]
query.add_conditions([["c5", "=", "9"]])
assert query.get_conditions() == [
["c1", "=", "a"],
["c5", "=", "9"],
]
query.set_conditions([["c6", "=", "10"]])
assert query.get_conditions() == [
["c6", "=", "10"],
]
query.set_arrayjoin("not_tags")
assert query.get_arrayjoin() == "not_tags"
query.set_granularity(7200)
assert query.get_granularity() == 7200
query.set_prewhere([["pc6", "=", "10"]])
assert query.get_prewhere() == [["pc6", "=", "10"]]
def process_query(self, query: Query, request_settings: RequestSettings,) -> None:
max_prewhere_conditions: int = (
self.__max_prewhere_conditions or settings.MAX_PREWHERE_CONDITIONS
)
prewhere_keys = query.get_data_source().get_prewhere_candidates()
if not prewhere_keys:
return
prewhere_conditions: Sequence[Condition] = []
# Add any condition to PREWHERE if:
# - It is a single top-level condition (not OR-nested), and
# - Any of its referenced columns are in prewhere_keys
conditions = query.get_conditions()
if not conditions:
return
prewhere_candidates = [
(util.columns_in_expr(cond[0]), cond)
for cond in conditions
if util.is_condition(cond)
and any(col in prewhere_keys for col in util.columns_in_expr(cond[0]))
]
# Use the condition that has the highest priority (based on the
# position of its columns in the prewhere keys list)
prewhere_candidates = sorted(
[
(
min(
prewhere_keys.index(col) for col in cols if col in prewhere_keys
),
cond,
)
for cols, cond in prewhere_candidates
],
key=lambda priority_and_col: priority_and_col[0],
)
if prewhere_candidates:
prewhere_conditions = [cond for _, cond in prewhere_candidates][
:max_prewhere_conditions
]
query.set_conditions(
list(filter(lambda cond: cond not in prewhere_conditions, conditions))
)
query.set_prewhere(prewhere_conditions)
def process_query(self, query: Query,
request_settings: RequestSettings) -> None:
conditions = query.get_conditions()
if not conditions:
return
# Enable the processor only if we have enough data in the flattened
# columns. Which have been deployed at BEGINNING_OF_TIME. If the query
# starts earlier than that we do not apply the optimization.
if self.__beginning_of_time:
apply_optimization = False
for condition in conditions:
if (is_condition(condition) and isinstance(condition[0], str)
and condition[0] in self.__timestamp_cols
and condition[1] in (">=", ">")
and isinstance(condition[2], str)):
try:
start_ts = parse_datetime(condition[2])
if (start_ts -
self.__beginning_of_time).total_seconds() > 0:
apply_optimization = True
except Exception:
# We should not get here, it means the from timestamp is malformed
# Returning here is just for safety
logger.error(
"Cannot parse start date for NestedFieldOptimizer: %r",
condition,
)
return
if not apply_optimization:
return
# Do not use flattened tags if tags are being unpacked anyway. In that case
# using flattened tags only implies loading an additional column thus making
# the query heavier and slower
if self.__has_tags(query.get_arrayjoin_from_ast()):
return
if query.get_groupby_from_ast():
for expression in query.get_groupby_from_ast():
if self.__has_tags(expression):
return
if self.__has_tags(query.get_having_from_ast()):
return
if query.get_orderby_from_ast():
for orderby in query.get_orderby_from_ast():
if self.__has_tags(orderby.expression):
return
new_conditions = []
positive_like_expression: List[str] = []
negative_like_expression: List[str] = []
for c in conditions:
keyvalue = self.__is_optimizable(c, self.__nested_col)
if not keyvalue:
new_conditions.append(c)
else:
expression = f"{escape_field(keyvalue.nested_col_key)}={escape_field(keyvalue.value)}"
if keyvalue.operand == Operand.EQ:
positive_like_expression.append(expression)
else:
negative_like_expression.append(expression)
if positive_like_expression:
# Positive conditions "=" are all merged together in one LIKE expression
positive_like_expression = sorted(positive_like_expression)
like_formatted = f"%|{'|%|'.join(positive_like_expression)}|%"
new_conditions.append(
[self.__flattened_col, "LIKE", like_formatted])
for expression in negative_like_expression:
# Negative conditions "!=" cannot be merged together. We can still transform
# them into NOT LIKE statements, but each condition has to be one
# statement.
not_like_formatted = f"%|{expression}|%"
new_conditions.append(
[self.__flattened_col, "NOT LIKE", not_like_formatted])
query.set_conditions(new_conditions)
