def test_first_level_conditions() -> None:
c1 = binary_condition(
ConditionFunctions.EQ,
Column(None, "table1", "column1"),
Literal(None, "test"),
)
c2 = binary_condition(
ConditionFunctions.EQ,
Column(None, "table2", "column2"),
Literal(None, "test"),
)
c3 = binary_condition(
ConditionFunctions.EQ,
Column(None, "table3", "column3"),
Literal(None, "test"),
)
cond = binary_condition(
BooleanFunctions.AND,
binary_condition(BooleanFunctions.AND, c1, c2),
c3,
)
assert get_first_level_and_conditions(cond) == [c1, c2, c3]
cond = binary_condition(
BooleanFunctions.AND,
FunctionCall(None, "equals",
(FunctionCall(None, "and", (c1, c2)), Literal(None, 1))),
c3,
)
assert get_first_level_and_conditions(cond) == [c1, c2, c3]
cond = binary_condition(
BooleanFunctions.OR,
binary_condition(BooleanFunctions.AND, c1, c2),
c3,
)
assert get_first_level_or_conditions(cond) == [
binary_condition(BooleanFunctions.AND, c1, c2),
c3,
]
cond = binary_condition(
ConditionFunctions.EQ,
binary_condition(BooleanFunctions.OR, c1,
binary_condition(BooleanFunctions.AND, c2, c3)),
Literal(None, 1),
)
assert get_first_level_or_conditions(cond) == [
c1,
binary_condition(BooleanFunctions.AND, c2, c3),
]
def visit_function_call(self, exp: FunctionCall) -> str:
if exp.function_name == "array":
# Workaround for https://github.com/ClickHouse/ClickHouse/issues/11622
# Some distributed queries fail when arrays are passed as array(1,2,3)
# and work when they are passed as [1, 2, 3]
return self._alias(f"[{self.__visit_params(exp.parameters)}]",
exp.alias)
if exp.function_name == "tuple" and len(exp.parameters) > 1:
# Some distributed queries fail when tuples are passed as tuple(1,2,3)
# and work when they are passed as (1, 2, 3)
# to be safe, only do this for when a tuple has more than one element otherwise clickhouse
# will interpret (1) -> 1 which will break things like 1 IN tuple(1)
return self._alias(f"({self.__visit_params(exp.parameters)})",
exp.alias)
elif exp.function_name == BooleanFunctions.AND:
formatted = (c.accept(self)
for c in get_first_level_and_conditions(exp))
return " AND ".join(formatted)
elif exp.function_name == BooleanFunctions.OR:
formatted = (c.accept(self)
for c in get_first_level_or_conditions(exp))
return f"({' OR '.join(formatted)})"
ret = f"{escape_identifier(exp.function_name)}({self.__visit_params(exp.parameters)})"
return self._alias(ret, exp.alias)
def is_skippable_condition(conditions: Expression) -> bool:
"""
A condition composed of a bunch of has(column, ...) conditions OR'ed together
can be ignored when looking for filter keys because these are the conditions
used for the bloom filter index on the array column.
"""
for column_name in column_names:
has_pattern = FunctionCall(
String("has"),
(Column(column_name=String(column_name)), Literal(Any(str))),
)
if all(
has_pattern.match(c) for c in get_first_level_or_conditions(conditions)
):
return True
return False
def visit_function_call(self, exp: FunctionCall) -> str:
if exp.function_name == "array":
# Workaround for https://github.com/ClickHouse/ClickHouse/issues/11622
# Some distributed queries fail when arrays are passed as array(1,2,3)
# and work when they are passed as [1, 2, 3]
return self._alias(f"[{self.__visit_params(exp.parameters)}]", exp.alias)
elif exp.function_name == BooleanFunctions.AND:
formatted = (c.accept(self) for c in get_first_level_and_conditions(exp))
return " AND ".join(formatted)
elif exp.function_name == BooleanFunctions.OR:
formatted = (c.accept(self) for c in get_first_level_or_conditions(exp))
return f"({' OR '.join(formatted)})"
ret = f"{escape_identifier(exp.function_name)}({self.__visit_params(exp.parameters)})"
return self._alias(ret, exp.alias)
def __classify_combined_conditions(self, condition: Expression) -> ConditionClass:
if not isinstance(condition, FunctionExpr):
return ConditionClass.IRRELEVANT
elif condition.function_name in (BooleanFunctions.AND, BooleanFunctions.OR):
conditions = (
get_first_level_and_conditions(condition)
if condition.function_name == BooleanFunctions.AND
else get_first_level_or_conditions(condition)
)
classified = {self.__classify_combined_conditions(c) for c in conditions}
if ConditionClass.NOT_OPTIMIZABLE in classified:
return ConditionClass.NOT_OPTIMIZABLE
elif ConditionClass.OPTIMIZABLE in classified:
return ConditionClass.OPTIMIZABLE
else:
return ConditionClass.IRRELEVANT
else:
return self.__classify_condition(condition)
def test_first_level_conditions() -> None:
c1 = binary_condition(
None,
ConditionFunctions.EQ,
Column(None, "table1", "column1"),
Literal(None, "test"),
)
c2 = binary_condition(
None,
ConditionFunctions.EQ,
Column(None, "table2", "column2"),
Literal(None, "test"),
)
c3 = binary_condition(
None,
ConditionFunctions.EQ,
Column(None, "table3", "column3"),
Literal(None, "test"),
)
cond = binary_condition(
None,
BooleanFunctions.AND,
binary_condition(None, BooleanFunctions.AND, c1, c2),
c3,
)
assert get_first_level_and_conditions(cond) == [c1, c2, c3]
cond = binary_condition(
None,
BooleanFunctions.OR,
binary_condition(None, BooleanFunctions.AND, c1, c2),
c3,
)
assert get_first_level_or_conditions(cond) == [
binary_condition(None, BooleanFunctions.AND, c1, c2),
c3,
]
def _get_condition_without_redundant_checks(
self, condition: Expression, query: Query
) -> Expression:
"""Optimizes the case where the query condition contains the following:
valueOf('my_tag') != '' AND valueOf('my_tag') == "something"
^ ^
| |
existence check value check
the existence check in this clause is redundant and prevents the hashmap
optimization from being applied.
This function will remove all tag existence checks
from the condition IFF they are ANDed with a value check for the *same tag name*
Side effects:
This function works by flattening first level AND conditions to find clauses where
existence checks and value checks are ANDed together. When the AND conditions are recombined,
they are not guaranteed to be in the same structure (but are guaranteed to be functionally equivalent)
Example:
┌───┐ ┌───┐
│AND│ │AND│
├──┬┘ └┬──┤
│ │ │ │
┌──┴┐ c a ┌┴──┐
│AND│ becomes │AND│
└┬─┬┘ ├──┬┘
│ │ │ │
a b b c
"""
if not isinstance(condition, FunctionExpr):
return condition
elif condition.function_name == BooleanFunctions.OR:
sub_conditions = get_first_level_or_conditions(condition)
pruned_conditions = [
self._get_condition_without_redundant_checks(c, query)
for c in sub_conditions
]
return combine_or_conditions(pruned_conditions)
elif condition.function_name == BooleanFunctions.AND:
sub_conditions = get_first_level_and_conditions(condition)
tag_eq_match_strings = set()
matched_tag_exists_conditions = {}
for condition_id, cond in enumerate(sub_conditions):
tag_exist_match = None
for tag_exists_pattern in self.__tag_exists_patterns:
tag_exist_match = tag_exists_pattern.match(cond)
if tag_exist_match:
matched_tag_exists_conditions[condition_id] = tag_exist_match
if not tag_exist_match:
eq_match = self.__optimizable_pattern.match(cond)
if eq_match:
tag_eq_match_strings.add(eq_match.string(KEY_MAPPING_PARAM))
useful_conditions = []
for condition_id, cond in enumerate(sub_conditions):
tag_exist_match = matched_tag_exists_conditions.get(condition_id, None)
if tag_exist_match:
requested_tag = tag_exist_match.string("key")
if requested_tag in tag_eq_match_strings:
# the clause is redundant, thus we continue the loop
# and do not add it to useful_conditions
continue
useful_conditions.append(
self._get_condition_without_redundant_checks(cond, query)
)
return combine_and_conditions(useful_conditions)
else:
return condition