def test_having_special_case(input_query: ClickhouseQuery,
expected_query: ClickhouseQuery) -> None:
MappingOptimizer(
column_name="tags",
hash_map_name="_tags_hash_map",
killswitch="tags_hash_map_enabled",
).process_query(input_query, HTTPQuerySettings())
assert input_query == expected_query
def test_tags_hash_map(query: ClickhouseQuery, expected_condition: Expression,) -> None:
set_config("tags_hash_map_enabled", 1)
MappingOptimizer(
column_name="tags",
hash_map_name="_tags_hash_map",
killswitch="tags_hash_map_enabled",
).process_query(query, HTTPRequestSettings())
assert query.get_condition() == expected_condition
def test_recursive_useless_condition(
input_query: ClickhouseQuery,
expected_query: ClickhouseQuery,
) -> None:
# copy the condition to the having condition so that we test both being
# applied in one test
input_query.set_ast_having(deepcopy(input_query.get_condition()))
expected_query.set_ast_having(deepcopy(expected_query.get_condition()))
MappingOptimizer(
column_name="tags",
hash_map_name="_tags_hash_map",
killswitch="tags_hash_map_enabled",
).process_query(input_query, HTTPQuerySettings())
assert input_query == expected_query
def test_useless_has_condition(
input_query: ClickhouseQuery,
expected_query: ClickhouseQuery,
) -> None:
from snuba.query.processors.empty_tag_condition_processor import (
EmptyTagConditionProcessor, )
# change the existence expression to be a has(tags, 'my_tag') expression for boh queries
# this allows reuse of the previous test cases
EmptyTagConditionProcessor().process_query(input_query,
HTTPQuerySettings())
EmptyTagConditionProcessor().process_query(expected_query,
HTTPQuerySettings())
MappingOptimizer(
column_name="tags",
hash_map_name="_tags_hash_map",
killswitch="tags_hash_map_enabled",
).process_query(input_query, HTTPQuerySettings())
assert input_query == expected_query
# during create statement
# (https://github.com/ClickHouse/ClickHouse/issues/12586), so the
# materialization is added with a migration.
skipped_cols_on_creation={"_tags_hash_map"},
)
storage = WritableTableStorage(
storage_key=StorageKey.TRANSACTIONS,
storage_set_key=StorageSetKey.TRANSACTIONS,
schema=schema,
query_processors=[
NestedFieldConditionOptimizer(
"contexts",
"_contexts_flattened",
{"start_ts", "finish_ts"},
BEGINNING_OF_TIME,
),
MappingOptimizer("tags", "_tags_hash_map", "tags_hash_map_enabled"),
TransactionColumnProcessor(),
ArrayJoinKeyValueOptimizer("tags"),
ArrayJoinKeyValueOptimizer("measurements"),
PrewhereProcessor(),
],
stream_loader=KafkaStreamLoader(
processor=TransactionsMessageProcessor(), default_topic="events",
),
query_splitters=[TimeSplitQueryStrategy(timestamp_col="finish_ts")],
writer_options={"insert_allow_materialized_columns": 1},
)