def __init__(self, query: Query, settings: RequestSettings,) -> None:
# Clickhouse query structure
# Referencing them here directly since it makes it easier
# to process this query independently from the Clickhouse Query
# and there is no risk in doing so since they are immutable.
self.__selected_columns = query.get_selected_columns_from_ast()
self.__condition = query.get_condition_from_ast()
self.__groupby = query.get_groupby_from_ast()
self.__having = query.get_having_from_ast()
self.__orderby = query.get_orderby_from_ast()
self.__data_source = query.get_data_source()
self.__arrayjoin = query.get_arrayjoin_from_ast()
self.__granularity = query.get_granularity()
self.__limit = query.get_limit()
self.__limitby = query.get_limitby()
self.__offset = query.get_offset()
if self.__having:
assert self.__groupby, "found HAVING clause with no GROUP BY"
self.__turbo = settings.get_turbo()
self.__final = query.get_final()
self.__sample = query.get_sample()
self.__hastotals = query.has_totals()
self.__prewhere = query.get_prewhere_ast()
self.__settings = settings
self.__sql_data_list: Optional[Sequence[Tuple[str, str]]] = None
self.__formatted_query: Optional[str] = None
self.__sql_data: Optional[Mapping[str, str]] = None
def test_query_parameters() -> None:
query = Query(
Table("my_table", ColumnSet([])),
limitby=(100, "environment"),
limit=100,
offset=50,
totals=True,
granularity=60,
)
assert query.get_limitby() == (100, "environment")
assert query.get_limit() == 100
assert query.get_offset() == 50
assert query.has_totals() is True
assert query.get_granularity() == 60
assert query.get_from_clause().table_name == "my_table"
def execute(
self,
query: Query,
request_settings: RequestSettings,
runner: SplitQueryRunner,
) -> Optional[QueryResult]:
"""
If a query is:
- ORDER BY timestamp DESC
- has no grouping
- has an offset/limit
- has a large time range
We know we have to reverse-sort the entire set of rows to return the small
chunk at the end of the time range, so optimistically split the time range
into smaller increments, and start with the last one, so that we can potentially
avoid querying the entire range.
"""
limit = query.get_limit()
if limit is None or query.get_groupby():
return None
if query.get_offset() >= 1000:
return None
orderby = query.get_orderby()
if not orderby or orderby[0] != f"-{self.__timestamp_col}":
return None
conditions = query.get_conditions() or []
from_date_str = next(
(condition[2] for condition in conditions
if _identify_condition(condition, self.__timestamp_col, ">=")),
None,
)
to_date_str = next(
(condition[2] for condition in conditions
if _identify_condition(condition, self.__timestamp_col, "<")),
None,
)
from_date_ast, to_date_ast = get_time_range(query,
self.__timestamp_col)
if not from_date_str or not to_date_str:
return None
date_align, split_step = state.get_configs([("date_align_seconds", 1),
("split_step", 3600)
] # default 1 hour
)
to_date = util.parse_datetime(to_date_str, date_align)
from_date = util.parse_datetime(from_date_str, date_align)
if from_date != from_date_ast:
logger.warning(
"Mismatch in start date on time splitter.",
extra={
"ast": str(from_date_ast),
"legacy": str(from_date)
},
exc_info=True,
)
metrics.increment("mismatch.ast_from_date")
remaining_offset = query.get_offset()
overall_result = None
split_end = to_date
split_start = max(split_end - timedelta(seconds=split_step), from_date)
total_results = 0
while split_start < split_end and total_results < limit:
# We need to make a copy to use during the query execution because we replace
# the start-end conditions on the query at each iteration of this loop.
split_query = copy.deepcopy(query)
_replace_condition(split_query, self.__timestamp_col, ">=",
split_start.isoformat())
_replace_ast_condition(split_query, self.__timestamp_col, ">=",
LiteralExpr(None, split_start))
_replace_condition(split_query, self.__timestamp_col, "<",
split_end.isoformat())
_replace_ast_condition(split_query, self.__timestamp_col, "<",
LiteralExpr(None, split_end))
# Because its paged, we have to ask for (limit+offset) results
# and set offset=0 so we can then trim them ourselves.
split_query.set_offset(0)
split_query.set_limit(limit - total_results + remaining_offset)
# At every iteration we only append the "data" key from the results returned by
# the runner. The "extra" key is only populated at the first iteration of the
# loop and never changed.
result = runner(split_query, request_settings)
if overall_result is None:
overall_result = result
else:
overall_result.result["data"].extend(result.result["data"])
if remaining_offset > 0 and len(overall_result.result["data"]) > 0:
to_trim = min(remaining_offset,
len(overall_result.result["data"]))
overall_result.result["data"] = overall_result.result["data"][
to_trim:]
remaining_offset -= to_trim
total_results = len(overall_result.result["data"])
if total_results < limit:
if len(result.result["data"]) == 0:
# If we got nothing from the last query, expand the range by a static factor
split_step = split_step * STEP_GROWTH
else:
# If we got some results but not all of them, estimate how big the time
# range should be for the next query based on how many results we got for
# our last query and its time range, and how many we have left to fetch.
remaining = limit - total_results
split_step = split_step * math.ceil(
remaining / float(len(result.result["data"])))
# Set the start and end of the next query based on the new range.
split_end = split_start
try:
split_start = max(
split_end - timedelta(seconds=split_step), from_date)
except OverflowError:
split_start = from_date
return overall_result
def execute(
self,
query: Query,
request_settings: RequestSettings,
runner: SplitQueryRunner,
) -> Optional[QueryResult]:
"""
If a query is:
- ORDER BY timestamp DESC
- has no grouping
- has an offset/limit
- has a large time range
We know we have to reverse-sort the entire set of rows to return the small
chunk at the end of the time range, so optimistically split the time range
into smaller increments, and start with the last one, so that we can potentially
avoid querying the entire range.
"""
limit = query.get_limit()
if limit is None or query.get_groupby_from_ast():
return None
if query.get_offset() >= 1000:
return None
orderby = query.get_orderby_from_ast()
if (not orderby or orderby[0].direction != OrderByDirection.DESC
or not isinstance(orderby[0].expression, ColumnExpr) or
not orderby[0].expression.column_name == self.__timestamp_col):
return None
from_date_ast, to_date_ast = get_time_range(query,
self.__timestamp_col)
if from_date_ast is None or to_date_ast is None:
return None
date_align, split_step = state.get_configs([("date_align_seconds", 1),
("split_step", 3600)
] # default 1 hour
)
assert isinstance(split_step, int)
remaining_offset = query.get_offset()
overall_result: Optional[QueryResult] = None
split_end = to_date_ast
split_start = max(split_end - timedelta(seconds=split_step),
from_date_ast)
total_results = 0
while split_start < split_end and total_results < limit:
# We need to make a copy to use during the query execution because we replace
# the start-end conditions on the query at each iteration of this loop.
split_query = copy.deepcopy(query)
_replace_ast_condition(split_query, self.__timestamp_col, ">=",
LiteralExpr(None, split_start))
_replace_ast_condition(split_query, self.__timestamp_col, "<",
LiteralExpr(None, split_end))
# Because its paged, we have to ask for (limit+offset) results
# and set offset=0 so we can then trim them ourselves.
split_query.set_offset(0)
split_query.set_limit(limit - total_results + remaining_offset)
# At every iteration we only append the "data" key from the results returned by
# the runner. The "extra" key is only populated at the first iteration of the
# loop and never changed.
result = runner(split_query, request_settings)
if overall_result is None:
overall_result = result
else:
overall_result.result["data"].extend(result.result["data"])
if remaining_offset > 0 and len(overall_result.result["data"]) > 0:
to_trim = min(remaining_offset,
len(overall_result.result["data"]))
overall_result.result["data"] = overall_result.result["data"][
to_trim:]
remaining_offset -= to_trim
total_results = len(overall_result.result["data"])
if total_results < limit:
if len(result.result["data"]) == 0:
# If we got nothing from the last query, expand the range by a static factor
split_step = split_step * STEP_GROWTH
else:
# If we got some results but not all of them, estimate how big the time
# range should be for the next query based on how many results we got for
# our last query and its time range, and how many we have left to fetch.
remaining = limit - total_results
split_step = split_step * math.ceil(
remaining / float(len(result.result["data"])))
# Set the start and end of the next query based on the new range.
split_end = split_start
try:
split_start = max(
split_end - timedelta(seconds=split_step),
from_date_ast)
except OverflowError:
split_start = from_date_ast
return overall_result
