def test_translation(mappers: TranslationMappers, query: SnubaQuery, expected: ClickhouseQuery) -> None: translated = QueryTranslator(mappers).translate(query) # TODO: consider providing an __eq__ method to the Query class. Or turn it into # a dataclass. assert expected.get_selected_columns() == translated.get_selected_columns() assert expected.get_groupby() == translated.get_groupby() assert expected.get_condition() == translated.get_condition() assert expected.get_arrayjoin() == translated.get_arrayjoin() assert expected.get_having() == translated.get_having() assert expected.get_orderby() == translated.get_orderby()
def test_replace_expression() -> None: """ Create a query with the new AST and replaces a function with a different function replaces f1(...) with tag(f1) """ column1 = Column(None, "t1", "c1") column2 = Column(None, "t1", "c2") function_1 = FunctionCall("alias", "f1", (column1, column2)) function_2 = FunctionCall("alias", "f2", (column2,)) condition = binary_condition(ConditionFunctions.EQ, function_1, Literal(None, "1")) prewhere = binary_condition(ConditionFunctions.EQ, function_1, Literal(None, "2")) orderby = OrderBy(OrderByDirection.ASC, function_2) query = Query( Table("my_table", ColumnSet([])), selected_columns=[SelectedExpression("alias", function_1)], array_join=None, condition=condition, groupby=[function_1], having=None, prewhere=prewhere, order_by=[orderby], ) def replace(exp: Expression) -> Expression: if isinstance(exp, FunctionCall) and exp.function_name == "f1": return FunctionCall(exp.alias, "tag", (Literal(None, "f1"),)) return exp query.transform_expressions(replace) expected_query = Query( Table("my_table", ColumnSet([])), selected_columns=[ SelectedExpression( "alias", FunctionCall("alias", "tag", (Literal(None, "f1"),)) ) ], array_join=None, condition=binary_condition( ConditionFunctions.EQ, FunctionCall("alias", "tag", (Literal(None, "f1"),)), Literal(None, "1"), ), groupby=[FunctionCall("alias", "tag", (Literal(None, "f1"),))], prewhere=binary_condition( ConditionFunctions.EQ, FunctionCall("alias", "tag", (Literal(None, "f1"),)), Literal(None, "2"), ), having=None, order_by=[orderby], ) assert query.get_selected_columns() == expected_query.get_selected_columns() assert query.get_condition() == expected_query.get_condition() assert query.get_groupby() == expected_query.get_groupby() assert query.get_having() == expected_query.get_having() assert query.get_orderby() == expected_query.get_orderby() assert list(query.get_all_expressions()) == list( expected_query.get_all_expressions() )
def execute( self, query: Query, request_settings: RequestSettings, runner: SplitQueryRunner, ) -> Optional[QueryResult]: """ If a query is: - ORDER BY timestamp DESC - has no grouping - has an offset/limit - has a large time range We know we have to reverse-sort the entire set of rows to return the small chunk at the end of the time range, so optimistically split the time range into smaller increments, and start with the last one, so that we can potentially avoid querying the entire range. """ limit = query.get_limit() if limit is None or query.get_groupby(): return None if query.get_offset() >= 1000: return None orderby = query.get_orderby() if not orderby or orderby[0] != f"-{self.__timestamp_col}": return None conditions = query.get_conditions() or [] from_date_str = next( (condition[2] for condition in conditions if _identify_condition(condition, self.__timestamp_col, ">=")), None, ) to_date_str = next( (condition[2] for condition in conditions if _identify_condition(condition, self.__timestamp_col, "<")), None, ) from_date_ast, to_date_ast = get_time_range(query, self.__timestamp_col) if not from_date_str or not to_date_str: return None date_align, split_step = state.get_configs([("date_align_seconds", 1), ("split_step", 3600) ] # default 1 hour ) to_date = util.parse_datetime(to_date_str, date_align) from_date = util.parse_datetime(from_date_str, date_align) if from_date != from_date_ast: logger.warning( "Mismatch in start date on time splitter.", extra={ "ast": str(from_date_ast), "legacy": str(from_date) }, exc_info=True, ) metrics.increment("mismatch.ast_from_date") remaining_offset = query.get_offset() overall_result = None split_end = to_date split_start = max(split_end - timedelta(seconds=split_step), from_date) total_results = 0 while split_start < split_end and total_results < limit: # We need to make a copy to use during the query execution because we replace # the start-end conditions on the query at each iteration of this loop. split_query = copy.deepcopy(query) _replace_condition(split_query, self.__timestamp_col, ">=", split_start.isoformat()) _replace_ast_condition(split_query, self.__timestamp_col, ">=", LiteralExpr(None, split_start)) _replace_condition(split_query, self.__timestamp_col, "<", split_end.isoformat()) _replace_ast_condition(split_query, self.__timestamp_col, "<", LiteralExpr(None, split_end)) # Because its paged, we have to ask for (limit+offset) results # and set offset=0 so we can then trim them ourselves. split_query.set_offset(0) split_query.set_limit(limit - total_results + remaining_offset) # At every iteration we only append the "data" key from the results returned by # the runner. The "extra" key is only populated at the first iteration of the # loop and never changed. result = runner(split_query, request_settings) if overall_result is None: overall_result = result else: overall_result.result["data"].extend(result.result["data"]) if remaining_offset > 0 and len(overall_result.result["data"]) > 0: to_trim = min(remaining_offset, len(overall_result.result["data"])) overall_result.result["data"] = overall_result.result["data"][ to_trim:] remaining_offset -= to_trim total_results = len(overall_result.result["data"]) if total_results < limit: if len(result.result["data"]) == 0: # If we got nothing from the last query, expand the range by a static factor split_step = split_step * STEP_GROWTH else: # If we got some results but not all of them, estimate how big the time # range should be for the next query based on how many results we got for # our last query and its time range, and how many we have left to fetch. remaining = limit - total_results split_step = split_step * math.ceil( remaining / float(len(result.result["data"]))) # Set the start and end of the next query based on the new range. split_end = split_start try: split_start = max( split_end - timedelta(seconds=split_step), from_date) except OverflowError: split_start = from_date return overall_result
def execute( self, query: Query, query_settings: QuerySettings, runner: SplitQueryRunner, ) -> Optional[QueryResult]: """ If a query is: - ORDER BY timestamp DESC - has no grouping - has an offset/limit - has a large time range We know we have to reverse-sort the entire set of rows to return the small chunk at the end of the time range, so optimistically split the time range into smaller increments, and start with the last one, so that we can potentially avoid querying the entire range. """ limit = query.get_limit() if limit is None or query.get_groupby(): return None if query.get_offset() >= 1000: return None orderby = query.get_orderby() if (not orderby or orderby[0].direction != OrderByDirection.DESC or not isinstance(orderby[0].expression, ColumnExpr) or not orderby[0].expression.column_name == self.__timestamp_col): return None from_date_ast, to_date_ast = get_time_range(query, self.__timestamp_col) if from_date_ast is None or to_date_ast is None: return None date_align, split_step = state.get_configs([("date_align_seconds", 1), ("split_step", 3600) ] # default 1 hour ) assert isinstance(split_step, int) remaining_offset = query.get_offset() overall_result: Optional[QueryResult] = None split_end = to_date_ast split_start = max(split_end - timedelta(seconds=split_step), from_date_ast) total_results = 0 while split_start < split_end and total_results < limit: # We need to make a copy to use during the query execution because we replace # the start-end conditions on the query at each iteration of this loop. split_query = copy.deepcopy(query) _replace_ast_condition(split_query, self.__timestamp_col, ">=", LiteralExpr(None, split_start)) _replace_ast_condition(split_query, self.__timestamp_col, "<", LiteralExpr(None, split_end)) # Because its paged, we have to ask for (limit+offset) results # and set offset=0 so we can then trim them ourselves. split_query.set_offset(0) split_query.set_limit(limit - total_results + remaining_offset) # At every iteration we only append the "data" key from the results returned by # the runner. The "extra" key is only populated at the first iteration of the # loop and never changed. result = runner(split_query, query_settings) if overall_result is None: overall_result = result else: overall_result.result["data"].extend(result.result["data"]) if remaining_offset > 0 and len(overall_result.result["data"]) > 0: to_trim = min(remaining_offset, len(overall_result.result["data"])) overall_result.result["data"] = overall_result.result["data"][ to_trim:] remaining_offset -= to_trim total_results = len(overall_result.result["data"]) if total_results < limit: if len(result.result["data"]) == 0: # If we got nothing from the last query, expand the range by a static factor split_step = split_step * STEP_GROWTH else: # If we got some results but not all of them, estimate how big the time # range should be for the next query based on how many results we got for # our last query and its time range, and how many we have left to fetch. remaining = limit - total_results split_step = split_step * math.ceil( remaining / float(len(result.result["data"]))) # Set the start and end of the next query based on the new range. split_end = split_start try: split_start = max( split_end - timedelta(seconds=split_step), from_date_ast) except OverflowError: split_start = from_date_ast return overall_result