def test_in_condition() -> None: in_condition = binary_condition( None, ConditionFunctions.IN, Column(None, None, "tags_key"), literals_tuple( None, [Literal(None, "t1"), Literal(None, "t2")]), ) assert is_in_condition(in_condition) match = is_in_condition_pattern(ColumnPattern( None, String("tags_key"))).match(in_condition) assert match is not None assert match.expression("tuple") == literals_tuple( None, [Literal(None, "t1"), Literal(None, "t2")]) assert match.expression("lhs") == Column(None, None, "tags_key")
def test_aliased_expressions_from_basic_condition() -> None: """ Iterates over the expressions in a basic condition when those expressions are aliased f(t1.c1) as a = t1.c2 as a2 """ c = Column(None, "t1", "c1") f1 = FunctionCall("a", "f", (c, )) c2 = Column("a2", "t1", "c2") condition = binary_condition(ConditionFunctions.EQ, f1, c2) ret = list(condition) expected = [c, f1, c2, condition] assert ret == expected
def test_iterate_over_query(): """ Creates a query with the new AST and iterate over all expressions. """ column1 = Column(None, "c1", "t1") column2 = Column(None, "c2", "t1") function_1 = FunctionCall("alias", "f1", (column1, column2)) function_2 = FunctionCall("alias", "f2", (column2,)) condition = binary_condition( None, ConditionFunctions.EQ, column1, Literal(None, "1") ) orderby = OrderBy(OrderByDirection.ASC, function_2) query = Query( {}, TableSource("my_table", ColumnSet([])), selected_columns=[function_1], array_join=None, condition=condition, groupby=[function_1], having=None, order_by=[orderby], ) expected_expressions = [ # selected columns column1, column2, function_1, # condition column1, Literal(None, "1"), condition, # groupby column1, column2, function_1, # order by column2, function_2, ] assert list(query.get_all_expressions()) == expected_expressions
def test_failure_rate_format_expressions() -> None: unprocessed = Query( QueryEntity(EntityKey.EVENTS, ColumnSet([])), selected_columns=[ SelectedExpression(name=None, expression=Column(None, None, "column2")), SelectedExpression("perf", FunctionCall("perf", "failure_rate", ())), ], ) expected = Query( QueryEntity(EntityKey.EVENTS, ColumnSet([])), selected_columns=[ SelectedExpression(name=None, expression=Column(None, None, "column2")), SelectedExpression( "perf", divide( FunctionCall( None, "countIf", (combine_and_conditions([ binary_condition( ConditionFunctions.NEQ, Column(None, None, "transaction_status"), Literal(None, code), ) for code in [0, 1, 2] ]), ), ), count(), "perf", ), ), ], ) failure_rate_processor().process_query(unprocessed, HTTPQuerySettings()) assert expected.get_selected_columns() == unprocessed.get_selected_columns( ) ret = unprocessed.get_selected_columns()[1].expression.accept( ClickhouseExpressionFormatter()) assert ret == ( "(divide(countIf(notEquals(transaction_status, 0) AND notEquals(transaction_status, 1) AND notEquals(transaction_status, 2)), count()) AS perf)" )
def simple_condition_builder(lhs: Expression, op: str, literal: Any) -> Expression: if op in UNARY_OPERATORS: if literal is not None: raise ParsingException( f"Right hand side operand {literal} provided to unary operator {op}", report=False, ) return unary_condition(OPERATOR_TO_FUNCTION[op], lhs) else: if literal is None: raise ParsingException( f"Missing right hand side operand for binary operator {op}", report=False, ) return binary_condition( OPERATOR_TO_FUNCTION[op], lhs, preprocess_literal(op, literal) )
def test_is_x_condition_functions() -> None: eq_condition = binary_condition( ConditionFunctions.EQ, Column(None, None, "test"), Literal(None, "1") ) assert is_any_binary_condition(eq_condition, ConditionFunctions.EQ) assert not is_any_binary_condition(eq_condition, ConditionFunctions.NEQ) un_condition = unary_condition( ConditionFunctions.IS_NOT_NULL, Column(None, None, "test") ) assert is_unary_condition(un_condition, ConditionFunctions.IS_NOT_NULL) assert not is_unary_condition(un_condition, ConditionFunctions.IS_NULL) assert not is_unary_condition(eq_condition, ConditionFunctions.IS_NOT_NULL) almost_condition = FunctionCall(None, "isNotNullish", (Column(None, None, "test"),)) assert is_condition(eq_condition) assert is_condition(un_condition) assert not is_condition(almost_condition)
def test_invalid_datetime() -> None: unprocessed = Query( QueryEntity(EntityKey.EVENTS, ColumnSet([])), selected_columns=[ SelectedExpression( "transaction.duration", Column("transaction.duration", None, "duration") ), ], condition=binary_condition( ConditionFunctions.EQ, Column("my_time", None, "time"), Literal(None, ""), ), ) entity = TransactionsEntity() processors = entity.get_query_processors() for processor in processors: if isinstance(processor, TimeSeriesProcessor): with pytest.raises(InvalidQueryException): processor.process_query(unprocessed, HTTPRequestSettings())
def test_binary_match() -> None: c1 = binary_condition( ConditionFunctions.EQ, Column(None, "table1", "column1"), Literal(None, "test"), ) lhs = ColumnPattern(String("table1"), String("column1")) rhs = LiteralPattern(String("test")) assert ( condition_pattern({ConditionFunctions.EQ}, lhs, rhs, True).match(c1) is not None ) assert ( condition_pattern({ConditionFunctions.EQ}, lhs, rhs, False).match(c1) is not None ) assert ( condition_pattern({ConditionFunctions.EQ}, rhs, lhs, True).match(c1) is not None ) assert condition_pattern({ConditionFunctions.EQ}, rhs, lhs, False).match(c1) is None
def test_first_level_conditions() -> None: c1 = binary_condition( None, ConditionFunctions.EQ, Column(None, "table1", "column1"), Literal(None, "test"), ) c2 = binary_condition( None, ConditionFunctions.EQ, Column(None, "table2", "column2"), Literal(None, "test"), ) c3 = binary_condition( None, ConditionFunctions.EQ, Column(None, "table3", "column3"), Literal(None, "test"), ) cond = binary_condition( None, BooleanFunctions.AND, binary_condition(None, BooleanFunctions.AND, c1, c2), c3, ) assert get_first_level_and_conditions(cond) == [c1, c2, c3] cond = binary_condition( None, BooleanFunctions.OR, binary_condition(None, BooleanFunctions.AND, c1, c2), c3, ) assert get_first_level_or_conditions(cond) == [ binary_condition(None, BooleanFunctions.AND, c1, c2), c3, ]
def replace_exp(exp: Expression) -> Expression: if matcher.match(exp) is not None: inner = replace(exp, alias=None) return FunctionCallExpr( exp.alias, "if", ( binary_condition( ConditionFunctions.IN, inner, literals_tuple( None, [ LiteralExpr(None, "1"), LiteralExpr(None, "True") ], ), ), LiteralExpr(None, "True"), LiteralExpr(None, "False"), ), ) return exp
MandatoryConditionEnforcer, OrgIdEnforcer, ProjectIdEnforcer, ) from snuba.request.request_settings import HTTPRequestSettings from snuba.state import set_config test_data = [ pytest.param( Query( Table("errors", ColumnSet([])), selected_columns=[], condition=binary_condition( BooleanFunctions.AND, in_condition(Column(None, None, "project_id"), [Literal(None, 123)]), binary_condition( "equals", Column(None, None, "org_id"), Literal(None, 1) ), ), ), True, id="Valid query. Both mandatory columns are there", ), pytest.param( Query( Table("errors", ColumnSet([])), selected_columns=[], condition=binary_condition( BooleanFunctions.AND, binary_condition( ConditionFunctions.IN,
ConditionFunctions, binary_condition, combine_and_conditions, ) from snuba.query.data_source.simple import Table from snuba.query.expressions import Column, FunctionCall, Literal from snuba.query.processors.mandatory_condition_applier import MandatoryConditionApplier from snuba.request.request_settings import HTTPRequestSettings test_data = [ pytest.param( "table1", [ binary_condition( ConditionFunctions.EQ, Column("deleted", None, "deleted"), Literal(None, "0"), ), ], id="Single Mandatory Condition TestCase", ), pytest.param( "table2", [ binary_condition( ConditionFunctions.EQ, Column("time", None, "time"), Literal(None, "1"), ), binary_condition( ConditionFunctions.EQ, Column("time2", None, "time2"), Literal(None, "2"),
data_source=Table("groupassignee_local", GROUPS_ASSIGNEE)) test_cases = [ pytest.param( Query( Table("my_table", ColumnSet([])), selected_columns=[ SelectedExpression("column1", Column(None, None, "column1")), SelectedExpression("column2", Column(None, "table1", "column2")), SelectedExpression("column3", Column("al", None, "column3")), ], condition=binary_condition( "eq", lhs=Column("al", None, "column3"), rhs=Literal(None, "blabla"), ), groupby=[ Column(None, None, "column1"), Column(None, "table1", "column2"), Column("al", None, "column3"), Column(None, None, "column4"), ], having=binary_condition( "eq", lhs=Column(None, None, "column1"), rhs=Literal(None, 123), ), order_by=[ OrderBy(OrderByDirection.ASC, Column(None, None, "column1")),
condition=binary_condition( None, BooleanFunctions.AND, binary_condition( None, ConditionFunctions.GTE, Column(None, None, "timestamp"), Literal(None, datetime(2020, 8, 1)), ), binary_condition( None, BooleanFunctions.AND, binary_condition( None, ConditionFunctions.LT, Column(None, None, "timestamp"), Literal(None, datetime(2020, 9, 1)), ), binary_condition( None, ConditionFunctions.EQ, build_mapping_expr( "tags[asd]", None, "tags", Literal(None, "asd"), ), Literal(None, "sdf"), ), ), ),
"count", FunctionCall("_snuba_count", "count", tuple())), ], groupby=[Column("_snuba_title", None, "title")], condition=binary_condition( "and", binary_condition( "equals", Column("_snuba_project_id", None, "project_id"), Literal(None, 1), ), binary_condition( "and", binary_condition( "greaterOrEquals", Column("_snuba_timestamp", None, "timestamp"), Literal(None, datetime.datetime(2021, 1, 15, 0, 0)), ), binary_condition( "less", Column("_snuba_timestamp", None, "timestamp"), Literal(None, datetime.datetime(2021, 1, 20, 0, 0)), ), ), ), ), selected_columns=[ SelectedExpression( "max_count",
def neq(col: Column, lit: Literal) -> FunctionCall: return binary_condition(ConditionFunctions.NEQ, col, lit)
(SubscriptableReference( None, Column(None, None, "tags"), Literal(None, "environment"), ), ), ), ), ], groupby=[Column(None, None, "project_id")], condition=binary_condition( BooleanFunctions.AND, binary_condition( ConditionFunctions.EQ, Column(None, None, "project_id"), Literal(None, 1), ), binary_condition( ConditionFunctions.GTE, Column(None, None, "timestamp"), Literal(None, datetime(2020, 1, 1, 12, 0)), ), ), ), selected_columns=[ SelectedExpression( "average", FunctionCall("average", "avg", (Column(None, None, "count_environment"), )), ), ], ),
) from snuba.query.data_source.simple import Entity as QueryEntity from snuba.query.dsl import multiply from snuba.query.exceptions import InvalidQueryException from snuba.query.expressions import Column, FunctionCall, Literal from snuba.query.logical import Query from snuba.query.processors.timeseries_processor import TimeSeriesProcessor from snuba.request.request_settings import HTTPRequestSettings from snuba.util import parse_datetime tests = [ pytest.param( 3600, binary_condition( ConditionFunctions.EQ, Column("my_time", None, "time"), Literal(None, "2020-01-01"), ), FunctionCall( "my_time", "toStartOfHour", (Column(None, None, "finish_ts"), Literal(None, "Universal")), ), binary_condition( ConditionFunctions.EQ, FunctionCall( "my_time", "toStartOfHour", (Column(None, None, "finish_ts"), Literal(None, "Universal")), ), Literal(None, parse_datetime("2020-01-01")),
from snuba.datasets.entities import EntityKey from snuba.datasets.entities.factory import get_entity from snuba.query import SelectedExpression from snuba.query.conditions import binary_condition from snuba.query.data_source.simple import Entity as QueryEntity from snuba.query.exceptions import InvalidQueryException from snuba.query.expressions import Column, Expression, Literal from snuba.query.logical import Query as LogicalQuery from snuba.query.validation.validators import NoTimeBasedConditionValidator tests = [ pytest.param( EntityKey.EVENTS, binary_condition( "equals", Column("_snuba_project_id", None, "project_id"), Literal(None, 1), ), id="equals", ), ] @pytest.mark.parametrize("key, condition", tests) # type: ignore def test_no_time_based_validation(key: EntityKey, condition: Expression) -> None: entity = get_entity(key) query = LogicalQuery( QueryEntity(key, entity.get_data_model()), selected_columns=[ SelectedExpression("time",
def test_nested_simple_condition() -> None: """ Iterates and maps expressions over a complex Condition: (A=B OR A=B) AND (A=B OR A=B) """ c1 = Column(None, "t1", "c1") c2 = Column(None, "t1", "c2") co1 = binary_condition(ConditionFunctions.EQ, c1, c2) c3 = Column(None, "t1", "c1") c4 = Column(None, "t1", "c2") co2 = binary_condition(ConditionFunctions.EQ, c3, c4) or1 = binary_condition(BooleanFunctions.OR, co1, co2) c5 = Column(None, "t1", "c1") c6 = Column(None, "t1", "c2") co4 = binary_condition(ConditionFunctions.EQ, c5, c6) c7 = Column(None, "t1", "c1") c8 = Column(None, "t1", "c2") co5 = binary_condition(ConditionFunctions.EQ, c7, c8) or2 = binary_condition(BooleanFunctions.OR, co4, co5) and1: Expression = binary_condition(BooleanFunctions.AND, or1, or2) ret = list(and1) expected = [c1, c2, co1, c3, c4, co2, or1, c5, c6, co4, c7, c8, co5, or2, and1] assert ret == expected cX = Column(None, "t1", "cX") co1_b = binary_condition(ConditionFunctions.EQ, c1, cX) co2_b = binary_condition(ConditionFunctions.EQ, c3, cX) or1_b = binary_condition(BooleanFunctions.OR, co1_b, co2_b) co4_b = binary_condition(ConditionFunctions.EQ, c5, cX) co5_b = binary_condition(ConditionFunctions.EQ, c7, cX) or2_b = binary_condition(BooleanFunctions.OR, co4_b, co5_b) and1_b = binary_condition(BooleanFunctions.AND, or1_b, or2_b) def replace_col(e: Expression) -> Expression: if isinstance(e, Column) and e.column_name == "c2": return cX return e and1 = and1.transform(replace_col) ret = list(and1) expected = [ c1, cX, co1_b, c3, cX, co2_b, or1_b, c5, cX, co4_b, c7, cX, co5_b, or2_b, and1_b, ] assert ret == expected
def test_first_level_conditions() -> None: c1 = binary_condition( ConditionFunctions.EQ, Column(None, "table1", "column1"), Literal(None, "test"), ) c2 = binary_condition( ConditionFunctions.EQ, Column(None, "table2", "column2"), Literal(None, "test"), ) c3 = binary_condition( ConditionFunctions.EQ, Column(None, "table3", "column3"), Literal(None, "test"), ) cond = binary_condition( BooleanFunctions.AND, binary_condition(BooleanFunctions.AND, c1, c2), c3, ) assert get_first_level_and_conditions(cond) == [c1, c2, c3] cond = binary_condition( BooleanFunctions.AND, FunctionCall( None, "equals", (FunctionCall(None, "and", (c1, c2)), Literal(None, 1)) ), c3, ) assert get_first_level_and_conditions(cond) == [c1, c2, c3] cond = binary_condition( BooleanFunctions.OR, binary_condition(BooleanFunctions.AND, c1, c2), c3, ) assert get_first_level_or_conditions(cond) == [ binary_condition(BooleanFunctions.AND, c1, c2), c3, ] cond = binary_condition( ConditionFunctions.EQ, binary_condition( BooleanFunctions.OR, c1, binary_condition(BooleanFunctions.AND, c2, c3) ), Literal(None, 1), ) assert get_first_level_or_conditions(cond) == [ c1, binary_condition(BooleanFunctions.AND, c2, c3), ]
( Literal(None, 3), FunctionCall( None, "g", (Column(None, None, "c"), ), ), ), ), ), SelectedExpression( "c", Column(None, None, "c"), ), ], condition=binary_condition(None, "less", Column(None, None, "a"), Literal(None, 3)), ), id="Basic query with no spaces and no ambiguous clause content", ), pytest.param( "MATCH (blah) WHERE a<3 COLLECT (2*(4-5)+3), g(c), c BY d, 2+7 ORDER BY f DESC", Query( {}, None, selected_columns=[ SelectedExpression( name="(2*(4-5)+3)", expression=FunctionCall( alias=None, function_name="plus", parameters=(
def eq(col: Column, lit: Literal) -> FunctionCall: return binary_condition(ConditionFunctions.EQ, col, lit) def neq(col: Column, lit: Literal) -> FunctionCall: return binary_condition(ConditionFunctions.NEQ, col, lit) def _and(ex1: Expression, ex2: Expression) -> FunctionCall: return binary_condition(BooleanFunctions.AND, ex1, ex2) # `errors > 0` has_errors = binary_condition(ConditionFunctions.GT, Column(None, None, "errors"), Literal(None, 0)) # `distinct_id != NIL` did_not_nil = neq(distinct_id, lit_nil) # `duration != MAX AND status == 1` duration_condition = _and(neq(duration, Literal(None, MAX_UINT32)), eq(status, Literal(None, 1))) # `status IN (2,3,4)` terminal_status = in_condition(status, [Literal(None, status) for status in [2, 3, 4]]) # These here are basically the same statements as the matview query sessions_raw_translators = TranslationMappers(columns=[ ColumnToCurriedFunction( None, "duration_quantiles", FunctionCall(None, "quantilesIf", quantiles),
FunctionCall(None, "c", (Column(None, None, "d"), )), ), ), ), ( tuplify(["emptyIfNull", ["project_id"]]), FunctionCall(None, "emptyIfNull", (Column(None, None, "project_id"), )), ), ( tuplify(["or", [["or", ["a", "b"]], "c"]]), binary_condition( BooleanFunctions.OR, binary_condition( BooleanFunctions.OR, Column(None, None, "a"), Column(None, None, "b"), ), Column(None, None, "c"), ), ), ( tuplify(["and", [["and", ["a", "b"]], "c"]]), binary_condition( BooleanFunctions.AND, binary_condition( BooleanFunctions.AND, Column(None, None, "a"), Column(None, None, "b"), ), Column(None, None, "c"),
def _and(ex1: Expression, ex2: Expression) -> FunctionCall: return binary_condition(BooleanFunctions.AND, ex1, ex2)
from snuba.datasets.entities import EntityKey from snuba.datasets.entities.factory import get_entity from snuba.query import SelectedExpression from snuba.query.conditions import binary_condition from snuba.query.data_source.simple import Entity as QueryEntity from snuba.query.exceptions import InvalidQueryException from snuba.query.expressions import Column, Expression, FunctionCall, Literal from snuba.query.logical import Query as LogicalQuery from snuba.query.validation.validators import EntityRequiredColumnValidator tests = [ pytest.param( EntityKey.SPANS, binary_condition( "equals", Column("_snuba_project_id", None, "project_id"), Literal(None, 1), ), id="spans has project required with =", ), pytest.param( EntityKey.SPANS, binary_condition( "in", Column("_snuba_project_id", None, "project_id"), FunctionCall(None, "tuple", (Literal(None, 1), )), ), id="in is also allowed", ), pytest.param( EntityKey.EVENTS,
"sentry:user": "******", "transaction": "transaction_name", "level": "level", } schema = WritableTableSchema( columns=all_columns, local_table_name="errors_local", dist_table_name="errors_dist", storage_set_key=StorageSetKey.EVENTS, mandatory_conditions=[ MandatoryCondition( ("deleted", "=", 0), binary_condition( None, ConditionFunctions.EQ, Column(None, None, "deleted"), Literal(None, 0), ), ) ], prewhere_candidates=[ "event_id", "group_id", "tags[sentry:release]", "message", "environment", "project_id", ], ) required_columns = [
def array_join_col(ops=None, groups=None, op_groups=None): conditions: List[Expression] = [] argument_name = "arg" argument = Argument(None, argument_name) if ops: conditions.append( binary_condition( ConditionFunctions.IN, tupleElement(None, argument, Literal(None, 1)), FunctionCall(None, "tuple", tuple(Literal(None, op) for op in ops)), )) if groups: conditions.append( binary_condition( ConditionFunctions.IN, tupleElement(None, argument, Literal(None, 2)), FunctionCall(None, "tuple", tuple(Literal(None, group) for group in groups)), )) if op_groups: conditions.append( binary_condition( ConditionFunctions.IN, FunctionCall( None, "tuple", ( tupleElement(None, argument, Literal(None, 1)), tupleElement(None, argument, Literal(None, 2)), ), ), FunctionCall( None, "tuple", tuple( FunctionCall(None, "tuple", (Literal(None, op), Literal(None, group))) for op, group in op_groups), ), )) cols = FunctionCall( None, "arrayMap", ( Lambda( None, ("x", "y", "z"), FunctionCall( None, "tuple", tuple(Argument(None, arg) for arg in ("x", "y", "z"))), ), Column(None, None, "spans.op"), Column(None, None, "spans.group"), Column(None, None, "spans.exclusive_time"), ), ) if conditions: cols = FunctionCall( None, "arrayFilter", ( Lambda(None, (argument_name, ), combine_and_conditions(conditions)), cols, ), ) return arrayJoin("snuba_all_spans", cols)
pytest.param( build_query( [], binary_condition( "notEquals", FunctionCall( None, "ifNull", ( FunctionCall( None, "arrayElement", ( Column(None, None, "tags.value"), FunctionCall( None, "indexOf", ( Column(None, None, "tags.key"), Literal(None, "query.error_reason"), ), ), ), ), Literal(None, ""), ), ), Literal(None, ""), ), ), FunctionCall( None,
test_cases = [ pytest.param( # Simple query with aliases and multiple tables Query( {}, TableSource("my_table", ColumnSet([])), selected_columns=[ SelectedExpression("column1", Column(None, None, "column1")), SelectedExpression("column2", Column(None, "table1", "column2")), SelectedExpression("column3", Column("al", None, "column3")), ], condition=binary_condition( None, "eq", lhs=Column("al", None, "column3"), rhs=Literal(None, "blabla"), ), groupby=[ Column(None, None, "column1"), Column(None, "table1", "column2"), Column("al", None, "column3"), Column(None, None, "column4"), ], having=binary_condition( None, "eq", lhs=Column(None, None, "column1"), rhs=Literal(None, 123), ), order_by=[