def test_in_condition() -> None:
in_condition = binary_condition(
None,
ConditionFunctions.IN,
Column(None, None, "tags_key"),
literals_tuple(
None,
[Literal(None, "t1"), Literal(None, "t2")]),
)
assert is_in_condition(in_condition)
match = is_in_condition_pattern(ColumnPattern(
None, String("tags_key"))).match(in_condition)
assert match is not None
assert match.expression("tuple") == literals_tuple(
None, [Literal(None, "t1"), Literal(None, "t2")])
assert match.expression("lhs") == Column(None, None, "tags_key")
def test_aliased_expressions_from_basic_condition() -> None:
"""
Iterates over the expressions in a basic condition when those expressions
are aliased
f(t1.c1) as a = t1.c2 as a2
"""
c = Column(None, "t1", "c1")
f1 = FunctionCall("a", "f", (c, ))
c2 = Column("a2", "t1", "c2")
condition = binary_condition(ConditionFunctions.EQ, f1, c2)
ret = list(condition)
expected = [c, f1, c2, condition]
assert ret == expected
def test_iterate_over_query():
"""
Creates a query with the new AST and iterate over all expressions.
"""
column1 = Column(None, "c1", "t1")
column2 = Column(None, "c2", "t1")
function_1 = FunctionCall("alias", "f1", (column1, column2))
function_2 = FunctionCall("alias", "f2", (column2,))
condition = binary_condition(
None, ConditionFunctions.EQ, column1, Literal(None, "1")
)
orderby = OrderBy(OrderByDirection.ASC, function_2)
query = Query(
{},
TableSource("my_table", ColumnSet([])),
selected_columns=[function_1],
array_join=None,
condition=condition,
groupby=[function_1],
having=None,
order_by=[orderby],
)
expected_expressions = [
# selected columns
column1,
column2,
function_1,
# condition
column1,
Literal(None, "1"),
condition,
# groupby
column1,
column2,
function_1,
# order by
column2,
function_2,
]
assert list(query.get_all_expressions()) == expected_expressions
def test_failure_rate_format_expressions() -> None:
unprocessed = Query(
QueryEntity(EntityKey.EVENTS, ColumnSet([])),
selected_columns=[
SelectedExpression(name=None,
expression=Column(None, None, "column2")),
SelectedExpression("perf", FunctionCall("perf", "failure_rate",
())),
],
)
expected = Query(
QueryEntity(EntityKey.EVENTS, ColumnSet([])),
selected_columns=[
SelectedExpression(name=None,
expression=Column(None, None, "column2")),
SelectedExpression(
"perf",
divide(
FunctionCall(
None,
"countIf",
(combine_and_conditions([
binary_condition(
ConditionFunctions.NEQ,
Column(None, None, "transaction_status"),
Literal(None, code),
) for code in [0, 1, 2]
]), ),
),
count(),
"perf",
),
),
],
)
failure_rate_processor().process_query(unprocessed, HTTPQuerySettings())
assert expected.get_selected_columns() == unprocessed.get_selected_columns(
)
ret = unprocessed.get_selected_columns()[1].expression.accept(
ClickhouseExpressionFormatter())
assert ret == (
"(divide(countIf(notEquals(transaction_status, 0) AND notEquals(transaction_status, 1) AND notEquals(transaction_status, 2)), count()) AS perf)"
)
def simple_condition_builder(lhs: Expression, op: str, literal: Any) -> Expression:
if op in UNARY_OPERATORS:
if literal is not None:
raise ParsingException(
f"Right hand side operand {literal} provided to unary operator {op}",
report=False,
)
return unary_condition(OPERATOR_TO_FUNCTION[op], lhs)
else:
if literal is None:
raise ParsingException(
f"Missing right hand side operand for binary operator {op}",
report=False,
)
return binary_condition(
OPERATOR_TO_FUNCTION[op], lhs, preprocess_literal(op, literal)
)
def test_is_x_condition_functions() -> None:
eq_condition = binary_condition(
ConditionFunctions.EQ, Column(None, None, "test"), Literal(None, "1")
)
assert is_any_binary_condition(eq_condition, ConditionFunctions.EQ)
assert not is_any_binary_condition(eq_condition, ConditionFunctions.NEQ)
un_condition = unary_condition(
ConditionFunctions.IS_NOT_NULL, Column(None, None, "test")
)
assert is_unary_condition(un_condition, ConditionFunctions.IS_NOT_NULL)
assert not is_unary_condition(un_condition, ConditionFunctions.IS_NULL)
assert not is_unary_condition(eq_condition, ConditionFunctions.IS_NOT_NULL)
almost_condition = FunctionCall(None, "isNotNullish", (Column(None, None, "test"),))
assert is_condition(eq_condition)
assert is_condition(un_condition)
assert not is_condition(almost_condition)
def test_invalid_datetime() -> None:
unprocessed = Query(
QueryEntity(EntityKey.EVENTS, ColumnSet([])),
selected_columns=[
SelectedExpression(
"transaction.duration", Column("transaction.duration", None, "duration")
),
],
condition=binary_condition(
ConditionFunctions.EQ, Column("my_time", None, "time"), Literal(None, ""),
),
)
entity = TransactionsEntity()
processors = entity.get_query_processors()
for processor in processors:
if isinstance(processor, TimeSeriesProcessor):
with pytest.raises(InvalidQueryException):
processor.process_query(unprocessed, HTTPRequestSettings())
def test_binary_match() -> None:
c1 = binary_condition(
ConditionFunctions.EQ, Column(None, "table1", "column1"), Literal(None, "test"),
)
lhs = ColumnPattern(String("table1"), String("column1"))
rhs = LiteralPattern(String("test"))
assert (
condition_pattern({ConditionFunctions.EQ}, lhs, rhs, True).match(c1) is not None
)
assert (
condition_pattern({ConditionFunctions.EQ}, lhs, rhs, False).match(c1)
is not None
)
assert (
condition_pattern({ConditionFunctions.EQ}, rhs, lhs, True).match(c1) is not None
)
assert condition_pattern({ConditionFunctions.EQ}, rhs, lhs, False).match(c1) is None
def test_first_level_conditions() -> None:
c1 = binary_condition(
None,
ConditionFunctions.EQ,
Column(None, "table1", "column1"),
Literal(None, "test"),
)
c2 = binary_condition(
None,
ConditionFunctions.EQ,
Column(None, "table2", "column2"),
Literal(None, "test"),
)
c3 = binary_condition(
None,
ConditionFunctions.EQ,
Column(None, "table3", "column3"),
Literal(None, "test"),
)
cond = binary_condition(
None,
BooleanFunctions.AND,
binary_condition(None, BooleanFunctions.AND, c1, c2),
c3,
)
assert get_first_level_and_conditions(cond) == [c1, c2, c3]
cond = binary_condition(
None,
BooleanFunctions.OR,
binary_condition(None, BooleanFunctions.AND, c1, c2),
c3,
)
assert get_first_level_or_conditions(cond) == [
binary_condition(None, BooleanFunctions.AND, c1, c2),
c3,
]
def replace_exp(exp: Expression) -> Expression:
if matcher.match(exp) is not None:
inner = replace(exp, alias=None)
return FunctionCallExpr(
exp.alias,
"if",
(
binary_condition(
ConditionFunctions.IN,
inner,
literals_tuple(
None,
[
LiteralExpr(None, "1"),
LiteralExpr(None, "True")
],
),
),
LiteralExpr(None, "True"),
LiteralExpr(None, "False"),
),
)
return exp
MandatoryConditionEnforcer,
OrgIdEnforcer,
ProjectIdEnforcer,
)
from snuba.request.request_settings import HTTPRequestSettings
from snuba.state import set_config
test_data = [
pytest.param(
Query(
Table("errors", ColumnSet([])),
selected_columns=[],
condition=binary_condition(
BooleanFunctions.AND,
in_condition(Column(None, None, "project_id"), [Literal(None, 123)]),
binary_condition(
"equals", Column(None, None, "org_id"), Literal(None, 1)
),
),
),
True,
id="Valid query. Both mandatory columns are there",
),
pytest.param(
Query(
Table("errors", ColumnSet([])),
selected_columns=[],
condition=binary_condition(
BooleanFunctions.AND,
binary_condition(
ConditionFunctions.IN,
ConditionFunctions,
binary_condition,
combine_and_conditions,
)
from snuba.query.data_source.simple import Table
from snuba.query.expressions import Column, FunctionCall, Literal
from snuba.query.processors.mandatory_condition_applier import MandatoryConditionApplier
from snuba.request.request_settings import HTTPRequestSettings
test_data = [
pytest.param(
"table1",
[
binary_condition(
ConditionFunctions.EQ,
Column("deleted", None, "deleted"),
Literal(None, "0"),
),
],
id="Single Mandatory Condition TestCase",
),
pytest.param(
"table2",
[
binary_condition(
ConditionFunctions.EQ, Column("time", None, "time"), Literal(None, "1"),
),
binary_condition(
ConditionFunctions.EQ,
Column("time2", None, "time2"),
Literal(None, "2"),
data_source=Table("groupassignee_local",
GROUPS_ASSIGNEE))
test_cases = [
pytest.param(
Query(
Table("my_table", ColumnSet([])),
selected_columns=[
SelectedExpression("column1", Column(None, None, "column1")),
SelectedExpression("column2", Column(None, "table1",
"column2")),
SelectedExpression("column3", Column("al", None, "column3")),
],
condition=binary_condition(
"eq",
lhs=Column("al", None, "column3"),
rhs=Literal(None, "blabla"),
),
groupby=[
Column(None, None, "column1"),
Column(None, "table1", "column2"),
Column("al", None, "column3"),
Column(None, None, "column4"),
],
having=binary_condition(
"eq",
lhs=Column(None, None, "column1"),
rhs=Literal(None, 123),
),
order_by=[
OrderBy(OrderByDirection.ASC, Column(None, None, "column1")),
condition=binary_condition(
None,
BooleanFunctions.AND,
binary_condition(
None,
ConditionFunctions.GTE,
Column(None, None, "timestamp"),
Literal(None, datetime(2020, 8, 1)),
),
binary_condition(
None,
BooleanFunctions.AND,
binary_condition(
None,
ConditionFunctions.LT,
Column(None, None, "timestamp"),
Literal(None, datetime(2020, 9, 1)),
),
binary_condition(
None,
ConditionFunctions.EQ,
build_mapping_expr(
"tags[asd]",
None,
"tags",
Literal(None, "asd"),
),
Literal(None, "sdf"),
),
),
),
"count", FunctionCall("_snuba_count", "count",
tuple())),
],
groupby=[Column("_snuba_title", None, "title")],
condition=binary_condition(
"and",
binary_condition(
"equals",
Column("_snuba_project_id", None, "project_id"),
Literal(None, 1),
),
binary_condition(
"and",
binary_condition(
"greaterOrEquals",
Column("_snuba_timestamp", None, "timestamp"),
Literal(None, datetime.datetime(2021, 1, 15, 0,
0)),
),
binary_condition(
"less",
Column("_snuba_timestamp", None, "timestamp"),
Literal(None, datetime.datetime(2021, 1, 20, 0,
0)),
),
),
),
),
selected_columns=[
SelectedExpression(
"max_count",
def neq(col: Column, lit: Literal) -> FunctionCall:
return binary_condition(ConditionFunctions.NEQ, col, lit)
(SubscriptableReference(
None,
Column(None, None, "tags"),
Literal(None, "environment"),
), ),
),
),
],
groupby=[Column(None, None, "project_id")],
condition=binary_condition(
BooleanFunctions.AND,
binary_condition(
ConditionFunctions.EQ,
Column(None, None, "project_id"),
Literal(None, 1),
),
binary_condition(
ConditionFunctions.GTE,
Column(None, None, "timestamp"),
Literal(None, datetime(2020, 1, 1, 12, 0)),
),
),
),
selected_columns=[
SelectedExpression(
"average",
FunctionCall("average", "avg",
(Column(None, None, "count_environment"), )),
),
],
),
)
from snuba.query.data_source.simple import Entity as QueryEntity
from snuba.query.dsl import multiply
from snuba.query.exceptions import InvalidQueryException
from snuba.query.expressions import Column, FunctionCall, Literal
from snuba.query.logical import Query
from snuba.query.processors.timeseries_processor import TimeSeriesProcessor
from snuba.request.request_settings import HTTPRequestSettings
from snuba.util import parse_datetime
tests = [
pytest.param(
3600,
binary_condition(
ConditionFunctions.EQ,
Column("my_time", None, "time"),
Literal(None, "2020-01-01"),
),
FunctionCall(
"my_time",
"toStartOfHour",
(Column(None, None, "finish_ts"), Literal(None, "Universal")),
),
binary_condition(
ConditionFunctions.EQ,
FunctionCall(
"my_time",
"toStartOfHour",
(Column(None, None, "finish_ts"), Literal(None, "Universal")),
),
Literal(None, parse_datetime("2020-01-01")),
from snuba.datasets.entities import EntityKey
from snuba.datasets.entities.factory import get_entity
from snuba.query import SelectedExpression
from snuba.query.conditions import binary_condition
from snuba.query.data_source.simple import Entity as QueryEntity
from snuba.query.exceptions import InvalidQueryException
from snuba.query.expressions import Column, Expression, Literal
from snuba.query.logical import Query as LogicalQuery
from snuba.query.validation.validators import NoTimeBasedConditionValidator
tests = [
pytest.param(
EntityKey.EVENTS,
binary_condition(
"equals",
Column("_snuba_project_id", None, "project_id"),
Literal(None, 1),
),
id="equals",
),
]
@pytest.mark.parametrize("key, condition", tests) # type: ignore
def test_no_time_based_validation(key: EntityKey,
condition: Expression) -> None:
entity = get_entity(key)
query = LogicalQuery(
QueryEntity(key, entity.get_data_model()),
selected_columns=[
SelectedExpression("time",
def test_nested_simple_condition() -> None:
"""
Iterates and maps expressions over a complex Condition:
(A=B OR A=B) AND (A=B OR A=B)
"""
c1 = Column(None, "t1", "c1")
c2 = Column(None, "t1", "c2")
co1 = binary_condition(ConditionFunctions.EQ, c1, c2)
c3 = Column(None, "t1", "c1")
c4 = Column(None, "t1", "c2")
co2 = binary_condition(ConditionFunctions.EQ, c3, c4)
or1 = binary_condition(BooleanFunctions.OR, co1, co2)
c5 = Column(None, "t1", "c1")
c6 = Column(None, "t1", "c2")
co4 = binary_condition(ConditionFunctions.EQ, c5, c6)
c7 = Column(None, "t1", "c1")
c8 = Column(None, "t1", "c2")
co5 = binary_condition(ConditionFunctions.EQ, c7, c8)
or2 = binary_condition(BooleanFunctions.OR, co4, co5)
and1: Expression = binary_condition(BooleanFunctions.AND, or1, or2)
ret = list(and1)
expected = [c1, c2, co1, c3, c4, co2, or1, c5, c6, co4, c7, c8, co5, or2, and1]
assert ret == expected
cX = Column(None, "t1", "cX")
co1_b = binary_condition(ConditionFunctions.EQ, c1, cX)
co2_b = binary_condition(ConditionFunctions.EQ, c3, cX)
or1_b = binary_condition(BooleanFunctions.OR, co1_b, co2_b)
co4_b = binary_condition(ConditionFunctions.EQ, c5, cX)
co5_b = binary_condition(ConditionFunctions.EQ, c7, cX)
or2_b = binary_condition(BooleanFunctions.OR, co4_b, co5_b)
and1_b = binary_condition(BooleanFunctions.AND, or1_b, or2_b)
def replace_col(e: Expression) -> Expression:
if isinstance(e, Column) and e.column_name == "c2":
return cX
return e
and1 = and1.transform(replace_col)
ret = list(and1)
expected = [
c1,
cX,
co1_b,
c3,
cX,
co2_b,
or1_b,
c5,
cX,
co4_b,
c7,
cX,
co5_b,
or2_b,
and1_b,
]
assert ret == expected
def test_first_level_conditions() -> None:
c1 = binary_condition(
ConditionFunctions.EQ, Column(None, "table1", "column1"), Literal(None, "test"),
)
c2 = binary_condition(
ConditionFunctions.EQ, Column(None, "table2", "column2"), Literal(None, "test"),
)
c3 = binary_condition(
ConditionFunctions.EQ, Column(None, "table3", "column3"), Literal(None, "test"),
)
cond = binary_condition(
BooleanFunctions.AND, binary_condition(BooleanFunctions.AND, c1, c2), c3,
)
assert get_first_level_and_conditions(cond) == [c1, c2, c3]
cond = binary_condition(
BooleanFunctions.AND,
FunctionCall(
None, "equals", (FunctionCall(None, "and", (c1, c2)), Literal(None, 1))
),
c3,
)
assert get_first_level_and_conditions(cond) == [c1, c2, c3]
cond = binary_condition(
BooleanFunctions.OR, binary_condition(BooleanFunctions.AND, c1, c2), c3,
)
assert get_first_level_or_conditions(cond) == [
binary_condition(BooleanFunctions.AND, c1, c2),
c3,
]
cond = binary_condition(
ConditionFunctions.EQ,
binary_condition(
BooleanFunctions.OR, c1, binary_condition(BooleanFunctions.AND, c2, c3)
),
Literal(None, 1),
)
assert get_first_level_or_conditions(cond) == [
c1,
binary_condition(BooleanFunctions.AND, c2, c3),
]
(
Literal(None, 3),
FunctionCall(
None,
"g",
(Column(None, None, "c"), ),
),
),
),
),
SelectedExpression(
"c",
Column(None, None, "c"),
),
],
condition=binary_condition(None, "less", Column(None, None, "a"),
Literal(None, 3)),
),
id="Basic query with no spaces and no ambiguous clause content",
),
pytest.param(
"MATCH (blah) WHERE a<3 COLLECT (2*(4-5)+3), g(c), c BY d, 2+7 ORDER BY f DESC",
Query(
{},
None,
selected_columns=[
SelectedExpression(
name="(2*(4-5)+3)",
expression=FunctionCall(
alias=None,
function_name="plus",
parameters=(
def eq(col: Column, lit: Literal) -> FunctionCall:
return binary_condition(ConditionFunctions.EQ, col, lit)
def neq(col: Column, lit: Literal) -> FunctionCall:
return binary_condition(ConditionFunctions.NEQ, col, lit)
def _and(ex1: Expression, ex2: Expression) -> FunctionCall:
return binary_condition(BooleanFunctions.AND, ex1, ex2)
# `errors > 0`
has_errors = binary_condition(ConditionFunctions.GT,
Column(None, None, "errors"), Literal(None, 0))
# `distinct_id != NIL`
did_not_nil = neq(distinct_id, lit_nil)
# `duration != MAX AND status == 1`
duration_condition = _and(neq(duration, Literal(None, MAX_UINT32)),
eq(status, Literal(None, 1)))
# `status IN (2,3,4)`
terminal_status = in_condition(status,
[Literal(None, status) for status in [2, 3, 4]])
# These here are basically the same statements as the matview query
sessions_raw_translators = TranslationMappers(columns=[
ColumnToCurriedFunction(
None,
"duration_quantiles",
FunctionCall(None, "quantilesIf", quantiles),
FunctionCall(None, "c", (Column(None, None, "d"), )),
),
),
),
(
tuplify(["emptyIfNull", ["project_id"]]),
FunctionCall(None, "emptyIfNull",
(Column(None, None, "project_id"), )),
),
(
tuplify(["or", [["or", ["a", "b"]], "c"]]),
binary_condition(
BooleanFunctions.OR,
binary_condition(
BooleanFunctions.OR,
Column(None, None, "a"),
Column(None, None, "b"),
),
Column(None, None, "c"),
),
),
(
tuplify(["and", [["and", ["a", "b"]], "c"]]),
binary_condition(
BooleanFunctions.AND,
binary_condition(
BooleanFunctions.AND,
Column(None, None, "a"),
Column(None, None, "b"),
),
Column(None, None, "c"),
def _and(ex1: Expression, ex2: Expression) -> FunctionCall:
return binary_condition(BooleanFunctions.AND, ex1, ex2)
from snuba.datasets.entities import EntityKey
from snuba.datasets.entities.factory import get_entity
from snuba.query import SelectedExpression
from snuba.query.conditions import binary_condition
from snuba.query.data_source.simple import Entity as QueryEntity
from snuba.query.exceptions import InvalidQueryException
from snuba.query.expressions import Column, Expression, FunctionCall, Literal
from snuba.query.logical import Query as LogicalQuery
from snuba.query.validation.validators import EntityRequiredColumnValidator
tests = [
pytest.param(
EntityKey.SPANS,
binary_condition(
"equals",
Column("_snuba_project_id", None, "project_id"),
Literal(None, 1),
),
id="spans has project required with =",
),
pytest.param(
EntityKey.SPANS,
binary_condition(
"in",
Column("_snuba_project_id", None, "project_id"),
FunctionCall(None, "tuple", (Literal(None, 1), )),
),
id="in is also allowed",
),
pytest.param(
EntityKey.EVENTS,
"sentry:user": "******",
"transaction": "transaction_name",
"level": "level",
}
schema = WritableTableSchema(
columns=all_columns,
local_table_name="errors_local",
dist_table_name="errors_dist",
storage_set_key=StorageSetKey.EVENTS,
mandatory_conditions=[
MandatoryCondition(
("deleted", "=", 0),
binary_condition(
None,
ConditionFunctions.EQ,
Column(None, None, "deleted"),
Literal(None, 0),
),
)
],
prewhere_candidates=[
"event_id",
"group_id",
"tags[sentry:release]",
"message",
"environment",
"project_id",
],
)
required_columns = [
def array_join_col(ops=None, groups=None, op_groups=None):
conditions: List[Expression] = []
argument_name = "arg"
argument = Argument(None, argument_name)
if ops:
conditions.append(
binary_condition(
ConditionFunctions.IN,
tupleElement(None, argument, Literal(None, 1)),
FunctionCall(None, "tuple",
tuple(Literal(None, op) for op in ops)),
))
if groups:
conditions.append(
binary_condition(
ConditionFunctions.IN,
tupleElement(None, argument, Literal(None, 2)),
FunctionCall(None, "tuple",
tuple(Literal(None, group) for group in groups)),
))
if op_groups:
conditions.append(
binary_condition(
ConditionFunctions.IN,
FunctionCall(
None,
"tuple",
(
tupleElement(None, argument, Literal(None, 1)),
tupleElement(None, argument, Literal(None, 2)),
),
),
FunctionCall(
None,
"tuple",
tuple(
FunctionCall(None, "tuple", (Literal(None, op),
Literal(None, group)))
for op, group in op_groups),
),
))
cols = FunctionCall(
None,
"arrayMap",
(
Lambda(
None,
("x", "y", "z"),
FunctionCall(
None, "tuple",
tuple(Argument(None, arg) for arg in ("x", "y", "z"))),
),
Column(None, None, "spans.op"),
Column(None, None, "spans.group"),
Column(None, None, "spans.exclusive_time"),
),
)
if conditions:
cols = FunctionCall(
None,
"arrayFilter",
(
Lambda(None,
(argument_name, ), combine_and_conditions(conditions)),
cols,
),
)
return arrayJoin("snuba_all_spans", cols)
pytest.param(
build_query(
[],
binary_condition(
"notEquals",
FunctionCall(
None,
"ifNull",
(
FunctionCall(
None,
"arrayElement",
(
Column(None, None, "tags.value"),
FunctionCall(
None,
"indexOf",
(
Column(None, None, "tags.key"),
Literal(None, "query.error_reason"),
),
),
),
),
Literal(None, ""),
),
),
Literal(None, ""),
),
),
FunctionCall(
None,
test_cases = [
pytest.param(
# Simple query with aliases and multiple tables
Query(
{},
TableSource("my_table", ColumnSet([])),
selected_columns=[
SelectedExpression("column1", Column(None, None, "column1")),
SelectedExpression("column2", Column(None, "table1",
"column2")),
SelectedExpression("column3", Column("al", None, "column3")),
],
condition=binary_condition(
None,
"eq",
lhs=Column("al", None, "column3"),
rhs=Literal(None, "blabla"),
),
groupby=[
Column(None, None, "column1"),
Column(None, "table1", "column2"),
Column("al", None, "column3"),
Column(None, None, "column4"),
],
having=binary_condition(
None,
"eq",
lhs=Column(None, None, "column1"),
rhs=Literal(None, 123),
),
order_by=[
