def drop(*, dataset: Dataset):
for statement in dataset.get_dataset_schemas().get_drop_statements():
clickhouse_rw.execute(statement.statement)
ensure_table_exists(dataset, force=True)
redis_client.flushdb()
return ("ok", 200, {"Content-Type": "text/plain"})
def validate_request_content(body, schema: RequestSchema, timer,
dataset: Dataset) -> Request:
source = dataset.get_dataset_schemas().get_read_schema().get_data_source()
try:
request = schema.validate(body, source)
except jsonschema.ValidationError as error:
raise BadRequest(str(error)) from error
timer.mark('validate_schema')
return request
def ensure_table_exists(dataset: Dataset, force: bool = False) -> None:
if not force and _ensured.get(dataset, False):
return
assert local_dataset_mode(), "Cannot create table in distributed mode"
from snuba import migrate
# We cannot build distributed tables this way. So this only works in local
# mode.
for statement in dataset.get_dataset_schemas().get_create_statements():
clickhouse_rw.execute(statement.statement)
migrate.run(clickhouse_rw, dataset)
_ensured[dataset] = True
def parse_query(body: MutableMapping[str, Any], dataset: Dataset) -> Query:
"""
Parses the query body generating the AST. This only takes into
account the initial query body. Extensions are parsed by extension
processors and are supposed to update the AST.
"""
try:
return _parse_query_impl(body, dataset)
except Exception as e:
# During the development there is no need to fail Snuba queries if the parser
# has an issue, anyway the production query is ran based on the old query
# representation.
# Once we will be actually using the ast to build the Clickhouse query
# this try/except block will disappear.
enforce_validity = state.get_config("query_parsing_enforce_validity",
0)
if enforce_validity:
raise e
else:
logger.exception("Failed to parse query")
source = dataset.get_dataset_schemas().get_read_schema(
).get_data_source()
return Query(body, source)
def _parse_query_impl(body: MutableMapping[str, Any],
dataset: Dataset) -> Query:
aggregate_exprs = []
for aggregation in body.get("aggregations", []):
assert isinstance(aggregation, (list, tuple))
aggregation_function = aggregation[0]
column_expr = aggregation[1]
column_expr = column_expr if column_expr else []
alias = aggregation[2]
alias = alias if alias else None
aggregate_exprs.append(
parse_aggregation(aggregation_function, column_expr, alias))
groupby_exprs = [
parse_expression(tuplify(group_by))
for group_by in to_list(body.get("groupby", []))
]
select_exprs = [
parse_expression(tuplify(select))
for select in body.get("selected_columns", [])
]
selected_cols = groupby_exprs + aggregate_exprs + select_exprs
arrayjoin = body.get("arrayjoin")
if arrayjoin:
array_join_expr: Optional[Expression] = parse_expression(
body["arrayjoin"])
else:
array_join_expr = None
where_expr = parse_conditions_to_expr(body.get("conditions", []), dataset,
arrayjoin)
having_expr = parse_conditions_to_expr(body.get("having", []), dataset,
arrayjoin)
orderby_exprs = []
for orderby in to_list(body.get("orderby", [])):
if isinstance(orderby, str):
match = NEGATE_RE.match(orderby)
assert match is not None, f"Invalid Order By clause {orderby}"
direction, col = match.groups()
orderby = col
elif is_function(orderby):
match = NEGATE_RE.match(orderby[0])
assert match is not None, f"Invalid Order By clause {orderby}"
direction, col = match.groups()
orderby = [col] + orderby[1:]
else:
raise ValueError(f"Invalid Order By clause {orderby}")
orderby_parsed = parse_expression(tuplify(orderby))
orderby_exprs.append(
OrderBy(
OrderByDirection.DESC
if direction == "-" else OrderByDirection.ASC,
orderby_parsed,
))
source = dataset.get_dataset_schemas().get_read_schema().get_data_source()
return Query(
body,
source,
selected_columns=selected_cols,
array_join=array_join_expr,
condition=where_expr,
groupby=groupby_exprs,
having=having_expr,
order_by=orderby_exprs,
)
def parse_conditions(
operand_builder: Callable[[Any], TExpression],
and_builder: Callable[[Sequence[TExpression]], Optional[TExpression]],
or_builder: Callable[[Sequence[TExpression]], Optional[TExpression]],
unpack_array_condition_builder: Callable[[TExpression, str, Any],
TExpression],
simple_condition_builder: Callable[[TExpression, str, Any], TExpression],
dataset: Dataset,
conditions: Any,
array_join: Optional[str],
depth: int = 0,
) -> Optional[TExpression]:
"""
Return a boolean expression suitable for putting in the WHERE clause of the
query. The expression is constructed by ANDing groups of OR expressions.
Expansion of columns is handled, as is replacement of columns with aliases,
if the column has already been expanded and aliased elsewhere.
operand_builder: Builds the TExpression representing the left hand side
of a simple condition. This can be as nested as the user wants
and_builder / or_builder: Combine a list of expressions in AND/OR
unpack_array_condition_builder: Deals with a special case where we unpack conditions
on array columns. More details in the code.
simple_condition_builder: Generates a simple condition made by expression on the
left hand side, an operator and a literal on the right hand side.
"""
from snuba.clickhouse.columns import Array
if not conditions:
return None
if depth == 0:
# dedupe conditions at top level, but keep them in order
sub = OrderedDict((
parse_conditions(
operand_builder,
and_builder,
or_builder,
unpack_array_condition_builder,
simple_condition_builder,
dataset,
cond,
array_join,
depth + 1,
),
None,
) for cond in conditions)
return and_builder([s for s in sub.keys() if s])
elif is_condition(conditions):
lhs, op, lit = dataset.process_condition(conditions)
# facilitate deduping IN conditions by sorting them.
if op in ("IN", "NOT IN") and isinstance(lit, tuple):
lit = tuple(sorted(lit))
# If the LHS is a simple column name that refers to an array column
# (and we are not arrayJoining on that column, which would make it
# scalar again) and the RHS is a scalar value, we assume that the user
# actually means to check if any (or all) items in the array match the
# predicate, so we return an `any(x == value for x in array_column)`
# type expression. We assume that operators looking for a specific value
# (IN, =, LIKE) are looking for rows where any array value matches, and
# exclusionary operators (NOT IN, NOT LIKE, !=) are looking for rows
# where all elements match (eg. all NOT LIKE 'foo').
columns = dataset.get_dataset_schemas().get_read_schema().get_columns()
if (isinstance(lhs, str) and lhs in columns
and isinstance(columns[lhs].type, Array)
and columns[lhs].base_name != array_join
and not isinstance(lit, (list, tuple))):
return unpack_array_condition_builder(operand_builder(lhs), op,
lit)
else:
return simple_condition_builder(operand_builder(lhs), op, lit)
elif depth == 1:
sub_expression = (parse_conditions(
operand_builder,
and_builder,
or_builder,
unpack_array_condition_builder,
simple_condition_builder,
dataset,
cond,
array_join,
depth + 1,
) for cond in conditions)
return or_builder([s for s in sub_expression if s])
else:
raise InvalidConditionException(str(conditions))