def build_request(
self, dataset: Dataset, timestamp: datetime, offset: Optional[int], timer: Timer
) -> Request:
"""
Returns a Request that can be used to run a query via `parse_and_run_query`.
:param dataset: The Dataset to build the request for
:param timestamp: Date that the query should run up until
:param offset: Maximum offset we should query for
"""
schema = RequestSchema.build_with_extensions(
dataset.get_extensions(), SubscriptionRequestSettings,
)
extra_conditions: Sequence[Condition] = []
if offset is not None:
extra_conditions = [[["ifnull", ["offset", 0]], "<=", offset]]
return validate_request_content(
{
"project": self.project_id,
"conditions": [*self.conditions, *extra_conditions],
"aggregations": self.aggregations,
"from_date": (timestamp - self.time_window).isoformat(),
"to_date": timestamp.isoformat(),
},
schema,
timer,
dataset,
SUBSCRIPTION_REFERRER,
)
def build_request(
self,
dataset: Dataset,
timestamp: datetime,
offset: Optional[int],
timer: Timer,
metrics: Optional[MetricsBackend] = None,
) -> Request:
schema = RequestSchema.build_with_extensions(
{},
SubscriptionRequestSettings,
Language.SNQL,
)
request = build_request(
{"query": self.query},
partial(
parse_snql_query,
[
self.validate_subscription,
partial(self.add_conditions, timestamp, offset),
],
),
SubscriptionRequestSettings,
schema,
dataset,
timer,
SUBSCRIPTION_REFERRER,
)
return request
def dataset_query(dataset: Dataset, body, timer: Timer) -> Response:
assert http_request.method == "POST"
with sentry_sdk.start_span(description="build_schema", op="validate"):
schema = RequestSchema.build_with_extensions(
dataset.get_extensions(), HTTPRequestSettings
)
request = build_request(body, schema, timer, dataset, http_request.referrer)
try:
result = parse_and_run_query(dataset, request, timer)
except QueryException as exception:
status = 500
details: Mapping[str, Any]
cause = exception.__cause__
if isinstance(cause, RateLimitExceeded):
status = 429
details = {
"type": "rate-limited",
"message": "rate limit exceeded",
}
elif isinstance(cause, ClickhouseError):
details = {
"type": "clickhouse",
"message": str(cause),
"code": cause.code,
}
elif isinstance(cause, Exception):
details = {
"type": "unknown",
"message": str(cause),
}
else:
raise # exception should have been chained
return Response(
json.dumps(
{"error": details, "timing": timer.for_json(), **exception.extra}
),
status,
{"Content-Type": "application/json"},
)
payload: MutableMapping[str, Any] = {**result.result, "timing": timer.for_json()}
if settings.STATS_IN_RESPONSE or request.settings.get_debug():
payload.update(result.extra)
return Response(json.dumps(payload), 200, {"Content-Type": "application/json"})
def dataset_query_view(*, dataset: Dataset, timer: Timer):
if http_request.method == "GET":
schema = RequestSchema.build_with_extensions(
dataset.get_extensions(), HTTPRequestSettings
)
return render_template(
"query.html",
query_template=json.dumps(schema.generate_template(), indent=4,),
)
elif http_request.method == "POST":
body = parse_request_body(http_request)
return dataset_query(dataset, body, timer)
else:
assert False, "unexpected fallthrough"
def snql_dataset_query_view(*, dataset: Dataset, timer: Timer) -> Union[Response, str]:
if http_request.method == "GET":
schema = RequestSchema.build_with_extensions(
{}, HTTPRequestSettings, Language.SNQL,
)
return render_template(
"query.html",
query_template=json.dumps(schema.generate_template(), indent=4,),
)
elif http_request.method == "POST":
body = parse_request_body(http_request)
_trace_transaction(dataset)
return dataset_query(dataset, body, timer, Language.SNQL)
else:
assert False, "unexpected fallthrough"
def dataset_query_view(*, dataset_name: str, timer: Timer):
dataset = get_dataset(dataset_name)
if http_request.method == 'GET':
schema = RequestSchema.build_with_extensions(dataset.get_extensions())
return render_template(
'query.html',
query_template=json.dumps(
schema.generate_template(),
indent=4,
),
)
elif http_request.method == 'POST':
body = parse_request_body(http_request)
return dataset_query(dataset, body, timer)
else:
assert False, 'unexpected fallthrough'
def dataset_query(dataset: Dataset, body, timer: Timer) -> Response:
assert http_request.method == "POST"
ensure_table_exists(dataset)
return format_result(
run_query(
dataset,
validate_request_content(
body,
RequestSchema.build_with_extensions(dataset.get_extensions(),
HTTPRequestSettings),
timer,
dataset,
http_request.referrer,
),
timer,
))
def test_build_request(body: MutableMapping[str, Any], language: Language,
condition: Expression) -> None:
dataset = get_dataset("events")
entity = dataset.get_default_entity()
schema = RequestSchema.build_with_extensions(
entity.get_extensions(),
HTTPRequestSettings,
language,
)
request = build_request(
body,
parse_legacy_query if language == Language.LEGACY else partial(
parse_snql_query, []),
HTTPRequestSettings,
schema,
dataset,
Timer("test"),
"my_request",
)
expected_query = Query(
from_clause=Entity(EntityKey.EVENTS, entity.get_data_model()),
selected_columns=[
SelectedExpression(
name="time",
expression=Column(alias="_snuba_time",
table_name=None,
column_name="time"),
),
SelectedExpression("count",
FunctionCall("_snuba_count", "count", tuple())),
],
condition=condition,
groupby=[Column("_snuba_time", None, "time")],
limit=1000,
granularity=60,
)
assert request.referrer == "my_request"
assert dict(request.body) == body
status, differences = request.query.equals(expected_query)
assert status == True, f"Query mismatch: {differences}"
def dataset_query(dataset, body, timer):
assert http_request.method == 'POST'
ensure_table_exists(dataset)
schema = RequestSchema.build_with_extensions(dataset.get_extensions())
query_result = parse_and_run_query(
dataset,
validate_request_content(body, schema, timer),
timer,
)
def json_default(obj):
if isinstance(obj, datetime):
return obj.isoformat()
elif isinstance(obj, UUID):
return str(obj)
return obj
return (json.dumps(query_result.result,
for_json=True,
default=json_default), query_result.status, {
'Content-Type': 'application/json'
})
def dataset_query(
dataset: Dataset, body: MutableMapping[str, Any], timer: Timer, language: Language
) -> Response:
assert http_request.method == "POST"
referrer = http_request.referrer or "<unknown>" # mypy
if language == Language.SNQL:
metrics.increment("snql.query.incoming", tags={"referrer": referrer})
parser: Callable[
[RequestParts, RequestSettings, Dataset],
Union[Query, CompositeQuery[Entity]],
] = partial(parse_snql_query, [])
else:
parser = parse_legacy_query
with sentry_sdk.start_span(description="build_schema", op="validate"):
schema = RequestSchema.build_with_extensions(
dataset.get_default_entity().get_extensions(), HTTPRequestSettings, language
)
request = build_request(
body, parser, HTTPRequestSettings, schema, dataset, timer, referrer
)
try:
result = parse_and_run_query(dataset, request, timer)
# Some metrics to track the adoption of SnQL
query_type = "simple"
if language == Language.SNQL:
if isinstance(request.query, CompositeQuery):
if isinstance(request.query.get_from_clause(), JoinClause):
query_type = "join"
else:
query_type = "subquery"
metrics.increment(
"snql.query.success", tags={"referrer": referrer, "type": query_type}
)
except QueryException as exception:
status = 500
details: Mapping[str, Any]
cause = exception.__cause__
if isinstance(cause, RateLimitExceeded):
status = 429
details = {
"type": "rate-limited",
"message": "rate limit exceeded",
}
elif isinstance(cause, ClickhouseError):
details = {
"type": "clickhouse",
"message": str(cause),
"code": cause.code,
}
elif isinstance(cause, Exception):
details = {
"type": "unknown",
"message": str(cause),
}
else:
raise # exception should have been chained
if language == Language.SNQL:
metrics.increment(
"snql.query.failed", tags={"referrer": referrer, "status": f"{status}"},
)
return Response(
json.dumps(
{"error": details, "timing": timer.for_json(), **exception.extra}
),
status,
{"Content-Type": "application/json"},
)
payload: MutableMapping[str, Any] = {**result.result, "timing": timer.for_json()}
if settings.STATS_IN_RESPONSE or request.settings.get_debug():
payload.update(result.extra)
return Response(json.dumps(payload), 200, {"Content-Type": "application/json"})