def build_request( self, dataset: Dataset, timestamp: datetime, offset: Optional[int], timer: Timer ) -> Request: """ Returns a Request that can be used to run a query via `parse_and_run_query`. :param dataset: The Dataset to build the request for :param timestamp: Date that the query should run up until :param offset: Maximum offset we should query for """ schema = RequestSchema.build_with_extensions( dataset.get_extensions(), SubscriptionRequestSettings, ) extra_conditions: Sequence[Condition] = [] if offset is not None: extra_conditions = [[["ifnull", ["offset", 0]], "<=", offset]] return validate_request_content( { "project": self.project_id, "conditions": [*self.conditions, *extra_conditions], "aggregations": self.aggregations, "from_date": (timestamp - self.time_window).isoformat(), "to_date": timestamp.isoformat(), }, schema, timer, dataset, SUBSCRIPTION_REFERRER, )
def build_request( self, dataset: Dataset, timestamp: datetime, offset: Optional[int], timer: Timer, metrics: Optional[MetricsBackend] = None, ) -> Request: schema = RequestSchema.build_with_extensions( {}, SubscriptionRequestSettings, Language.SNQL, ) request = build_request( {"query": self.query}, partial( parse_snql_query, [ self.validate_subscription, partial(self.add_conditions, timestamp, offset), ], ), SubscriptionRequestSettings, schema, dataset, timer, SUBSCRIPTION_REFERRER, ) return request
def dataset_query(dataset: Dataset, body, timer: Timer) -> Response: assert http_request.method == "POST" with sentry_sdk.start_span(description="build_schema", op="validate"): schema = RequestSchema.build_with_extensions( dataset.get_extensions(), HTTPRequestSettings ) request = build_request(body, schema, timer, dataset, http_request.referrer) try: result = parse_and_run_query(dataset, request, timer) except QueryException as exception: status = 500 details: Mapping[str, Any] cause = exception.__cause__ if isinstance(cause, RateLimitExceeded): status = 429 details = { "type": "rate-limited", "message": "rate limit exceeded", } elif isinstance(cause, ClickhouseError): details = { "type": "clickhouse", "message": str(cause), "code": cause.code, } elif isinstance(cause, Exception): details = { "type": "unknown", "message": str(cause), } else: raise # exception should have been chained return Response( json.dumps( {"error": details, "timing": timer.for_json(), **exception.extra} ), status, {"Content-Type": "application/json"}, ) payload: MutableMapping[str, Any] = {**result.result, "timing": timer.for_json()} if settings.STATS_IN_RESPONSE or request.settings.get_debug(): payload.update(result.extra) return Response(json.dumps(payload), 200, {"Content-Type": "application/json"})
def dataset_query_view(*, dataset: Dataset, timer: Timer): if http_request.method == "GET": schema = RequestSchema.build_with_extensions( dataset.get_extensions(), HTTPRequestSettings ) return render_template( "query.html", query_template=json.dumps(schema.generate_template(), indent=4,), ) elif http_request.method == "POST": body = parse_request_body(http_request) return dataset_query(dataset, body, timer) else: assert False, "unexpected fallthrough"
def snql_dataset_query_view(*, dataset: Dataset, timer: Timer) -> Union[Response, str]: if http_request.method == "GET": schema = RequestSchema.build_with_extensions( {}, HTTPRequestSettings, Language.SNQL, ) return render_template( "query.html", query_template=json.dumps(schema.generate_template(), indent=4,), ) elif http_request.method == "POST": body = parse_request_body(http_request) _trace_transaction(dataset) return dataset_query(dataset, body, timer, Language.SNQL) else: assert False, "unexpected fallthrough"
def dataset_query_view(*, dataset_name: str, timer: Timer): dataset = get_dataset(dataset_name) if http_request.method == 'GET': schema = RequestSchema.build_with_extensions(dataset.get_extensions()) return render_template( 'query.html', query_template=json.dumps( schema.generate_template(), indent=4, ), ) elif http_request.method == 'POST': body = parse_request_body(http_request) return dataset_query(dataset, body, timer) else: assert False, 'unexpected fallthrough'
def dataset_query(dataset: Dataset, body, timer: Timer) -> Response: assert http_request.method == "POST" ensure_table_exists(dataset) return format_result( run_query( dataset, validate_request_content( body, RequestSchema.build_with_extensions(dataset.get_extensions(), HTTPRequestSettings), timer, dataset, http_request.referrer, ), timer, ))
def test_build_request(body: MutableMapping[str, Any], language: Language, condition: Expression) -> None: dataset = get_dataset("events") entity = dataset.get_default_entity() schema = RequestSchema.build_with_extensions( entity.get_extensions(), HTTPRequestSettings, language, ) request = build_request( body, parse_legacy_query if language == Language.LEGACY else partial( parse_snql_query, []), HTTPRequestSettings, schema, dataset, Timer("test"), "my_request", ) expected_query = Query( from_clause=Entity(EntityKey.EVENTS, entity.get_data_model()), selected_columns=[ SelectedExpression( name="time", expression=Column(alias="_snuba_time", table_name=None, column_name="time"), ), SelectedExpression("count", FunctionCall("_snuba_count", "count", tuple())), ], condition=condition, groupby=[Column("_snuba_time", None, "time")], limit=1000, granularity=60, ) assert request.referrer == "my_request" assert dict(request.body) == body status, differences = request.query.equals(expected_query) assert status == True, f"Query mismatch: {differences}"
def dataset_query(dataset, body, timer): assert http_request.method == 'POST' ensure_table_exists(dataset) schema = RequestSchema.build_with_extensions(dataset.get_extensions()) query_result = parse_and_run_query( dataset, validate_request_content(body, schema, timer), timer, ) def json_default(obj): if isinstance(obj, datetime): return obj.isoformat() elif isinstance(obj, UUID): return str(obj) return obj return (json.dumps(query_result.result, for_json=True, default=json_default), query_result.status, { 'Content-Type': 'application/json' })
def dataset_query( dataset: Dataset, body: MutableMapping[str, Any], timer: Timer, language: Language ) -> Response: assert http_request.method == "POST" referrer = http_request.referrer or "<unknown>" # mypy if language == Language.SNQL: metrics.increment("snql.query.incoming", tags={"referrer": referrer}) parser: Callable[ [RequestParts, RequestSettings, Dataset], Union[Query, CompositeQuery[Entity]], ] = partial(parse_snql_query, []) else: parser = parse_legacy_query with sentry_sdk.start_span(description="build_schema", op="validate"): schema = RequestSchema.build_with_extensions( dataset.get_default_entity().get_extensions(), HTTPRequestSettings, language ) request = build_request( body, parser, HTTPRequestSettings, schema, dataset, timer, referrer ) try: result = parse_and_run_query(dataset, request, timer) # Some metrics to track the adoption of SnQL query_type = "simple" if language == Language.SNQL: if isinstance(request.query, CompositeQuery): if isinstance(request.query.get_from_clause(), JoinClause): query_type = "join" else: query_type = "subquery" metrics.increment( "snql.query.success", tags={"referrer": referrer, "type": query_type} ) except QueryException as exception: status = 500 details: Mapping[str, Any] cause = exception.__cause__ if isinstance(cause, RateLimitExceeded): status = 429 details = { "type": "rate-limited", "message": "rate limit exceeded", } elif isinstance(cause, ClickhouseError): details = { "type": "clickhouse", "message": str(cause), "code": cause.code, } elif isinstance(cause, Exception): details = { "type": "unknown", "message": str(cause), } else: raise # exception should have been chained if language == Language.SNQL: metrics.increment( "snql.query.failed", tags={"referrer": referrer, "status": f"{status}"}, ) return Response( json.dumps( {"error": details, "timing": timer.for_json(), **exception.extra} ), status, {"Content-Type": "application/json"}, ) payload: MutableMapping[str, Any] = {**result.result, "timing": timer.for_json()} if settings.STATS_IN_RESPONSE or request.settings.get_debug(): payload.update(result.extra) return Response(json.dumps(payload), 200, {"Content-Type": "application/json"})