def get_date_range_from_params(params, optional=False, validate_window=True):
"""
Gets a date range from standard date range params we pass to the api.
If `statsPeriod` is passed then convert to a time delta and make sure it
fits within our min/max period length. Values are in the format
<number><period_type>, where period type is one of `s` (seconds),
`m` (minutes), `h` (hours) or `d` (days).
Similarly, `statsPeriodStart` and `statsPeriodEnd` allow for selecting a
relative range, for example: 15 days ago through 8 days ago. This uses the same
format as `statsPeriod`
:param params:
If `start` end `end` are passed, validate them, convert to `datetime` and
returns them if valid.
:param optional: When True, if no params passed then return `(None, None)`.
:param validate_window: When True, validate against min / max time delta.
:return: A length 2 tuple containing start/end or raises an `InvalidParams`
exception
"""
now = timezone.now()
end = now
start = now - MAX_STATS_PERIOD
stats_period = params.get('statsPeriod')
stats_period_start = params.get('statsPeriodStart')
stats_period_end = params.get('statsPeriodEnd')
if stats_period is not None:
start = get_datetime_from_stats_period(stats_period, now)
elif stats_period_start or stats_period_end:
if not all([stats_period_start, stats_period_end]):
raise InvalidParams(
'statsPeriodStart and statsPeriodEnd are both required')
start = get_datetime_from_stats_period(stats_period_start, now)
end = get_datetime_from_stats_period(stats_period_end, now)
elif params.get('start') or params.get('end'):
if not all([params.get('start'), params.get('end')]):
raise InvalidParams('start and end are both required')
try:
start = parse_datetime_string(params['start'])
end = parse_datetime_string(params['end'])
except InvalidQuery as exc:
raise InvalidParams(exc.message)
elif optional:
return None, None
if start > end:
raise InvalidParams('start must be before end')
if validate_window:
delta = end - start
if delta > MAX_STATS_PERIOD:
raise InvalidParams(INVALID_PERIOD_ERROR)
return start, end
def test_converts_stats_period_start_end(self):
"""
Ensures that statsPeriodStart and statsPeriodEnd is converted to start/end.
"""
payload = {
"query_type": ExportQueryType.DISCOVER_STR,
"query_info": {
"env": "test",
"statsPeriodStart": "1w",
"statsPeriodEnd": "5d"
},
}
with self.feature({
"organizations:data-export": True,
"organizations:discover-basic": True
}):
response = self.get_valid_response(self.organization.slug,
status_code=201,
**payload)
data_export = ExportedData.objects.get(id=response.data["id"])
query_info = data_export.query_info
assert parse_datetime_string(
query_info["start"]) == parse_datetime_string(
"2020-05-12T14:00:00")
assert parse_datetime_string(
query_info["end"]) == parse_datetime_string("2020-05-14T14:00:00")
assert "statsPeriod" not in query_info
assert "statsPeriodStart" not in query_info
assert "statsPeriodSEnd" not in query_info
def test_preserves_start_end(self):
"""
Ensures that start/end is preserved
"""
payload = {
"query_type": ExportQueryType.DISCOVER_STR,
"query_info": {
"env": "test",
"start": "2020-05-18T14:00:00",
"end": "2020-05-19T14:00:00",
},
}
with self.feature("organizations:discover-query"):
response = self.get_valid_response(self.organization.slug, status_code=201, **payload)
data_export = ExportedData.objects.get(id=response.data["id"])
query_info = data_export.query_info
assert parse_datetime_string(query_info["start"]) == parse_datetime_string(
"2020-05-18T14:00:00"
)
assert parse_datetime_string(query_info["end"]) == parse_datetime_string(
"2020-05-19T14:00:00"
)
assert "statsPeriod" not in query_info
assert "statsPeriodStart" not in query_info
assert "statsPeriodSEnd" not in query_info
def get_date_range_from_params(params):
# Returns (start, end) or raises an `InvalidParams` exception
now = timezone.now()
end = now
start = now - MAX_STATS_PERIOD
stats_period = params.get('statsPeriod')
if stats_period is not None:
stats_period = parse_stats_period(stats_period)
if stats_period is None or stats_period < MIN_STATS_PERIOD or stats_period >= MAX_STATS_PERIOD:
raise InvalidParams('Invalid statsPeriod')
start = now - stats_period
elif params.get('start') or params.get('end'):
if not all([params.get('start'), params.get('end')]):
raise InvalidParams('start and end are both required')
try:
start = parse_datetime_string(params['start'])
end = parse_datetime_string(params['end'])
except InvalidQuery as exc:
raise InvalidParams(exc.message)
if start > end:
raise InvalidParams('start must be before end')
return (start, end)
def get_date_range_from_params(params, optional=False):
"""
Gets a date range from standard date range params we pass to the api.
If `statsPeriod` is passed then convert to a time delta and make sure it
fits within our min/max period length. Values are in the format
<number><period_type>, where period type is one of `s` (seconds),
`m` (minutes), `h` (hours) or `d` (days).
Similarly, `statsPeriodStart` and `statsPeriodEnd` allow for selecting a
relative range, for example: 15 days ago through 8 days ago. This uses the same
format as `statsPeriod`
:param params:
If `start` end `end` are passed, validate them, convert to `datetime` and
returns them if valid.
:param optional: When True, if no params passed then return `(None, None)`.
:return: A length 2 tuple containing start/end or raises an `InvalidParams`
exception
"""
now = timezone.now()
end = now
start = now - MAX_STATS_PERIOD
stats_period = params.get('statsPeriod')
stats_period_start = params.get('statsPeriodStart')
stats_period_end = params.get('statsPeriodEnd')
if stats_period is not None:
start = get_datetime_from_stats_period(stats_period, now)
elif stats_period_start or stats_period_end:
if not all([stats_period_start, stats_period_end]):
raise InvalidParams('statsPeriodStart and statsPeriodEnd are both required')
start = get_datetime_from_stats_period(stats_period_start, now)
end = get_datetime_from_stats_period(stats_period_end, now)
elif params.get('start') or params.get('end'):
if not all([params.get('start'), params.get('end')]):
raise InvalidParams('start and end are both required')
try:
start = parse_datetime_string(params['start'])
end = parse_datetime_string(params['end'])
except InvalidQuery as exc:
raise InvalidParams(exc.message)
elif optional:
return None, None
if start > end:
raise InvalidParams('start must be before end')
delta = end - start
if delta < MIN_STATS_PERIOD or delta > MAX_STATS_PERIOD:
raise InvalidParams(INVALID_PERIOD_ERROR)
return start, end
def get_date_range_from_params(params, optional=False):
"""
Gets a date range from standard date range params we pass to the api.
If `statsPeriod` is passed then convert to a time delta and make sure it
fits within our min/max period length. Values are in the format
<number><period_type>, where period type is one of `s` (seconds),
`m` (minutes), `h` (hours) or `d` (days).
Similarly, `statsPeriodStart` and `statsPeriodEnd` allow for selecting a
relative range, for example: 15 days ago through 8 days ago. This uses the same
format as `statsPeriod`
:param params:
If `start` end `end` are passed, validate them, convert to `datetime` and
returns them if valid.
:param optional: When True, if no params passed then return `(None, None)`.
:return: A length 2 tuple containing start/end or raises an `InvalidParams`
exception
"""
now = timezone.now()
start, end = default_start_end_dates(now)
stats_period = params.get("statsPeriod")
stats_period_start = params.get("statsPeriodStart")
stats_period_end = params.get("statsPeriodEnd")
if stats_period is not None:
start = get_datetime_from_stats_period(stats_period, now)
elif stats_period_start or stats_period_end:
if not all([stats_period_start, stats_period_end]):
raise InvalidParams(
"statsPeriodStart and statsPeriodEnd are both required")
start = get_datetime_from_stats_period(stats_period_start, now)
end = get_datetime_from_stats_period(stats_period_end, now)
elif params.get("start") or params.get("end"):
if not all([params.get("start"), params.get("end")]):
raise InvalidParams("start and end are both required")
try:
start = parse_datetime_string(params["start"])
end = parse_datetime_string(params["end"])
except InvalidQuery as e:
raise InvalidParams(str(e))
elif optional:
return None, None
if start >= end:
raise InvalidParams("start must be before end")
return start, end
def test_converts_stats_period_start_end(self):
"""
Ensures that statsPeriodStart and statsPeriodEnd is converted to start/end.
"""
payload = self.make_payload("discover", {"statsPeriodStart": "1w", "statsPeriodEnd": "5d"})
with self.feature("organizations:discover-query"):
response = self.get_valid_response(self.org.slug, status_code=201, **payload)
data_export = ExportedData.objects.get(id=response.data["id"])
query_info = data_export.query_info
assert parse_datetime_string(query_info["start"]) == parse_datetime_string(
"2020-05-12T14:00:00"
)
assert parse_datetime_string(query_info["end"]) == parse_datetime_string(
"2020-05-14T14:00:00"
)
assert "statsPeriod" not in query_info
assert "statsPeriodStart" not in query_info
assert "statsPeriodSEnd" not in query_info
def test_preserves_start_end(self):
"""
Ensures that start/end is preserved
"""
payload = self.discover_payload.copy()
payload["query_info"].update({"start": "2020-05-18T14:00:00", "end": "2020-05-19T14:00:00"})
with self.feature("organizations:discover-query"):
response = self.get_valid_response(self.org.slug, status_code=201, **payload)
data_export = ExportedData.objects.get(id=response.data["id"])
query_info = data_export.query_info
assert parse_datetime_string(query_info["start"]) == parse_datetime_string(
"2020-05-18T14:00:00"
)
assert parse_datetime_string(query_info["end"]) == parse_datetime_string(
"2020-05-19T14:00:00"
)
assert "statsPeriod" not in query_info
assert "statsPeriodStart" not in query_info
assert "statsPeriodSEnd" not in query_info
def visit_time_filter(self, node, children):
(search_key, _, operator, search_value) = children
if search_key.name in self.date_keys:
try:
search_value = parse_datetime_string(search_value)
except InvalidQuery as exc:
raise InvalidSearchQuery(exc.message)
return SearchFilter(search_key, operator, SearchValue(search_value))
else:
search_value = operator + search_value if operator != '=' else search_value
return self._handle_basic_filter(search_key, '=', SearchValue(search_value))
def visit_time_filter(self, node, children):
(search_key, _, operator, search_value) = children
if search_key.name in self.date_keys:
try:
search_value = parse_datetime_string(search_value)
except InvalidQuery as exc:
raise InvalidSearchQuery(six.text_type(exc))
return SearchFilter(search_key, operator, SearchValue(search_value))
else:
search_value = operator + search_value if operator != "=" else search_value
return self._handle_basic_filter(search_key, "=", SearchValue(search_value))
def get_date_range_from_params(params, optional=False):
"""
Gets a date range from standard date range params we pass to the api.
If `statsPeriod` is passed then convert to a time delta and make sure it
fits within our min/max period length. Values are in the format
<number><period_type>, where period type is one of `s` (seconds),
`m` (minutes), `h` (hours) or `d` (days).
:param params:
If `start` end `end` are passed, validate them, convert to `datetime` and
returns them if valid.
:param optional: When True, if no params passed then return `(None, None)`.
:return: A length 2 tuple containing start/end or raises an `InvalidParams`
exception
"""
now = timezone.now()
end = now
start = now - MAX_STATS_PERIOD
stats_period = params.get('statsPeriod')
if stats_period is not None:
stats_period = parse_stats_period(stats_period)
if stats_period is None or stats_period < MIN_STATS_PERIOD or stats_period >= MAX_STATS_PERIOD:
raise InvalidParams('Invalid statsPeriod')
start = now - stats_period
elif params.get('start') or params.get('end'):
if not all([params.get('start'), params.get('end')]):
raise InvalidParams('start and end are both required')
try:
start = parse_datetime_string(params['start'])
end = parse_datetime_string(params['end'])
except InvalidQuery as exc:
raise InvalidParams(exc.message)
if start > end:
raise InvalidParams('start must be before end')
elif optional:
return None, None
return start, end
def visit_aggregate_date_filter(self, node, children):
(negation, search_key, _, operator, search_value) = children
search_value = search_value[0]
operator = self.handle_negation(negation, operator)
is_date_aggregate = any(key in search_key.name for key in self.date_keys)
if is_date_aggregate:
try:
search_value = parse_datetime_string(search_value)
except InvalidQuery as exc:
raise InvalidSearchQuery(str(exc))
return AggregateFilter(search_key, operator, SearchValue(search_value))
else:
search_value = operator + search_value if operator != "=" else search_value
return AggregateFilter(search_key, "=", SearchValue(search_value))
def visit_aggregate_date_filter(self, node, children):
(search_key, _, operator, search_value) = children
search_value = search_value[0]
operator = operator[0] if not isinstance(operator, Node) else "="
is_date_aggregate = any(key in search_key.name for key in self.date_keys)
if is_date_aggregate:
try:
search_value = parse_datetime_string(search_value)
except InvalidQuery as exc:
raise InvalidSearchQuery(six.text_type(exc))
return AggregateFilter(search_key, operator, SearchValue(search_value))
else:
search_value = operator + search_value if operator != "=" else search_value
return AggregateFilter(search_key, "=", SearchValue(search_value))
def visit_time_filter(self, node, children):
search_key_node, operator, search_value = children
search_key = search_key_node.text
try:
search_value = parse_datetime_string(search_value)
except InvalidQuery as exc:
raise InvalidSearchQuery(exc.message)
try:
return SearchFilter(
SearchKey(search_key),
operator,
SearchValue(search_value, FIELD_LOOKUP[search_key]['type']),
)
except KeyError:
raise InvalidSearchQuery('Unsupported search term: %s' % (search_key,))
def visit_time_filter(self, node, children):
search_key, _, operator, search_value = children
try:
search_value = parse_datetime_string(search_value)
except InvalidQuery as exc:
raise InvalidSearchQuery(exc.message)
try:
return SearchFilter(
search_key,
operator,
SearchValue(search_value),
)
except KeyError:
raise InvalidSearchQuery('Unsupported search term: %s' %
(search_key, ))
def visit_time_filter(self, node, children):
search_key, _, operator, search_value = children
if search_key.name in self.date_keys:
try:
search_value = parse_datetime_string(search_value)
except InvalidQuery as exc:
raise InvalidSearchQuery(exc.message)
else:
search_value = operator + search_value if operator != '=' else search_value
operator = '='
try:
return SearchFilter(
search_key,
operator,
SearchValue(search_value),
)
except KeyError:
raise InvalidSearchQuery('Unsupported search term: %s' % (search_key,))
def get(self, request: Request, organization) -> Response:
if not self.has_feature(organization, request):
return Response(status=404)
use_snql = self.has_snql_feature(organization, request)
sentry_sdk.set_tag("discover.use-snql", use_snql)
try:
params = self.get_snuba_params(request, organization)
except NoProjects:
return Response([])
with sentry_sdk.start_span(op="discover.endpoint", description="trend_dates"):
middle_date = request.GET.get("middle")
if middle_date:
try:
middle = parse_datetime_string(middle_date)
except InvalidQuery:
raise ParseError(detail=f"{middle_date} is not a valid date format")
if middle <= params["start"] or middle >= params["end"]:
raise ParseError(
detail="The middle date should be within the duration of the query"
)
else:
middle = params["start"] + timedelta(
seconds=(params["end"] - params["start"]).total_seconds() * 0.5
)
middle = datetime.strftime(middle, DateArg.date_format)
trend_type = request.GET.get("trendType", REGRESSION)
if trend_type not in TREND_TYPES:
raise ParseError(detail=f"{trend_type} is not a supported trend type")
trend_function = request.GET.get("trendFunction", "p50()")
try:
function, columns, _ = parse_function(trend_function)
except InvalidSearchQuery as error:
raise ParseError(detail=error)
if len(columns) == 0:
# Default to duration
column = "transaction.duration"
else:
column = columns[0]
selected_columns = self.get_field_list(organization, request)
orderby = self.get_orderby(request)
query = request.GET.get("query")
if use_snql:
with self.handle_query_errors():
trend_query = TrendQueryBuilder(
dataset=Dataset.Discover,
params=params,
selected_columns=selected_columns,
auto_fields=False,
auto_aggregations=True,
use_aggregate_conditions=True,
)
snql_trend_columns = self.resolve_trend_columns(
trend_query, function, column, middle
)
trend_query.columns.extend(snql_trend_columns.values())
trend_query.aggregates.extend(snql_trend_columns.values())
trend_query.params["aliases"] = self.get_snql_function_aliases(
snql_trend_columns, trend_type
)
# Both orderby and conditions need to be resolved after the columns because of aliasing
trend_query.orderby = trend_query.resolve_orderby(orderby)
where, having = trend_query.resolve_conditions(query, use_aggregate_conditions=True)
trend_query.where += where
trend_query.having += having
else:
params["aliases"] = self.get_function_aliases(trend_type)
trend_columns = self.get_trend_columns(function, column, middle)
def data_fn(offset, limit):
if use_snql:
trend_query.offset = Offset(offset)
trend_query.limit = Limit(limit)
result = raw_snql_query(
trend_query.get_snql_query(),
referrer="api.trends.get-percentage-change.wip-snql",
)
result = discover.transform_results(
result, trend_query.function_alias_map, {}, None
)
return result
else:
return discover.query(
selected_columns=selected_columns + trend_columns,
query=query,
params=params,
orderby=orderby,
offset=offset,
limit=limit,
referrer="api.trends.get-percentage-change",
auto_fields=True,
auto_aggregations=True,
use_aggregate_conditions=True,
)
with self.handle_query_errors():
return self.paginate(
request=request,
paginator=GenericOffsetPaginator(data_fn=data_fn),
on_results=self.build_result_handler(
request,
organization,
params,
trend_function,
selected_columns,
orderby,
query,
use_snql,
),
default_per_page=5,
max_per_page=5,
)
try:
search_value = SearchValue(int(search_value.text))
except ValueError:
raise InvalidSearchQuery('Invalid numeric query: %s' %
(search_key, ))
return SearchFilter(search_key, operator, search_value)
else:
search_value = SearchValue(
operator +
search_value.text if operator != '=' else search_value.text, )
return self._handle_basic_filter(search_key, '=', search_value)
def visit_time_filter(self, node, (search_key, _, operator, search_value)):
if search_key.name in self.date_keys:
try:
search_value = parse_datetime_string(search_value)
except InvalidQuery as exc:
raise InvalidSearchQuery(exc.message)
return SearchFilter(search_key, operator,
SearchValue(search_value))
else:
search_value = operator + search_value if operator != '=' else search_value
return self._handle_basic_filter(search_key, '=',
SearchValue(search_value))
def visit_rel_time_filter(self, node, (search_key, _, value)):
if search_key.name in self.date_keys:
try:
from_val, to_val = parse_datetime_range(value.text)
except InvalidQuery as exc:
raise InvalidSearchQuery(exc.message)
def node_visitor(token):
if token["type"] == "spaces":
return None
if token["type"] == "filter":
# Filters with an invalid reason raises to signal to the test
# runner that we should expect this exception
if token.get("invalid"):
raise InvalidSearchQuery(token["invalid"]["reason"])
# Transform the operator to match for list values
if token["value"]["type"] in ["valueTextList", "valueNumberList"]:
operator = "NOT IN" if token["negated"] else "IN"
else:
# Negate the operator if the filter is negated to match
operator = token["operator"] or "="
operator = f"!{operator}" if token["negated"] else operator
key = node_visitor(token["key"])
value = node_visitor(token["value"])
if token["filter"] == "boolean" and token["negated"]:
operator = "="
value = SearchValue(raw_value=1 if value.raw_value == 0 else 0)
return SearchFilter(key, operator, value)
if token["type"] == "keySimple":
return SearchKey(name=token["value"])
if token["type"] == "keyExplicitTag":
return SearchKey(name=f"tags[{token['key']['value']}]")
if token["type"] == "keyAggregate":
name = node_visitor(token["name"]).name
# Consistent join aggregate function parameters
args = ", ".join(arg["value"]["value"]
for arg in token["args"]["args"])
return AggregateKey(name=f"{name}({args})")
if token["type"] == "valueText":
# Noramlize values by removing the escaped quotes
value = token["value"].replace('\\"', '"')
return SearchValue(raw_value=value)
if token["type"] == "valueNumber":
return SearchValue(
raw_value=parse_numeric_value(token["value"], token["unit"]))
if token["type"] == "valueTextList":
return SearchValue(
raw_value=[item["value"]["value"] for item in token["items"]])
if token["type"] == "valueNumberList":
return SearchValue(raw_value=[
item["value"]["rawValue"] for item in token["items"]
])
if token["type"] == "valueIso8601Date":
return SearchValue(raw_value=parse_datetime_string(token["value"]))
if token["type"] == "valueDuration":
return SearchValue(
raw_value=parse_duration(token["value"], token["unit"]))
if token["type"] == "valueRelativeDate":
return SearchValue(
raw_value=parse_duration(token["value"], token["unit"]))
if token["type"] == "valueBoolean":
return SearchValue(raw_value=int(token["value"]))
if token["type"] == "freeText":
if token["quoted"]:
# Normalize quotes
value = token["value"].replace('\\"', '"')
else:
# Normalize spacing
value = token["value"].strip(" ")
if value == "":
return None
return SearchFilter(
key=SearchKey(name="message"),
operator="=",
value=SearchValue(raw_value=value),
)
def get(self, request, organization):
if not self.has_feature(organization, request):
return Response(status=404)
try:
params = self.get_snuba_params(request, organization)
except NoProjects:
return Response([])
with sentry_sdk.start_span(op="discover.endpoint", description="trend_dates"):
middle_date = request.GET.get("middle")
if middle_date:
try:
middle = parse_datetime_string(middle_date)
except InvalidQuery:
raise ParseError(detail="{} is not a valid date format".format(middle_date))
if middle <= params["start"] or middle >= params["end"]:
raise ParseError(
detail="The middle date should be within the duration of the query"
)
else:
middle = params["start"] + timedelta(
seconds=(params["end"] - params["start"]).total_seconds() * 0.5
)
start, middle, end = (
datetime.strftime(params["start"], DateArg.date_format),
datetime.strftime(middle, DateArg.date_format),
datetime.strftime(params["end"], DateArg.date_format),
)
trend_type = request.GET.get("trendType", REGRESSION)
if trend_type not in TREND_TYPES:
raise ParseError(detail=u"{} is not a supported trend type".format(trend_type))
params["aliases"] = self.get_function_aliases(trend_type)
trend_function = request.GET.get("trendFunction", "p50()")
function, columns = parse_function(trend_function)
trend_columns = self.get_trend_columns(function, columns, start, middle, end)
selected_columns = request.GET.getlist("field")[:]
orderby = self.get_orderby(request)
query = request.GET.get("query")
def data_fn(offset, limit):
return discover.query(
selected_columns=selected_columns + trend_columns,
query=query,
params=params,
orderby=orderby,
offset=offset,
limit=limit,
referrer="api.trends.get-percentage-change",
auto_fields=True,
auto_aggregations=True,
use_aggregate_conditions=True,
)
with self.handle_query_errors():
return self.paginate(
request=request,
paginator=GenericOffsetPaginator(data_fn=data_fn),
on_results=self.build_result_handler(
request, organization, params, trend_function, selected_columns, orderby, query
),
default_per_page=5,
max_per_page=5,
)
