def get_datetime_from_stats_period(stats_period, now=None):
if now is None:
now = timezone.now()
stats_period = parse_stats_period(stats_period)
if stats_period is None:
raise InvalidParams("Invalid statsPeriod")
return now - stats_period
def get_datetime_from_stats_period(stats_period, now=None):
if now is None:
now = timezone.now()
stats_period = parse_stats_period(stats_period)
if stats_period is None:
raise InvalidParams('Invalid statsPeriod')
return now - stats_period
def get_date_range_from_params(params):
# Returns (start, end) or raises an `InvalidParams` exception
now = timezone.now()
end = now
start = now - MAX_STATS_PERIOD
stats_period = params.get('statsPeriod')
if stats_period is not None:
stats_period = parse_stats_period(stats_period)
if stats_period is None or stats_period < MIN_STATS_PERIOD or stats_period >= MAX_STATS_PERIOD:
raise InvalidParams('Invalid statsPeriod')
start = now - stats_period
elif params.get('start') or params.get('end'):
if not all([params.get('start'), params.get('end')]):
raise InvalidParams('start and end are both required')
try:
start = parse_datetime_string(params['start'])
end = parse_datetime_string(params['end'])
except InvalidQuery as exc:
raise InvalidParams(exc.message)
if start > end:
raise InvalidParams('start must be before end')
return (start, end)
def get(self, request, organization):
try:
snuba_args = self.get_snuba_query_args(request, organization)
except OrganizationEventsError as exc:
return Response({'detail': exc.message}, status=400)
interval = parse_stats_period(request.GET.get('interval', '1h'))
if interval is None:
interval = timedelta(hours=1)
rollup = int(interval.total_seconds())
result = raw_query(aggregations=[
('count()', '', 'count'),
],
orderby='time',
groupby=['time'],
rollup=rollup,
referrer='api.organization-events-stats',
**snuba_args)
serializer = SnubaTSResultSerializer(organization, None, request.user)
return Response(
serializer.serialize(
SnubaTSResult(result, snuba_args['start'], snuba_args['end'],
rollup), ),
status=200,
)
def get(self, request, organization):
try:
snuba_args = self.get_snuba_query_args(request, organization)
except OrganizationEventsError as exc:
return Response({'detail': exc.message}, status=400)
except NoProjects:
return Response({'data': []})
interval = parse_stats_period(request.GET.get('interval', '1h'))
if interval is None:
interval = timedelta(hours=1)
rollup = int(interval.total_seconds())
result = raw_query(
aggregations=[
('count()', '', 'count'),
],
orderby='time',
groupby=['time'],
rollup=rollup,
referrer='api.organization-events-stats',
limit=10000,
**snuba_args
)
serializer = SnubaTSResultSerializer(organization, None, request.user)
return Response(
serializer.serialize(
SnubaTSResult(result, snuba_args['start'], snuba_args['end'], rollup),
),
status=200,
)
def get_date_range_rollup_from_params(
params,
minimum_interval="1h",
default_interval="",
round_range=False,
max_points=MAX_ROLLUP_POINTS,
):
"""
Similar to `get_date_range_from_params`, but this also parses and validates
an `interval`, as `get_rollup_from_request` would do.
This also optionally rounds the returned range to the given `interval`.
The rounding uses integer arithmetic on unix timestamps, so might yield
unexpected results when the interval is > 1d.
"""
minimum_interval = parse_stats_period(minimum_interval).total_seconds()
interval = parse_stats_period(params.get("interval", default_interval))
interval = minimum_interval if interval is None else interval.total_seconds(
)
if interval <= 0:
raise InvalidParams("Interval cannot result in a zero duration.")
# round the interval up to the minimum
interval = int(minimum_interval * math.ceil(interval / minimum_interval))
start, end = get_date_range_from_params(params)
date_range = end - start
# round the range up to a multiple of the interval
if round_range:
date_range = timedelta(
seconds=int(interval *
math.ceil(date_range.total_seconds() / interval)))
if date_range.total_seconds() / interval > max_points:
raise InvalidParams(
"Your interval and date range would create too many results. "
"Use a larger interval, or a smaller date range.")
if round_range:
end_ts = int(interval * math.ceil(to_timestamp(end) / interval))
end = to_datetime(end_ts)
start = end - date_range
return start, end, interval
def test_parse_stats_period():
assert parse_stats_period("3s") == datetime.timedelta(seconds=3)
assert parse_stats_period("30m") == datetime.timedelta(minutes=30)
assert parse_stats_period("1h") == datetime.timedelta(hours=1)
assert parse_stats_period("20d") == datetime.timedelta(days=20)
assert parse_stats_period("20f") is None
assert parse_stats_period("-1s") is None
assert parse_stats_period("4w") == datetime.timedelta(weeks=4)
def test_parse_stats_period():
assert parse_stats_period('3s') == datetime.timedelta(seconds=3)
assert parse_stats_period('30m') == datetime.timedelta(minutes=30)
assert parse_stats_period('1h') == datetime.timedelta(hours=1)
assert parse_stats_period('20d') == datetime.timedelta(days=20)
assert parse_stats_period('20f') is None
assert parse_stats_period('-1s') is None
assert parse_stats_period('4w') == datetime.timedelta(weeks=4)
def get_datetime_from_stats_period(stats_period, now=None):
if now is None:
now = timezone.now()
parsed_stats_period = parse_stats_period(stats_period)
if parsed_stats_period is None:
raise InvalidParams(f"Invalid statsPeriod: {stats_period!r}")
try:
return now - parsed_stats_period
except OverflowError:
raise InvalidParams(f"Invalid statsPeriod: {stats_period!r}")
def get(self, request, organization):
"""
Returns a time series view over statsPeriod over interval.
"""
try:
lookup = SnubaLookup.get(request.GET['tag'])
except KeyError:
raise ResourceDoesNotExist
try:
start, end = get_date_range_from_params(request.GET)
except InvalidParams as exc:
return Response({'detail': exc.message}, status=400)
interval = parse_stats_period(request.GET.get('interval', '1h'))
if interval is None:
interval = timedelta(hours=1)
try:
project_ids = self.get_project_ids(request, organization)
except ValueError:
return Response({'detail': 'Invalid project ids'}, status=400)
if not project_ids:
return self.empty()
environment = self.get_environment(request, organization)
query_condition = self.get_query_condition(request, organization)
rollup = int(interval.total_seconds())
data = query(
end=end,
start=start,
rollup=rollup,
selected_columns=lookup.selected_columns,
aggregations=[
('count()', '', 'count'),
],
filter_keys={
'project_id': project_ids,
},
conditions=lookup.conditions + query_condition + environment,
groupby=['time'] + lookup.columns,
orderby='time',
)
serializer = SnubaTSResultSerializer(organization, lookup, request.user)
return Response(
serializer.serialize(
SnubaTSResult(data, start, end, rollup),
),
status=200,
)
def get_rollup(self, request, params):
interval = parse_stats_period(request.GET.get("interval", "1h"))
if interval is None:
interval = timedelta(hours=1)
date_range = params["end"] - params["start"]
if date_range.total_seconds() / interval.total_seconds() > MAX_POINTS:
raise InvalidSearchQuery(
"Your interval and date range would create too many results. "
"Use a larger interval, or a smaller date range.")
return int(interval.total_seconds())
def test_parse_stats_period():
assert parse_stats_period('3s') == datetime.timedelta(seconds=3)
assert parse_stats_period('30m') == datetime.timedelta(minutes=30)
assert parse_stats_period('1h') == datetime.timedelta(hours=1)
assert parse_stats_period('20d') == datetime.timedelta(days=20)
assert parse_stats_period('20f') is None
assert parse_stats_period('-1s') is None
def get_rollup(self, request, params):
interval = parse_stats_period(request.GET.get("interval", "24h"))
if interval is None:
interval = timedelta(hours=1)
date_range = params["end"] - params["start"]
if date_range.total_seconds() / interval.total_seconds() > MAX_POINTS:
raise ProjectEventsError(
"Your interval and date range would create too many results. "
"Use a larger interval, or a smaller date range.")
# The minimum interval is one hour on the server
return max(int(interval.total_seconds()), 3600)
def get_date_range(params: Mapping) -> Tuple[datetime, datetime, int]:
"""Get start, end, rollup for the given parameters.
Apply a similar logic as `sessions_v2.get_constrained_date_range`,
but with fewer constraints. More constraints may be added in the future.
Note that this function returns a right-exclusive date range [start, end),
contrary to the one used in sessions_v2.
"""
interval = parse_stats_period(params.get("interval", "1h"))
interval = int(3600 if interval is None else interval.total_seconds())
# hard code min. allowed resolution to 10 seconds
allowed_resolution = AllowedResolution.ten_seconds
smallest_interval, interval_str = allowed_resolution.value
if interval % smallest_interval != 0 or interval < smallest_interval:
raise InvalidParams(
f"The interval has to be a multiple of the minimum interval of {interval_str}."
)
if ONE_DAY % interval != 0:
raise InvalidParams(
"The interval should divide one day without a remainder.")
start, end = get_date_range_from_params(params)
date_range = end - start
date_range = timedelta(
seconds=int(interval *
math.ceil(date_range.total_seconds() / interval)))
if date_range.total_seconds() / interval > MAX_POINTS:
raise InvalidParams(
"Your interval and date range would create too many results. "
"Use a larger interval, or a smaller date range.")
end_ts = int(interval * math.ceil(to_timestamp(end) / interval))
end = to_datetime(end_ts)
start = end - date_range
# NOTE: The sessions_v2 implementation cuts the `end` time to now + 1 minute
# if `end` is in the future. This allows for better real time results when
# caching is enabled on the snuba queries. Removed here for simplicity,
# but we might want to reconsider once caching becomes an issue for metrics.
return start, end, interval
def validate_range(self, attrs, source):
has_start = bool(attrs.get('start'))
has_end = bool(attrs.get('end'))
has_range = bool(attrs.get('range'))
if has_start != has_end or has_range == has_start:
raise serializers.ValidationError('Either start and end dates or range is required')
# Populate start and end if only range is provided
if (attrs.get(source)):
delta = parse_stats_period(attrs[source])
if (delta is None):
raise serializers.ValidationError('Invalid range')
attrs['start'] = timezone.now() - delta
attrs['end'] = timezone.now()
return attrs
def get(self, request, organization):
try:
if features.has("organizations:events-v2",
organization,
actor=request.user):
params = self.get_filter_params(request, organization)
snuba_args = self.get_snuba_query_args(request, organization,
params)
else:
snuba_args = self.get_snuba_query_args_legacy(
request, organization)
except (OrganizationEventsError, InvalidSearchQuery) as exc:
raise ParseError(detail=six.text_type(exc))
except NoProjects:
return Response({"data": []})
interval = parse_stats_period(request.GET.get("interval", "1h"))
if interval is None:
interval = timedelta(hours=1)
rollup = int(interval.total_seconds())
snuba_args = self.get_field(request, snuba_args)
result = snuba.transform_aliases_and_query(
skip_conditions=True,
aggregations=snuba_args.get("aggregations"),
conditions=snuba_args.get("conditions"),
filter_keys=snuba_args.get("filter_keys"),
start=snuba_args.get("start"),
end=snuba_args.get("end"),
orderby="time",
groupby=["time"],
rollup=rollup,
referrer="api.organization-events-stats",
limit=10000,
)
serializer = SnubaTSResultSerializer(organization, None, request.user)
return Response(
serializer.serialize(
snuba.SnubaTSResult(result, snuba_args["start"],
snuba_args["end"], rollup)),
status=200,
)
def get(self, request, organization):
try:
snuba_args = self.get_snuba_query_args(request, organization)
except OrganizationEventsError as exc:
return Response({'detail': exc.message}, status=400)
except NoProjects:
return Response({'data': []})
interval = parse_stats_period(request.GET.get('interval', '1h'))
if interval is None:
interval = timedelta(hours=1)
rollup = int(interval.total_seconds())
y_axis = request.GET.get('yAxis', None)
if not y_axis or y_axis == 'event_count':
aggregations = [('count()', '', 'count')]
elif y_axis == 'user_count':
aggregations = [
('uniq', 'tags[sentry:user]', 'count'),
]
snuba_args['filter_keys']['tags_key'] = ['sentry:user']
else:
return Response(
{'detail': 'Param yAxis value %s not recognized.' % y_axis}, status=400)
result = raw_query(
aggregations=aggregations,
orderby='time',
groupby=['time'],
rollup=rollup,
referrer='api.organization-events-stats',
limit=10000,
**snuba_args
)
serializer = SnubaTSResultSerializer(organization, None, request.user)
return Response(
serializer.serialize(
SnubaTSResult(result, snuba_args['start'], snuba_args['end'], rollup),
),
status=200,
)
def validate_range(self, attrs, source):
has_start = bool(attrs.get('start'))
has_end = bool(attrs.get('end'))
has_range = bool(attrs.get('range'))
if has_start != has_end or has_range == has_start:
raise serializers.ValidationError('Either start and end dates or range is required')
# Populate start and end if only range is provided
if (attrs.get(source)):
delta = parse_stats_period(attrs[source])
if (delta is None):
raise serializers.ValidationError('Invalid range')
attrs['start'] = timezone.now() - delta
attrs['end'] = timezone.now()
return attrs
def _params_to_camunda(self, endpoint, method, params):
"""
Edits the supplied parameter making sure that they work for Camunda's API interface
"""
_params = dict()
for param in params:
_params[param] = params[param]
params = _params
if "limit" in params:
params["maxResults"] = params["limit"]
del params["limit"]
# We support date parameters that allow for syntax -7d etc (supported in
# Sentry). Such parameters
for param in params:
meta_info = self._get_param_meta(endpoint, method, param)
if meta_info and meta_info["type"] == "date":
params[param] = parse_stats_period(params[param])
return params
def get(self, request, organization):
try:
snuba_args = self.get_snuba_query_args_legacy(request, organization)
except OrganizationEventsError as exc:
return Response({"detail": exc.message}, status=400)
except NoProjects:
return Response({"data": []})
interval = parse_stats_period(request.GET.get("interval", "1h"))
if interval is None:
interval = timedelta(hours=1)
rollup = int(interval.total_seconds())
y_axis = request.GET.get("yAxis", None)
if not y_axis or y_axis == "event_count":
aggregations = [("count()", "", "count")]
elif y_axis == "user_count":
aggregations = [("uniq", "tags[sentry:user]", "count")]
snuba_args["filter_keys"]["tags_key"] = ["sentry:user"]
else:
return Response({"detail": "Param yAxis value %s not recognized." % y_axis}, status=400)
result = raw_query(
aggregations=aggregations,
orderby="time",
groupby=["time"],
rollup=rollup,
referrer="api.organization-events-stats",
limit=10000,
**snuba_args
)
serializer = SnubaTSResultSerializer(organization, None, request.user)
return Response(
serializer.serialize(
SnubaTSResult(result, snuba_args["start"], snuba_args["end"], rollup)
),
status=200,
)
def get_date_range_from_params(params, optional=False):
"""
Gets a date range from standard date range params we pass to the api.
If `statsPeriod` is passed then convert to a time delta and make sure it
fits within our min/max period length. Values are in the format
<number><period_type>, where period type is one of `s` (seconds),
`m` (minutes), `h` (hours) or `d` (days).
:param params:
If `start` end `end` are passed, validate them, convert to `datetime` and
returns them if valid.
:param optional: When True, if no params passed then return `(None, None)`.
:return: A length 2 tuple containing start/end or raises an `InvalidParams`
exception
"""
now = timezone.now()
end = now
start = now - MAX_STATS_PERIOD
stats_period = params.get('statsPeriod')
if stats_period is not None:
stats_period = parse_stats_period(stats_period)
if stats_period is None or stats_period < MIN_STATS_PERIOD or stats_period >= MAX_STATS_PERIOD:
raise InvalidParams('Invalid statsPeriod')
start = now - stats_period
elif params.get('start') or params.get('end'):
if not all([params.get('start'), params.get('end')]):
raise InvalidParams('start and end are both required')
try:
start = parse_datetime_string(params['start'])
end = parse_datetime_string(params['end'])
except InvalidQuery as exc:
raise InvalidParams(exc.message)
if start > end:
raise InvalidParams('start must be before end')
elif optional:
return None, None
return start, end
def get_rollup(self, request):
interval = parse_stats_period(request.GET.get("interval", "1h"))
if interval is None:
interval = timedelta(hours=1)
return int(interval.total_seconds())
def _get_constrained_date_range(params, allow_minute_resolution=False):
interval = parse_stats_period(params.get("interval", "1h"))
interval = int(3600 if interval is None else interval.total_seconds())
smallest_interval = ONE_MINUTE if allow_minute_resolution else ONE_HOUR
if interval % smallest_interval != 0 or interval < smallest_interval:
interval_str = "one minute" if allow_minute_resolution else "one hour"
raise InvalidParams(
f"The interval has to be a multiple of the minimum interval of {interval_str}."
)
if interval > ONE_DAY:
raise InvalidParams("The interval has to be less than one day.")
if ONE_DAY % interval != 0:
raise InvalidParams(
"The interval should divide one day without a remainder.")
using_minute_resolution = interval % ONE_HOUR != 0
start, end = get_date_range_from_params(params)
# if `end` is explicitly given, we add a second to it, so it is treated as
# inclusive. the rounding logic down below will take care of the rest.
if params.get("end"):
end += timedelta(seconds=1)
date_range = end - start
# round the range up to a multiple of the interval.
# the minimum is 1h so the "totals" will not go out of sync, as they will
# use the materialized storage due to no grouping on the `started` column.
rounding_interval = int(math.ceil(interval / ONE_HOUR) * ONE_HOUR)
date_range = timedelta(
seconds=int(rounding_interval *
math.ceil(date_range.total_seconds() / rounding_interval)))
if using_minute_resolution:
if date_range.total_seconds() > 6 * ONE_HOUR:
raise InvalidParams(
"The time-range when using one-minute resolution intervals is restricted to 6 hours."
)
if (datetime.now(tz=pytz.utc) - start).total_seconds() > 30 * ONE_DAY:
raise InvalidParams(
"The time-range when using one-minute resolution intervals is restricted to the last 30 days."
)
if date_range.total_seconds() / interval > MAX_POINTS:
raise InvalidParams(
"Your interval and date range would create too many results. "
"Use a larger interval, or a smaller date range.")
end_ts = int(rounding_interval *
math.ceil(to_timestamp(end) / rounding_interval))
end = to_datetime(end_ts)
# when expanding the rounding interval, we would adjust the end time too far
# to the future, in which case the start time would not actually contain our
# desired date range. adjust for this by extend the time by another interval.
# for example, when "45m" means the range from 08:49:00-09:34:00, our rounding
# has to go from 08:00:00 to 10:00:00.
if rounding_interval > interval and (end - date_range) > start:
date_range += timedelta(seconds=rounding_interval)
start = end - date_range
return start, end, interval
def get_constrained_date_range(
params, allow_minute_resolution=False, max_points=MAX_POINTS
) -> Tuple[datetime, datetime, int]:
interval = parse_stats_period(params.get("interval", "1h"))
interval = int(3600 if interval is None else interval.total_seconds())
smallest_interval = ONE_MINUTE if allow_minute_resolution else ONE_HOUR
if interval % smallest_interval != 0 or interval < smallest_interval:
interval_str = "one minute" if allow_minute_resolution else "one hour"
raise InvalidParams(
f"The interval has to be a multiple of the minimum interval of {interval_str}."
)
if interval > ONE_DAY:
raise InvalidParams("The interval has to be less than one day.")
if ONE_DAY % interval != 0:
raise InvalidParams("The interval should divide one day without a remainder.")
using_minute_resolution = interval % ONE_HOUR != 0
start, end = get_date_range_from_params(params)
now = get_now()
# if `end` is explicitly given, we add a second to it, so it is treated as
# inclusive. the rounding logic down below will take care of the rest.
if params.get("end"):
end += timedelta(seconds=1)
date_range = end - start
# round the range up to a multiple of the interval.
# the minimum is 1h so the "totals" will not go out of sync, as they will
# use the materialized storage due to no grouping on the `started` column.
# NOTE: we can remove the difference between `interval` / `rounding_interval`
# as soon as snuba can provide us with grouped totals in the same query
# as the timeseries (using `WITH ROLLUP` in clickhouse)
rounding_interval = int(math.ceil(interval / ONE_HOUR) * ONE_HOUR)
date_range = timedelta(
seconds=int(rounding_interval * math.ceil(date_range.total_seconds() / rounding_interval))
)
if using_minute_resolution:
if date_range.total_seconds() > 6 * ONE_HOUR:
raise InvalidParams(
"The time-range when using one-minute resolution intervals is restricted to 6 hours."
)
if (now - start).total_seconds() > 30 * ONE_DAY:
raise InvalidParams(
"The time-range when using one-minute resolution intervals is restricted to the last 30 days."
)
if date_range.total_seconds() / interval > max_points:
raise InvalidParams(
"Your interval and date range would create too many results. "
"Use a larger interval, or a smaller date range."
)
end_ts = int(rounding_interval * math.ceil(to_timestamp(end) / rounding_interval))
end = to_datetime(end_ts)
# when expanding the rounding interval, we would adjust the end time too far
# to the future, in which case the start time would not actually contain our
# desired date range. adjust for this by extend the time by another interval.
# for example, when "45m" means the range from 08:49:00-09:34:00, our rounding
# has to go from 08:00:00 to 10:00:00.
if rounding_interval > interval and (end - date_range) > start:
date_range += timedelta(seconds=rounding_interval)
start = end - date_range
# snuba <-> sentry has a 5 minute cache for *exact* queries, which these
# are because of the way we do our rounding. For that reason we round the end
# of "realtime" queries to one minute into the future to get a one-minute cache instead.
if end > now:
end = to_datetime(ONE_MINUTE * (math.floor(to_timestamp(now) / ONE_MINUTE) + 1))
return start, end, interval
def validate_interval(self, interval):
if parse_stats_period(interval) is None:
raise serializers.ValidationError("Invalid interval")
return interval
def get(self, request, organization):
"""
Returns a top-N view based on queryset over time period, as well as previous
period.
"""
try:
lookup = SnubaLookup.get(request.GET['tag'])
except KeyError:
raise ResourceDoesNotExist
stats_period = parse_stats_period(request.GET.get('statsPeriod', '24h'))
if stats_period is None or stats_period < self.MIN_STATS_PERIOD or stats_period >= self.MAX_STATS_PERIOD:
return Response({'detail': 'Invalid statsPeriod'}, status=400)
try:
limit = int(request.GET.get('limit', '5'))
except ValueError:
return Response({'detail': 'Invalid limit'}, status=400)
if limit > self.MAX_LIMIT:
return Response({'detail': 'Invalid limit: max %d' % self.MAX_LIMIT}, status=400)
if limit <= 0:
return self.empty()
try:
project_ids = self.get_project_ids(request, organization)
except ValueError:
return Response({'detail': 'Invalid project ids'}, status=400)
if not project_ids:
return self.empty()
environment = self.get_environment(request, organization)
query_condition = self.get_query_condition(request, organization)
aggregations = [('count()', '', 'count')]
# If we pass `?topk` this means we also are
# layering on top_projects and total_projects for each value.
if 'topk' in request.GET:
try:
topk = int(request.GET['topk'])
except ValueError:
return Response({'detail': 'Invalid topk'}, status=400)
aggregations += [
('topK(%d)' % topk, 'project_id', 'top_projects'),
('uniq', 'project_id', 'total_projects'),
]
now = timezone.now()
data = query(
end=now,
start=now - stats_period,
selected_columns=lookup.selected_columns,
aggregations=aggregations,
filter_keys={
'project_id': project_ids,
},
conditions=lookup.conditions + query_condition + environment,
groupby=lookup.columns,
orderby='-count',
limit=limit,
)
if not data['data']:
return self.empty()
# Convert our results from current period into a condition
# to be used in the next query for the previous period.
# This way our values overlap to be able to deduce a delta.
values = []
is_null = False
for row in data['data']:
value = lookup.encoder(value_from_row(row, lookup.columns))
if value is None:
is_null = True
else:
values.append(value)
previous = query(
end=now - stats_period,
start=now - (stats_period * 2),
selected_columns=lookup.selected_columns,
aggregations=[
('count()', '', 'count'),
],
filter_keys={
'project_id': project_ids,
},
conditions=lookup.conditions + query_condition + environment + [
[lookup.filter_key, 'IN', values] if values else [],
[lookup.tagkey, 'IS NULL', None] if is_null else [],
],
groupby=lookup.columns,
)
serializer = SnubaResultSerializer(organization, lookup, request.user)
return Response(
serializer.serialize(
SnubaResultSet(data, previous),
),
status=200,
)
def get(self, request, organization):
"""
Returns a top-N view based on queryset over time period, as well as previous
period.
"""
try:
lookup = SnubaLookup.get(request.GET['tag'])
except KeyError:
raise ResourceDoesNotExist
stats_period = parse_stats_period(request.GET.get(
'statsPeriod', '24h'))
if stats_period is None or stats_period < self.MIN_STATS_PERIOD or stats_period >= self.MAX_STATS_PERIOD:
return Response({'detail': 'Invalid statsPeriod'}, status=400)
try:
limit = int(request.GET.get('limit', '5'))
except ValueError:
return Response({'detail': 'Invalid limit'}, status=400)
if limit > self.MAX_LIMIT:
return Response(
{'detail': 'Invalid limit: max %d' % self.MAX_LIMIT},
status=400)
if limit <= 0:
return self.empty()
try:
project_ids = self.get_project_ids(request, organization)
except ValueError:
return Response({'detail': 'Invalid project ids'}, status=400)
if not project_ids:
return self.empty()
environment = self.get_environment(request, organization)
query_condition = self.get_query_condition(request, organization)
aggregations = [('count()', '', 'count')]
# If we pass `?topk` this means we also are
# layering on top_projects and total_projects for each value.
if 'topk' in request.GET:
try:
topk = int(request.GET['topk'])
except ValueError:
return Response({'detail': 'Invalid topk'}, status=400)
aggregations += [
('topK(%d)' % topk, 'project_id', 'top_projects'),
('uniq', 'project_id', 'total_projects'),
]
now = timezone.now()
data = query(
end=now,
start=now - stats_period,
selected_columns=lookup.selected_columns,
aggregations=aggregations,
filter_keys={
'project_id': project_ids,
},
conditions=lookup.conditions + query_condition + environment,
groupby=lookup.columns,
orderby='-count',
limit=limit,
)
if not data['data']:
return self.empty()
# Convert our results from current period into a condition
# to be used in the next query for the previous period.
# This way our values overlap to be able to deduce a delta.
values = []
is_null = False
for row in data['data']:
value = lookup.encoder(value_from_row(row, lookup.columns))
if value is None:
is_null = True
else:
values.append(value)
previous = query(
end=now - stats_period,
start=now - (stats_period * 2),
selected_columns=lookup.selected_columns,
aggregations=[
('count()', '', 'count'),
],
filter_keys={
'project_id': project_ids,
},
conditions=lookup.conditions + query_condition + environment + [
[lookup.filter_key, 'IN', values] if values else [],
[lookup.tagkey, 'IS NULL', None] if is_null else [],
],
groupby=lookup.columns,
)
serializer = SnubaResultSerializer(organization, lookup, request.user)
return Response(
serializer.serialize(SnubaResultSet(data, previous), ),
status=200,
)
def get_event_stats_data(
self,
request: Request,
organization: Organization,
get_event_stats: Callable[
[Sequence[str], str, Dict[str, str], int, bool, Optional[timedelta]], SnubaTSResult
],
top_events: int = 0,
query_column: str = "count()",
params: Optional[Dict[str, Any]] = None,
query: Optional[str] = None,
allow_partial_buckets: bool = False,
zerofill_results: bool = True,
comparison_delta: Optional[timedelta] = None,
) -> Dict[str, Any]:
with self.handle_query_errors():
with sentry_sdk.start_span(
op="discover.endpoint", description="base.stats_query_creation"
):
columns = request.GET.getlist("yAxis", [query_column])
if query is None:
query = request.GET.get("query")
if params is None:
try:
# events-stats is still used by events v1 which doesn't require global views
params = self.get_snuba_params(
request, organization, check_global_views=False
)
except NoProjects:
return {"data": []}
try:
rollup = get_rollup_from_request(
request,
params,
default_interval=None,
error=InvalidSearchQuery(),
top_events=top_events,
)
# If the user sends an invalid interval, use the default instead
except InvalidSearchQuery:
sentry_sdk.set_tag("user.invalid_interval", request.GET.get("interval"))
date_range = params["end"] - params["start"]
stats_period = parse_stats_period(get_interval_from_range(date_range, False))
rollup = int(stats_period.total_seconds()) if stats_period is not None else 3600
if comparison_delta is not None:
retention = quotas.get_event_retention(organization=organization)
comparison_start = params["start"] - comparison_delta
if retention and comparison_start < timezone.now() - timedelta(days=retention):
raise ValidationError("Comparison period is outside your retention window")
# Backwards compatibility for incidents which uses the old
# column aliases as it straddles both versions of events/discover.
# We will need these aliases until discover2 flags are enabled for all
# users.
# We need these rollup columns to generate correct events-stats results
column_map = {
"user_count": "count_unique(user)",
"event_count": "count()",
"epm()": "epm(%d)" % rollup,
"eps()": "eps(%d)" % rollup,
"tpm()": "tpm(%d)" % rollup,
"tps()": "tps(%d)" % rollup,
}
query_columns = [column_map.get(column, column) for column in columns]
with sentry_sdk.start_span(op="discover.endpoint", description="base.stats_query"):
result = get_event_stats(
query_columns, query, params, rollup, zerofill_results, comparison_delta
)
serializer = SnubaTSResultSerializer(organization, None, request.user)
with sentry_sdk.start_span(op="discover.endpoint", description="base.stats_serialization"):
# When the request is for top_events, result can be a SnubaTSResult in the event that
# there were no top events found. In this case, result contains a zerofilled series
# that acts as a placeholder.
is_multiple_axis = len(query_columns) > 1
if top_events > 0 and isinstance(result, dict):
results = {}
for key, event_result in result.items():
if is_multiple_axis:
results[key] = self.serialize_multiple_axis(
serializer,
event_result,
columns,
query_columns,
allow_partial_buckets,
zerofill_results=zerofill_results,
)
else:
# Need to get function alias if count is a field, but not the axis
results[key] = serializer.serialize(
event_result,
column=resolve_axis_column(query_columns[0]),
allow_partial_buckets=allow_partial_buckets,
zerofill_results=zerofill_results,
)
serialized_result = results
elif is_multiple_axis:
serialized_result = self.serialize_multiple_axis(
serializer,
result,
columns,
query_columns,
allow_partial_buckets,
zerofill_results=zerofill_results,
)
else:
extra_columns = None
if comparison_delta:
extra_columns = ["comparisonCount"]
serialized_result = serializer.serialize(
result,
resolve_axis_column(query_columns[0]),
allow_partial_buckets=allow_partial_buckets,
zerofill_results=zerofill_results,
extra_columns=extra_columns,
)
return serialized_result
def unfurl_discover(
data: HttpRequest,
integration: Integration,
links: List[UnfurlableUrl],
user: Optional["User"],
) -> UnfurledUrl:
orgs_by_slug = {org.slug: org for org in integration.organizations.all()}
unfurls = {}
for link in links:
org_slug = link.args["org_slug"]
org = orgs_by_slug.get(org_slug)
# If the link shared is an org w/o the slack integration do not unfurl
if not org:
continue
if not features.has("organizations:discover-basic", org):
continue
params = link.args["query"]
query_id = params.get("id", None)
saved_query = {}
if query_id:
try:
response = client.get(
auth=ApiKey(organization=org, scope_list=["org:read"]),
path=f"/organizations/{org_slug}/discover/saved/{query_id}/",
)
except Exception as exc:
logger.error(
f"Failed to load saved query for unfurl: {exc}",
exc_info=True,
)
else:
saved_query = response.data
# Override params from Discover Saved Query if they aren't in the URL
params.setlist(
"order",
params.getlist("sort")
or (to_list(saved_query.get("orderby")) if saved_query.get("orderby") else []),
)
params.setlist("name", params.getlist("name") or to_list(saved_query.get("name")))
fields = params.getlist("field") or to_list(saved_query.get("fields"))
# Mimic Discover to pick the first aggregate as the yAxis option if
# one isn't specified.
axis_options = [field for field in fields if is_aggregate(field)] + [DEFAULT_AXIS_OPTION]
params.setlist(
"yAxis", params.getlist("yAxis") or to_list(saved_query.get("yAxis", axis_options[0]))
)
params.setlist("field", params.getlist("field") or to_list(saved_query.get("fields")))
params.setlist(
"project",
params.getlist("project")
or (to_list(saved_query.get("project")) if saved_query.get("project") else []),
)
# Only override if key doesn't exist since we want to account for
# an intermediate state where the query could have been cleared
if "query" not in params:
params.setlist(
"query", params.getlist("query") or to_list(saved_query.get("query", ""))
)
display_mode = str(params.get("display") or saved_query.get("display", "default"))
if "daily" in display_mode:
params.setlist("interval", ["1d"])
if "top5" in display_mode:
params.setlist(
"topEvents",
params.getlist("topEvents") or to_list(saved_query.get("topEvents", f"{TOP_N}")),
)
y_axis = params.getlist("yAxis")[0]
if display_mode != "dailytop5":
display_mode = get_top5_display_mode(y_axis)
else:
# topEvents param persists in the URL in some cases, we want to discard
# it if it's not a top n display type.
params.pop("topEvents", None)
if "previous" in display_mode:
stats_period = params.getlist("statsPeriod", [DEFAULT_PERIOD])[0]
parsed_period = parse_stats_period(stats_period)
if parsed_period and parsed_period <= timedelta(days=MAX_PERIOD_DAYS_INCLUDE_PREVIOUS):
stats_period = get_double_period(stats_period)
params.setlist("statsPeriod", [stats_period])
endpoint = "events-stats/"
if "worldmap" in display_mode:
endpoint = "events-geo/"
params.setlist("field", params.getlist("yAxis"))
params.pop("sort", None)
try:
resp = client.get(
auth=ApiKey(organization=org, scope_list=["org:read"]),
user=user,
path=f"/organizations/{org_slug}/{endpoint}",
params=params,
)
except Exception as exc:
logger.error(
f"Failed to load {endpoint} for unfurl: {exc}",
exc_info=True,
)
continue
chart_data = {"seriesName": params.get("yAxis"), "stats": resp.data}
style = display_modes.get(display_mode, display_modes["default"])
try:
url = generate_chart(style, chart_data)
except RuntimeError as exc:
logger.error(
f"Failed to generate chart for discover unfurl: {exc}",
exc_info=True,
)
continue
unfurls[link.url] = SlackDiscoverMessageBuilder(
title=link.args["query"].get("name", "Dashboards query"),
chart_url=url,
).build()
analytics.record(
"integrations.slack.chart_unfurl",
organization_id=integration.organizations.all()[0].id,
user_id=user.id if user else None,
unfurls_count=len(unfurls),
)
return unfurls
