def get_attrs(self, item_list, user):
attrs = super(StreamGroupSerializer, self).get_attrs(item_list, user)
# we need to compute stats at 1d (1h resolution), and 14d
group_ids = [g.id for g in item_list]
now = timezone.now()
hourly_stats = tsdb.get_range(
model=tsdb.models.group,
keys=group_ids,
end=now,
start=now - timedelta(days=1),
rollup=3600,
)
daily_stats = tsdb.get_range(
model=tsdb.models.group,
keys=group_ids,
end=now,
start=now - timedelta(days=14),
rollup=3600 * 24,
)
for item in item_list:
attrs[item].update({
'hourly_stats': hourly_stats[item.id],
'daily_stats': daily_stats[item.id],
})
return attrs
def get(self, request, project_id):
"""
Retrieve event counts for a project
**Draft:** This endpoint may change in the future without notice.
Return a set of points representing a normalized timestamp and the
number of events seen in the period.
{method} {path}?since=1421092384.822244&until=1434052399.443363
Query ranges are limited to Sentry's configured time-series resolutions.
Parameters:
- since: a timestamp to set the start of the query
- until: a timestamp to set the end of the query
- resolution: an explicit resolution to search for (i.e. 10s)
**Note:** resolution should not be used unless you're familiar with Sentry
internals as it's restricted to pre-defined values.
"""
project = Project.objects.get_from_cache(id=project_id)
assert_perm(project, request.user, request.auth)
data = tsdb.get_range(model=tsdb.models.project, keys=[project.id], **self._parse_args(request))[project.id]
return Response(data)
def get(self, request, team_id):
team = Team.objects.get(id=team_id)
assert_perm(team, request.user, request.auth)
projects = Project.objects.get_for_user(
team=team,
user=request.user,
)
if not projects:
return Response([])
data = tsdb.get_range(
model=tsdb.models.project,
keys=[p.id for p in projects],
**self._parse_args(request)
).values()
summarized = []
for n in range(len(data[0])):
total = sum(d[n][1] for d in data)
summarized.append((data[0][n][0], total))
return Response(summarized)
def get(self, request, project):
try:
environment_id = self._get_environment_id_from_request(
request,
project.organization_id,
)
except Environment.DoesNotExist:
raise ResourceDoesNotExist
group_ids = request.GET.getlist('id')
if not group_ids:
return Response(status=204)
group_list = Group.objects.filter(project=project, id__in=group_ids)
group_ids = [g.id for g in group_list]
if not group_ids:
return Response(status=204)
data = tsdb.get_range(
model=tsdb.models.group, keys=group_ids, **self._parse_args(
request,
environment_id,
)
)
return Response({six.text_type(k): v for k, v in data.items()})
def get(self, request, group):
data = tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
**self._parse_args(request)
)[group.id]
return Response(data)
def check_project_alerts(project_id, **kwargs):
"""
Given 'when' and 'count', which should signify recent times we compare it to
historical data for this project and if over a given threshold, create an
alert.
"""
from sentry.app import tsdb
from sentry.constants import DEFAULT_ALERT_PROJECT_THRESHOLD
from sentry.models import ProjectOption, Alert
threshold, min_events = ProjectOption.objects.get_value(
project_id, 'alert:threshold', DEFAULT_ALERT_PROJECT_THRESHOLD)
if not threshold and min_events:
return
end = datetime.now().replace(tzinfo=utc) - timedelta(seconds=10)
start = end - timedelta(minutes=5)
results = [v for _, v in tsdb.get_range(
tsdb.models.project,
[project_id],
start=start,
end=end,
rollup=10,
)[project_id]]
half_intervals = int(len(results) / 2)
previous_data, current_data = results[:half_intervals], results[half_intervals:]
if not current_data:
return
current_avg = sum(current_data) / len(current_data)
# if there first few points within previous data are empty, assume that the
# project hasn't been active long enough for rates to be valid
if not any(previous_data[:3]):
return
if min_events > current_avg:
return
mean = math.mean(previous_data)
dev = math.mad(previous_data)
previous_avg = (mean + dev * 2)
pct_increase = (current_avg / previous_avg * 100) - 100
logger.info('Rate of events for project %d changed from %.2f to %2.f',
project_id, previous_avg, current_avg)
if pct_increase > threshold and current_avg > previous_avg:
Alert.maybe_alert(
project_id=project_id,
message='Rate of events increased from %.2f to %.2f' % (previous_avg, current_avg),
)
def organizations(metrics, since, until):
"""
Fetch metrics for organizations.
"""
from django.utils import timezone
from sentry.app import tsdb
from sentry.models import Organization
stdout = click.get_text_stream('stdout')
stderr = click.get_text_stream('stderr')
def aggregate(series):
return sum(value for timestamp, value in series)
metrics = OrderedDict((name, getattr(tsdb.models, name)) for name in metrics)
if not metrics:
return
if until is None:
until = timezone.now()
if since is None:
since = until - timedelta(minutes=60)
if until < since:
raise click.ClickException('invalid time range provided: {} to {}'.format(since, until))
stderr.write(
'Dumping {} from {} to {}...\n'.format(
', '.join(metrics.keys()),
since,
until,
),
)
objects = Organization.objects.all()
for chunk in chunked(objects, 100):
instances = OrderedDict((instance.pk, instance) for instance in chunk)
results = {}
for metric in metrics.values():
results[metric] = tsdb.get_range(metric, instances.keys(), since, until)
for key, instance in six.iteritems(instances):
values = []
for metric in metrics.values():
values.append(aggregate(results[metric][key]))
stdout.write(
'{} {} {}\n'.format(
instance.id,
instance.slug,
' '.join(map(six.binary_type, values)),
),
)
def get(self, request):
key = request.GET['key']
data = tsdb.get_range(
model=tsdb.models.internal,
keys=[key],
**self._parse_args(request)
)[key]
return Response(data)
def get(self, request, organization_slug):
organization = Organization.objects.get_from_cache(
slug=organization_slug,
)
assert_perm(organization, request.user, request.auth)
group = request.GET.get('group')
if not group:
keys = [organization.id]
elif group == 'project':
team_list = Team.objects.get_for_user(
organization=organization,
user=request.user,
)
project_list = []
for team in team_list:
project_list.extend(Project.objects.get_for_user(
team=team,
user=request.user,
))
keys = [p.id for p in project_list]
else:
raise ValueError('Invalid group: %s' % group)
if not keys:
return Response([])
stat = request.GET.get('stat', 'received')
if stat == 'received':
if group == 'project':
stat_model = tsdb.models.project_total_received
else:
stat_model = tsdb.models.organization_total_received
elif stat == 'rejected':
if group == 'project':
stat_model = tsdb.models.project_total_rejected
else:
stat_model = tsdb.models.organization_total_rejected
else:
raise ValueError('Invalid stat: %s' % stat)
data = tsdb.get_range(
model=stat_model,
keys=keys,
**self._parse_args(request)
)
if not group:
data = data[organization.id]
return Response(data)
def prepare_project_series(start__stop, project, rollup=60 * 60 * 24):
start, stop = start__stop
resolution, series = tsdb.get_optimal_rollup_series(start, stop, rollup)
assert resolution == rollup, 'resolution does not match requested value'
clean = functools.partial(clean_series, start, stop, rollup)
return merge_series(
reduce(
merge_series,
map(
clean,
tsdb.get_range(
tsdb.models.group,
list(
project.group_set.filter(
status=GroupStatus.RESOLVED,
resolved_at__gte=start,
resolved_at__lt=stop,
).values_list('id', flat=True),
),
start,
stop,
rollup=rollup,
).values(),
),
clean([(timestamp, 0) for timestamp in series]),
),
clean(
tsdb.get_range(
tsdb.models.project,
[project.id],
start,
stop,
rollup=rollup,
)[project.id],
),
lambda resolved, total: (
resolved,
total - resolved, # unresolved
),
)
def get(self, request, project_id):
project = Project.objects.get(
id=project_id,
)
assert_perm(project, request.user)
data = tsdb.get_range(
model=tsdb.models.project,
keys=[project.id],
**self._parse_args(request)
)[project.id]
return Response(data)
def get(self, request, project):
group_ids = request.GET.getlist('id')
if not group_ids:
return Response(status=204)
group_list = Group.objects.filter(project=project, id__in=group_ids)
group_ids = [g.id for g in group_list]
if not group_ids:
return Response(status=204)
data = tsdb.get_range(model=tsdb.models.group, keys=group_ids, **self._parse_args(request))
return Response({six.text_type(k): v for k, v in data.items()})
def get(self, request, group_id):
group = Group.objects.get(
id=group_id,
)
assert_perm(group, request.user, request.auth)
data = tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
**self._parse_args(request)
)[group.id]
return Response(data)
def get(self, request, team):
"""
Retrieve Event Counts for a Team
````````````````````````````````
.. caution::
This endpoint may change in the future without notice.
Return a set of points representing a normalized timestamp and the
number of events seen in the period.
Query ranges are limited to Sentry's configured time-series
resolutions.
:pparam string organization_slug: the slug of the organization.
:pparam string team_slug: the slug of the team.
:qparam string stat: the name of the stat to query (``"received"``,
``"rejected"``)
:qparam timestamp since: a timestamp to set the start of the query
in seconds since UNIX epoch.
:qparam timestamp until: a timestamp to set the end of the query
in seconds since UNIX epoch.
:qparam string resolution: an explicit resolution to search
for (eg: ``10s``). This should not be
used unless you are familiar with Sentry's
internals as it's restricted to pre-defined
values.
:auth: required
"""
projects = Project.objects.get_for_user(
team=team,
user=request.user,
)
if not projects:
return Response([])
data = list(tsdb.get_range(
model=tsdb.models.project,
keys=[p.id for p in projects],
**self._parse_args(request)
).values())
summarized = []
for n in range(len(data[0])):
total = sum(d[n][1] for d in data)
summarized.append((data[0][n][0], total))
return Response(summarized)
def with_event_counts(project_list):
from sentry.app import tsdb
end = timezone.now()
start = end - datetime.timedelta(days=1)
tsdb_results = tsdb.get_range(
model=tsdb.models.project,
keys=[p.id for p in project_list],
start=start,
end=end,
)
for project in project_list:
yield project, sum(t[1] for t in tsdb_results[project.id])
def get(self, request, project):
"""
Retrieve Event Counts for a Project
```````````````````````````````````
.. caution::
This endpoint may change in the future without notice.
Return a set of points representing a normalized timestamp and the
number of events seen in the period.
Query ranges are limited to Sentry's configured time-series
resolutions.
:pparam string organization_slug: the slug of the organization.
:pparam string project_slug: the slug of the project.
:qparam string stat: the name of the stat to query (``"received"``,
``"rejected"``, ``"blacklisted"``, ``generated``)
:qparam timestamp since: a timestamp to set the start of the query
in seconds since UNIX epoch.
:qparam timestamp until: a timestamp to set the end of the query
in seconds since UNIX epoch.
:qparam string resolution: an explicit resolution to search
for (eg: ``10s``). This should not be
used unless you are familiar with Sentry's
internals as it's restricted to pre-defined
values.
:auth: required
"""
stat = request.GET.get('stat', 'received')
if stat == 'received':
stat_model = tsdb.models.project_total_received
elif stat == 'rejected':
stat_model = tsdb.models.project_total_rejected
elif stat == 'blacklisted':
stat_model = tsdb.models.project_total_blacklisted
elif stat == 'generated':
stat_model = tsdb.models.project
else:
raise ValueError('Invalid stat: %s' % stat)
data = tsdb.get_range(
model=stat_model,
keys=[project.id],
**self._parse_args(request)
)[project.id]
return Response(data)
def get_stats(request, organization, team=None, project=None):
minutes = int(request.REQUEST.get('minutes', 15))
if not team and project:
project_list = [project]
elif team:
project_list = Project.objects.get_for_user(team=team, user=request.user)
else:
return HttpResponse(status=400)
cutoff = timedelta(minutes=minutes)
end = timezone.now()
start = end - cutoff
# TODO(dcramer): this is used in an unreleased feature. reimplement it using
# new API and tsdb
results = tsdb.get_range(
model=tsdb.models.project,
keys=[p.id for p in project_list],
start=start,
end=end,
)
num_events = 0
for project, points in results.iteritems():
num_events += sum(p[1] for p in points)
# XXX: This is too slow if large amounts of groups are resolved
# TODO(dcramer); move this into tsdb
num_resolved = Group.objects.filter(
project__in=project_list,
status=GroupStatus.RESOLVED,
resolved_at__gte=start,
).aggregate(t=Sum('times_seen'))['t'] or 0
data = {
'events': num_events,
'resolved': num_resolved,
}
response = HttpResponse(json.dumps(data))
response['Content-Type'] = 'application/json'
return response
def prepare_project_calendar_series(interval, project):
start, stop = get_calendar_query_range(interval, 3)
rollup = 60 * 60 * 24
series = tsdb.get_range(
tsdb.models.project,
[project.id],
start,
stop,
rollup=rollup,
)[project.id]
return clean_calendar_data(
project,
series,
start,
stop,
rollup,
)
def prepare_project_calendar_series(interval, project):
start, stop = get_calendar_query_range(interval, 3)
rollup = 60 * 60 * 24
series = tsdb.get_range(
tsdb.models.project,
[project.id],
start,
stop,
rollup=rollup,
)[project.id]
return clean_calendar_data(
project,
series,
start,
stop,
rollup,
)
def get(self, request, team):
"""
Retrieve event counts for a team
**Draft:** This endpoint may change in the future without notice.
Return a set of points representing a normalized timestamp and the
number of events seen in the period.
{method} {path}?since=1421092384.822244&until=1434052399.443363
Query ranges are limited to Sentry's configured time-series resolutions.
Parameters:
- since: a timestamp to set the start of the query
- until: a timestamp to set the end of the query
- resolution: an explicit resolution to search for (i.e. 10s)
**Note:** resolution should not be used unless you're familiar with Sentry
internals as it's restricted to pre-defined values.
"""
projects = Project.objects.get_for_user(
team=team,
user=request.user,
)
if not projects:
return Response([])
data = tsdb.get_range(
model=tsdb.models.project,
keys=[p.id for p in projects],
**self._parse_args(request)
).values()
summarized = []
for n in range(len(data[0])):
total = sum(d[n][1] for d in data)
summarized.append((data[0][n][0], total))
return Response(summarized)
def get_stats(request, team=None, project=None):
minutes = int(request.REQUEST.get('minutes', 15))
if not team and project:
project_list = [project]
else:
project_list = Project.objects.get_for_user(request.user, team=team)
cutoff = datetime.timedelta(minutes=minutes)
end = timezone.now()
start = end - cutoff
# TODO(dcramer): this is used in an unreleased feature. reimplement it using
# new API and tsdb
results = tsdb.get_range(
model=tsdb.models.project,
keys=[p.id for p in project_list],
start=start,
end=end,
)
num_events = 0
for project, points in results.iteritems():
num_events += sum(p[1] for p in points)
# XXX: This is too slow if large amounts of groups are resolved
# TODO(dcramer); move this into tsdb
num_resolved = Group.objects.filter(
project__in=project_list,
status=STATUS_RESOLVED,
resolved_at__gte=start,
).aggregate(t=Sum('times_seen'))['t'] or 0
data = {
'events': num_events,
'resolved': num_resolved,
}
response = HttpResponse(json.dumps(data))
response['Content-Type'] = 'application/json'
return response
def get_attrs(self, item_list, user):
attrs = super(StreamGroupSerializer, self).get_attrs(item_list, user)
# we need to compute stats at 1d (1h resolution), and 14d
group_ids = [g.id for g in item_list]
if self.stats_period:
days = 14 if self.stats_period == '14d' else 1
now = timezone.now()
stats = tsdb.rollup(tsdb.get_range(
model=tsdb.models.group,
keys=group_ids,
end=now,
start=now - timedelta(days=days),
), 3600 * days)
for item in item_list:
attrs[item].update({
'stats': stats[item.id],
})
return attrs
def get(self, request, team_id):
team = Team.objects.get(id=team_id)
assert_perm(team, request.user, request.auth)
projects = Project.objects.get_for_user(request.user, team=team)
if not projects:
return Response([])
data = tsdb.get_range(model=tsdb.models.project,
keys=[p.id for p in projects],
**self._parse_args(request)).values()
summarized = []
for n in range(len(data[0])):
total = sum(d[n][1] for d in data)
summarized.append((data[0][n][0], total))
return Response(summarized)
def get_attrs(self, item_list, user):
attrs = super(StreamGroupSerializer, self).get_attrs(item_list, user)
# we need to compute stats at 1d (1h resolution), and 14d
group_ids = [g.id for g in item_list]
if self.stats_period:
days = 14 if self.stats_period == '14d' else 1
now = timezone.now()
stats = tsdb.rollup(tsdb.get_range(
model=tsdb.models.group,
keys=group_ids,
end=now,
start=now - timedelta(days=days),
), 3600 * days)
for item in item_list:
attrs[item].update({
'stats': stats[item.id],
})
return attrs
def get(self, request, team):
"""
Retrieve event counts for a team
**Draft:** This endpoint may change in the future without notice.
Return a set of points representing a normalized timestamp and the
number of events seen in the period.
{method} {path}?since=1421092384.822244&until=1434052399.443363
Query ranges are limited to Sentry's configured time-series resolutions.
Parameters:
- since: a timestamp to set the start of the query
- until: a timestamp to set the end of the query
- resolution: an explicit resolution to search for (i.e. 10s)
**Note:** resolution should not be used unless you're familiar with Sentry
internals as it's restricted to pre-defined values.
"""
projects = Project.objects.get_for_user(
team=team,
user=request.user,
)
if not projects:
return Response([])
data = tsdb.get_range(model=tsdb.models.project,
keys=[p.id for p in projects],
**self._parse_args(request)).values()
summarized = []
for n in range(len(data[0])):
total = sum(d[n][1] for d in data)
summarized.append((data[0][n][0], total))
return Response(summarized)
def get(self, request, project):
try:
environment_id = self._get_environment_id_from_request(
request, project.organization_id)
except Environment.DoesNotExist:
raise ResourceDoesNotExist
group_ids = request.GET.getlist("id")
if not group_ids:
return Response(status=204)
group_list = Group.objects.filter(project=project, id__in=group_ids)
group_ids = [g.id for g in group_list]
if not group_ids:
return Response(status=204)
data = tsdb.get_range(model=tsdb.models.group,
keys=group_ids,
**self._parse_args(request, environment_id))
return Response({str(k): v for k, v in data.items()})
def get_attrs(self, item_list, user):
attrs = super(StreamGroupSerializer, self).get_attrs(item_list, user)
# we need to compute stats at 1d (1h resolution), and 14d
group_ids = [g.id for g in item_list]
if self.stats_period:
segments, interval = self.STATS_PERIOD_CHOICES[self.stats_period]
now = timezone.now()
stats = tsdb.get_range(
model=tsdb.models.group,
keys=group_ids,
end=now,
start=now - ((segments - 1) * interval),
rollup=int(interval.total_seconds()),
)
for item in item_list:
attrs[item].update({
'stats': stats[item.id],
})
return attrs
def get_attrs(self, item_list, user):
attrs = super(StreamGroupSerializer, self).get_attrs(item_list, user)
if self.stats_period:
# we need to compute stats at 1d (1h resolution), and 14d
group_ids = [g.id for g in item_list]
segments, interval = self.STATS_PERIOD_CHOICES[self.stats_period]
now = timezone.now()
stats = tsdb.get_range(
model=tsdb.models.group,
keys=group_ids,
end=now,
start=now - ((segments - 1) * interval),
rollup=int(interval.total_seconds()),
)
for item in item_list:
attrs[item].update({
'stats': stats[item.id],
})
return attrs
def prepare_project_series(start__stop, project, rollup=60 * 60 * 24):
start, stop = start__stop
resolution, series = tsdb.get_optimal_rollup_series(start, stop, rollup)
assert resolution == rollup, "resolution does not match requested value"
clean = partial(clean_series, start, stop, rollup)
issue_ids = project.group_set.filter(
status=GroupStatus.RESOLVED, resolved_at__gte=start, resolved_at__lt=stop
).values_list("id", flat=True)
tsdb_range = _query_tsdb_chunked(tsdb.get_range, issue_ids, start, stop, rollup)
return merge_series(
reduce(
merge_series,
map(clean, tsdb_range.values()),
clean([(timestamp, 0) for timestamp in series]),
),
clean(
tsdb.get_range(tsdb.models.project, [project.id], start, stop, rollup=rollup)[
project.id
]
),
lambda resolved, total: (resolved, total - resolved), # unresolved
)
def get(self, request, group):
"""
Retrieve an Issue
`````````````````
Return details on an individual issue. This returns the basic stats for
the issue (title, last seen, first seen), some overall numbers (number
of comments, user reports) as well as the summarized event data.
:pparam string issue_id: the ID of the issue to retrieve.
:auth: required
"""
# TODO(dcramer): handle unauthenticated/public response
data = serialize(group, request.user)
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=100)
seen_by = self._get_seen_by(request, group)
# find first seen release
if group.first_release is None:
try:
first_release = GroupTagValue.objects.filter(
group=group,
key__in=('sentry:release', 'release'),
).order_by('first_seen')[0]
except IndexError:
first_release = None
else:
first_release = first_release.value
else:
first_release = group.first_release.version
if first_release is not None:
# find last seen release
try:
last_release = GroupTagValue.objects.filter(
group=group,
key__in=('sentry:release', 'release'),
).order_by('-last_seen')[0]
except IndexError:
last_release = None
else:
last_release = last_release.value
else:
last_release = None
action_list = self._get_actions(request, group)
now = timezone.now()
hourly_stats = tsdb.rollup(
tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=1),
), 3600)[group.id]
daily_stats = tsdb.rollup(
tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=30),
), 3600 * 24)[group.id]
if first_release:
first_release = self._get_release_info(request, group,
first_release)
if last_release:
last_release = self._get_release_info(request, group, last_release)
tags = list(GroupTagKey.objects.filter(group=group, )[:100])
data.update({
'firstRelease':
first_release,
'lastRelease':
last_release,
'activity':
serialize(activity, request.user),
'seenBy':
seen_by,
'pluginActions':
action_list,
'userReportCount':
UserReport.objects.filter(group=group).count(),
'tags':
sorted(serialize(tags, request.user), key=lambda x: x['name']),
'stats': {
'24h': hourly_stats,
'30d': daily_stats,
}
})
return Response(data)
def get(self, request, organization):
"""
Retrieve Event Counts for an Organization
`````````````````````````````````````````
.. caution::
This endpoint may change in the future without notice.
Return a set of points representing a normalized timestamp and the
number of events seen in the period.
:pparam string organization_slug: the slug of the organization for
which the stats should be
retrieved.
:qparam string stat: the name of the stat to query (``"received"``,
``"rejected"``, ``"blacklisted"``)
:qparam timestamp since: a timestamp to set the start of the query
in seconds since UNIX epoch.
:qparam timestamp until: a timestamp to set the end of the query
in seconds since UNIX epoch.
:qparam string resolution: an explicit resolution to search
for (eg: ``10s``). This should not be
used unless you are familiar with Sentry's
internals as it's restricted to pre-defined
values.
:auth: required
"""
group = request.GET.get('group')
if not group:
keys = [organization.id]
elif group == 'project':
team_list = Team.objects.get_for_user(
organization=organization,
user=request.user,
)
project_list = []
for team in team_list:
project_list.extend(Project.objects.get_for_user(
team=team,
user=request.user,
))
keys = [p.id for p in project_list]
else:
raise ValueError('Invalid group: %s' % group)
if not keys:
return Response([])
stat = request.GET.get('stat', 'received')
if stat == 'received':
if group == 'project':
stat_model = tsdb.models.project_total_received
else:
stat_model = tsdb.models.organization_total_received
elif stat == 'rejected':
if group == 'project':
stat_model = tsdb.models.project_total_rejected
else:
stat_model = tsdb.models.organization_total_rejected
elif stat == 'blacklisted':
if group == 'project':
stat_model = tsdb.models.project_total_blacklisted
else:
stat_model = tsdb.models.organization_total_blacklisted
else:
raise ValueError('Invalid stat: %s' % stat)
data = tsdb.get_range(
model=stat_model,
keys=keys,
**self._parse_args(request)
)
if not group:
data = data[organization.id]
return Response(data)
def get_attrs(self, item_list, user):
attach_foreignkey(item_list, Group.project, ['team'])
if user.is_authenticated() and item_list:
bookmarks = set(GroupBookmark.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', flat=True))
seen_groups = dict(GroupSeen.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', 'last_seen'))
else:
bookmarks = set()
seen_groups = {}
tag_counts = defaultdict(dict)
tag_results = GroupTagKey.objects.filter(
group__in=item_list,
).values_list('key', 'group', 'values_seen')
for key, group_id, values_seen in tag_results:
tag_counts[key][group_id] = values_seen
# we need to compute stats at 1d (1h resolution), and 14d/30d (1 day res)
group_ids = [g.id for g in item_list]
now = timezone.now()
hourly_stats = tsdb.get_range(
model=tsdb.models.group,
keys=group_ids,
end=now,
start=now - timedelta(days=1),
rollup=3600,
)
daily_stats = tsdb.get_range(
model=tsdb.models.group,
keys=group_ids,
end=now,
start=now - timedelta(days=30),
rollup=3600 * 24,
)
result = {}
for item in item_list:
active_date = item.active_at or item.last_seen
tags = {}
for key in tag_counts.iterkeys():
label = TAG_LABELS.get(key, key.replace('_', ' ')).lower() + 's'
try:
value = tag_counts[key].get(item.id, 0)
except KeyError:
value = 0
tags[key] = {
'label': label,
'count': value,
}
result[item] = {
'is_bookmarked': item.id in bookmarks,
'has_seen': seen_groups.get(item.id, active_date) > active_date,
'tags': tags,
'hourly_stats': hourly_stats[item.id],
'daily_stats': daily_stats[item.id],
}
return result
def test_unmerge(self, mock_eventstream):
eventstream_state = object()
mock_eventstream.start_unmerge = Mock(return_value=eventstream_state)
def shift(i):
return timedelta(seconds=1 << i)
now = timezone.now().replace(microsecond=0) - shift(16)
project = self.create_project()
source = self.create_group(project)
sequence = itertools.count(0)
tag_values = itertools.cycle(['red', 'green', 'blue'])
user_values = itertools.cycle([
{
'id': 1
},
{
'id': 2
},
])
for environment in ('production', ''):
EnvironmentProject.objects.create(
environment=Environment.objects.create(
organization_id=project.organization_id,
name=environment,
),
project=project,
)
def create_message_event(template, parameters, environment, release):
i = next(sequence)
event_id = uuid.UUID(fields=(
i,
0x0,
0x1000,
0x80,
0x80,
0x808080808080,
), ).hex
tags = [['color', next(tag_values)]]
if environment:
tags.append(['environment', environment])
if release:
tags.append(['sentry:release', release])
event = Event.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
message='%s' % (id, ),
datetime=now + shift(i),
data={
'environment': environment,
'type': 'default',
'metadata': {
'title': template % parameters,
},
'logentry': {
'message': template,
'params': parameters,
'formatted': template % parameters,
},
'user': next(user_values),
'tags': tags,
},
)
with self.tasks():
Group.objects.add_tags(
source,
Environment.objects.get(
organization_id=project.organization_id,
name=environment),
tags=event.tags,
)
EventMapping.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
date_added=event.datetime,
)
UserReport.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
name='Log Hat',
email='*****@*****.**',
comments='Quack',
)
if release:
Release.get_or_create(
project=project,
version=event.get_tag('sentry:release'),
date_added=event.datetime,
)
features.record([event])
return event
events = OrderedDict()
for event in (create_message_event('This is message #%s.',
i,
environment='production',
release='version')
for i in xrange(10)):
events.setdefault(get_fingerprint(event), []).append(event)
for event in (create_message_event('This is message #%s!',
i,
environment='production',
release='version')
for i in xrange(10, 16)):
events.setdefault(get_fingerprint(event), []).append(event)
event = create_message_event('This is message #%s!',
17,
environment='',
release=None)
events.setdefault(get_fingerprint(event), []).append(event)
assert len(events) == 2
assert sum(map(len, events.values())) == 17
# XXX: This is super contrived considering that it doesn't actually go
# through the event pipeline, but them's the breaks, eh?
for fingerprint in events.keys():
GroupHash.objects.create(
project=project,
group=source,
hash=fingerprint,
)
production_environment = Environment.objects.get(
organization_id=project.organization_id, name='production')
assert set([
(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(
source.project_id, source.id, [production_environment.id])
]) == set([(u'color', 3), (u'environment', 1), (u'sentry:release', 1)])
if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'):
assert set([
(gtv.key, gtv.value, gtv.times_seen,
Environment.objects.get(pk=gtv._key.environment_id).name)
for gtv in GroupTagValue.objects.filter(
project_id=source.project_id,
group_id=source.id,
).exclude(_key__environment_id=0)
]) == set([
('color', 'red', 6, 'production'),
('sentry:release', 'version', 16, 'production'),
('color', 'blue', 5, 'production'),
('color', 'green', 5, 'production'),
('environment', 'production', 16, 'production'),
('color', 'green', 1, ''),
])
else:
assert set([(gtv.key, gtv.value, gtv.times_seen)
for gtv in GroupTagValue.objects.filter(
project_id=source.project_id,
group_id=source.id,
)]) == set([
(u'color', u'red', 6),
(u'color', u'green', 6),
(u'color', u'blue', 5),
(u'environment', u'production', 16),
(u'sentry:release', u'version', 16),
])
assert features.compare(source) == [
(source.id, {
'exception:message:character-shingles': None,
'exception:stacktrace:application-chunks': None,
'exception:stacktrace:pairs': None,
'message:message:character-shingles': 1.0
}),
]
with self.tasks():
unmerge.delay(
source.project_id,
source.id,
None,
[events.keys()[1]],
None,
batch_size=5,
)
assert list(
Group.objects.filter(id=source.id).values_list(
'times_seen',
'first_seen',
'last_seen',
)) == [(
10,
now + shift(0),
now + shift(9),
)]
source_activity = Activity.objects.get(
group_id=source.id,
type=Activity.UNMERGE_SOURCE,
)
destination = Group.objects.get(
id=source_activity.data['destination_id'], )
mock_eventstream.start_unmerge.assert_called_once_with(
source.project_id, [events.keys()[1]], source.id, destination.id)
mock_eventstream.end_unmerge.assert_called_once_with(eventstream_state)
assert list(
Group.objects.filter(id=destination.id).values_list(
'times_seen',
'first_seen',
'last_seen',
)) == [(
7,
now + shift(10),
now + shift(16),
)]
assert source_activity.data == {
'destination_id': destination.id,
'fingerprints': [events.keys()[1]],
}
assert source.id != destination.id
assert source.project == destination.project
assert Activity.objects.get(
group_id=destination.id,
type=Activity.UNMERGE_DESTINATION,
).data == {
'source_id': source.id,
'fingerprints': [events.keys()[1]],
}
source_event_event_ids = map(
lambda event: event.event_id,
events.values()[0],
)
assert source.event_set.count() == 10
assert set(
EventMapping.objects.filter(group_id=source.id, ).values_list(
'event_id', flat=True)) == set(source_event_event_ids)
assert set(
UserReport.objects.filter(group_id=source.id, ).values_list(
'event_id', flat=True)) == set(source_event_event_ids)
assert set(
GroupHash.objects.filter(group_id=source.id, ).values_list(
'hash', flat=True)) == set([events.keys()[0]])
assert set(
GroupRelease.objects.filter(group_id=source.id, ).values_list(
'environment', 'first_seen', 'last_seen')) == set([
(
u'production',
now + shift(0),
now + shift(9),
),
])
assert set([
(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(
source.project_id, source.id, [production_environment.id])
]) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
])
if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'):
env_filter = {'_key__environment_id': production_environment.id}
else:
env_filter = {}
assert set([
(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen)
for gtv in GroupTagValue.objects.filter(
project_id=source.project_id, group_id=source.id, **env_filter)
]) == set([
(
u'color',
u'red',
4,
now + shift(0),
now + shift(9),
),
(
u'color',
u'green',
3,
now + shift(1),
now + shift(7),
),
(
u'color',
u'blue',
3,
now + shift(2),
now + shift(8),
),
(
u'environment',
u'production',
10,
now + shift(0),
now + shift(9),
),
(
u'sentry:release',
u'version',
10,
now + shift(0),
now + shift(9),
),
])
destination_event_event_ids = map(
lambda event: event.event_id,
events.values()[1],
)
assert destination.event_set.count() == 7
assert set(
EventMapping.objects.filter(group_id=destination.id, ).values_list(
'event_id', flat=True)) == set(destination_event_event_ids)
assert set(
UserReport.objects.filter(group_id=destination.id, ).values_list(
'event_id', flat=True)) == set(destination_event_event_ids)
assert set(
GroupHash.objects.filter(group_id=destination.id, ).values_list(
'hash', flat=True)) == set([events.keys()[1]])
assert set(
GroupRelease.objects.filter(group_id=destination.id, ).values_list(
'environment', 'first_seen', 'last_seen')) == set([
(
u'production',
now + shift(10),
now + shift(15),
),
])
assert set([
(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(
source.project_id, source.id, [production_environment.id])
]) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
])
if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'):
assert set([(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen,
gtv.last_seen)
for gtv in GroupTagValue.objects.filter(
project_id=destination.project_id,
group_id=destination.id,
**env_filter)]) == set([
(
u'color',
u'red',
2,
now + shift(12),
now + shift(15),
),
(
u'color',
u'green',
2,
now + shift(10),
now + shift(13),
),
(
u'color',
u'blue',
2,
now + shift(11),
now + shift(14),
),
(
u'environment',
u'production',
6,
now + shift(10),
now + shift(15),
),
(
u'sentry:release',
u'version',
6,
now + shift(10),
now + shift(15),
),
])
else:
assert set([(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen,
gtv.last_seen)
for gtv in GroupTagValue.objects.filter(
project_id=destination.project_id,
group_id=destination.id,
**env_filter)]) == set([
(
u'color',
u'red',
2,
now + shift(12),
now + shift(15),
),
(
u'color',
u'green',
3,
now + shift(10),
now + shift(16),
),
(
u'color',
u'blue',
2,
now + shift(11),
now + shift(14),
),
(
u'environment',
u'production',
6,
now + shift(10),
now + shift(15),
),
(
u'sentry:release',
u'version',
6,
now + shift(10),
now + shift(15),
),
])
rollup_duration = 3600
time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(15),
rollup_duration,
)
environment_time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(15),
rollup_duration,
environment_ids=[production_environment.id],
)
def get_expected_series_values(rollup, events, function=None):
if function is None:
def function(aggregate, event):
return (aggregate if aggregate is not None else 0) + 1
expected = {}
for event in events:
k = float((to_timestamp(event.datetime) // rollup_duration) *
rollup_duration)
expected[k] = function(expected.get(k), event)
return expected
def assert_series_contains(expected, actual, default=0):
actual = dict(actual)
for key, value in expected.items():
assert actual.get(key, 0) == value
for key in set(actual.keys()) - set(expected.keys()):
assert actual.get(key, 0) == default
for series in [time_series, environment_time_series]:
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[0]),
series[source.id],
0,
)
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[1][:-1]),
series[destination.id],
0,
)
time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
environment_time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
environment_id=production_environment.id,
)
def collect_by_user_tag(aggregate, event):
aggregate = aggregate if aggregate is not None else set()
aggregate.add(
get_event_user_from_interface(
event.data['user'], ).tag_value, )
return aggregate
for series in [time_series, environment_time_series]:
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[0],
collect_by_user_tag,
).items()
},
series[source.id],
)
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[1],
collect_by_user_tag,
).items()
},
time_series[destination.id],
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_releases_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_release(group, aggregate, event):
aggregate = aggregate if aggregate is not None else {}
release = event.get_tag('sentry:release')
if not release:
return aggregate
release = GroupRelease.objects.get(
group_id=group.id,
environment=event.data['environment'],
release_id=Release.objects.get(
organization_id=project.organization_id,
version=release,
).id,
).id
aggregate[release] = aggregate.get(release, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
functools.partial(
collect_by_release,
source,
),
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
functools.partial(
collect_by_release,
destination,
),
),
time_series[destination.id],
{},
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_environments_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_environment(aggregate, event):
aggregate = aggregate if aggregate is not None else {}
environment = Environment.objects.get(
organization_id=project.organization_id,
name=event.data['environment'],
).id
aggregate[environment] = aggregate.get(environment, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
collect_by_environment,
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
collect_by_environment,
),
time_series[destination.id],
{},
)
source_similar_items = features.compare(source)
assert source_similar_items[0] == (source.id, {
'exception:message:character-shingles':
None,
'exception:stacktrace:application-chunks':
None,
'exception:stacktrace:pairs':
None,
'message:message:character-shingles':
1.0,
})
assert source_similar_items[1][0] == destination.id
assert source_similar_items[1][1][
'message:message:character-shingles'] < 1.0
destination_similar_items = features.compare(destination)
assert destination_similar_items[0] == (destination.id, {
'exception:message:character-shingles':
None,
'exception:stacktrace:application-chunks':
None,
'exception:stacktrace:pairs':
None,
'message:message:character-shingles':
1.0
})
assert destination_similar_items[1][0] == source.id
assert destination_similar_items[1][1][
'message:message:character-shingles'] < 1.0
def check_project_alerts(project_id, **kwargs):
"""
Given 'when' and 'count', which should signify recent times we compare it to
historical data for this project and if over a given threshold, create an
alert.
"""
from sentry.app import tsdb
from sentry.constants import DEFAULT_ALERT_PROJECT_THRESHOLD
from sentry.models import ProjectOption, Alert
threshold, min_events = ProjectOption.objects.get_value(
project_id, 'alert:threshold', DEFAULT_ALERT_PROJECT_THRESHOLD)
if not threshold and min_events:
return
end = datetime.now().replace(tzinfo=utc) - timedelta(seconds=10)
start = end - timedelta(minutes=5)
results = [
v for _, v in tsdb.get_range(
tsdb.models.project,
[project_id],
start=start,
end=end,
rollup=10,
)[project_id]
]
half_intervals = int(len(results) / 2)
previous_data, current_data = results[:half_intervals], results[
half_intervals:]
if not current_data:
return
current_avg = sum(current_data) / len(current_data)
# if there first few points within previous data are empty, assume that the
# project hasn't been active long enough for rates to be valid
if not any(previous_data[:3]):
return
if min_events > current_avg:
return
mean = math.mean(previous_data)
dev = math.mad(previous_data)
previous_avg = (mean + dev * 2)
pct_increase = (current_avg / previous_avg * 100) - 100
logger.info('Rate of events for project %d changed from %.2f to %2.f',
project_id, previous_avg, current_avg)
if pct_increase > threshold and current_avg > previous_avg:
Alert.maybe_alert(
project_id=project_id,
message='Rate of events increased from %.2f to %.2f' %
(previous_avg, current_avg),
)
def get(self, request, organization):
"""
Retrieve event counts for an organization
**Draft:** This endpoint may change in the future without notice.
Return a set of points representing a normalized timestamp and the
number of events seen in the period.
{method} {path}?since=1421092384.822244&until=1434052399.443363
Query ranges are limited to Sentry's configured time-series resolutions.
Parameters:
- since: a timestamp to set the start of the query
- until: a timestamp to set the end of the query
- resolution: an explicit resolution to search for (i.e. 10s)
- stat: the name of the stat to query (received, rejected)
**Note:** resolution should not be used unless you're familiar with Sentry
internals as it's restricted to pre-defined values.
"""
group = request.GET.get('group')
if not group:
keys = [organization.id]
elif group == 'project':
team_list = Team.objects.get_for_user(
organization=organization,
user=request.user,
)
project_list = []
for team in team_list:
project_list.extend(
Project.objects.get_for_user(
team=team,
user=request.user,
))
keys = [p.id for p in project_list]
else:
raise ValueError('Invalid group: %s' % group)
if not keys:
return Response([])
stat = request.GET.get('stat', 'received')
if stat == 'received':
if group == 'project':
stat_model = tsdb.models.project_total_received
else:
stat_model = tsdb.models.organization_total_received
elif stat == 'rejected':
if group == 'project':
stat_model = tsdb.models.project_total_rejected
else:
stat_model = tsdb.models.organization_total_rejected
else:
raise ValueError('Invalid stat: %s' % stat)
data = tsdb.get_range(model=stat_model,
keys=keys,
**self._parse_args(request))
if not group:
data = data[organization.id]
return Response(data)
def get_attrs(self, item_list, user):
attach_foreignkey(item_list, Group.project, ['team'])
if user.is_authenticated() and item_list:
bookmarks = set(GroupBookmark.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', flat=True))
seen_groups = dict(GroupSeen.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', 'last_seen'))
else:
bookmarks = set()
seen_groups = {}
tag_counts = defaultdict(dict)
tag_results = GroupTagKey.objects.filter(
group__in=item_list,
).values_list('key', 'group', 'values_seen')
for key, group_id, values_seen in tag_results:
tag_counts[key][group_id] = values_seen
# we need to compute stats at 1d (1h resolution), and 14d/30d (1 day res)
group_ids = [g.id for g in item_list]
now = timezone.now()
hourly_stats = tsdb.get_range(
model=tsdb.models.group,
keys=group_ids,
end=now,
start=now - timedelta(days=1),
rollup=3600,
)
daily_stats = tsdb.get_range(
model=tsdb.models.group,
keys=group_ids,
end=now,
start=now - timedelta(days=30),
rollup=3600 * 24,
)
result = {}
for item in item_list:
active_date = item.active_at or item.last_seen
tags = {}
for key in tag_counts.iterkeys():
label = TAG_LABELS.get(key, key.replace('_', ' ')).lower()
try:
value = tag_counts[key].get(item.id, 0)
except KeyError:
value = 0
tags[key] = {
'label': label,
'count': value,
}
result[item] = {
'is_bookmarked': item.id in bookmarks,
'has_seen': seen_groups.get(item.id, active_date) > active_date,
'tags': tags,
'hourly_stats': hourly_stats[item.id],
'daily_stats': daily_stats[item.id],
}
return result
def get(self, request, group):
"""
Retrieve an aggregate
Return details on an individual aggregate.
{method} {path}
"""
# TODO(dcramer): handle unauthenticated/public response
data = serialize(group, request.user)
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=7)
seen_by = self._get_seen_by(request, group)
# find first seen release
if group.first_release is None:
try:
first_release = GroupTagValue.objects.filter(
group=group,
key='sentry:release',
).order_by('first_seen')[0]
except IndexError:
first_release = None
else:
first_release = {
'version': first_release.value,
# TODO(dcramer): this should look it up in Release
'dateCreated': first_release.first_seen,
}
else:
first_release = group.first_release.version
if first_release is not None:
# find last seen release
try:
last_release = GroupTagValue.objects.filter(
group=group,
key='sentry:release',
).order_by('-last_seen')[0]
except IndexError:
last_release = None
else:
last_release = {
'version': last_release.value,
# TODO(dcramer): this should look it up in Release
'dateCreated': last_release.first_seen,
}
else:
last_release = None
action_list = self._get_actions(request, group)
now = timezone.now()
hourly_stats = tsdb.rollup(tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=1),
), 3600)[group.id]
daily_stats = tsdb.rollup(tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=30),
), 3600 * 24)[group.id]
data.update({
'firstRelease': first_release,
'lastRelease': last_release,
'activity': serialize(activity, request.user),
'seenBy': serialize(seen_by, request.user),
'pluginActions': action_list,
'stats': {
'24h': hourly_stats,
'30d': daily_stats,
}
})
return Response(data)
def get(self, request, organization):
"""
Retrieve event counts for an organization
**Draft:** This endpoint may change in the future without notice.
Return a set of points representing a normalized timestamp and the
number of events seen in the period.
{method} {path}?since=1421092384.822244&until=1434052399.443363
Query ranges are limited to Sentry's configured time-series resolutions.
Parameters:
- since: a timestamp to set the start of the query
- until: a timestamp to set the end of the query
- resolution: an explicit resolution to search for (i.e. 10s)
- stat: the name of the stat to query (received, rejected)
**Note:** resolution should not be used unless you're familiar with Sentry
internals as it's restricted to pre-defined values.
"""
group = request.GET.get('group')
if not group:
keys = [organization.id]
elif group == 'project':
team_list = Team.objects.get_for_user(
organization=organization,
user=request.user,
)
project_list = []
for team in team_list:
project_list.extend(
Project.objects.get_for_user(
team=team,
user=request.user,
))
keys = [p.id for p in project_list]
else:
raise ValueError('Invalid group: %s' % group)
if not keys:
return Response([])
stat = request.GET.get('stat', 'received')
if stat == 'received':
if group == 'project':
stat_model = tsdb.models.project_total_received
else:
stat_model = tsdb.models.organization_total_received
elif stat == 'rejected':
if group == 'project':
stat_model = tsdb.models.project_total_rejected
else:
stat_model = tsdb.models.organization_total_rejected
else:
raise ValueError('Invalid stat: %s' % stat)
data = tsdb.get_range(
model=stat_model, keys=keys, **self._parse_args(request))
if not group:
data = data[organization.id]
return Response(data)
def prepare_project_series((start, stop), project, rollup=60 * 60 * 24):
resolution, series = tsdb.get_optimal_rollup_series(start, stop, rollup)
assert resolution == rollup, 'resolution does not match requested value'
clean = functools.partial(clean_series, start, stop, rollup)
return merge_series(
reduce(
merge_series,
map(
clean,
tsdb.get_range(
tsdb.models.group,
project.group_set.filter(
status=GroupStatus.RESOLVED,
resolved_at__gte=start,
resolved_at__lt=stop,
).values_list('id', flat=True),
start,
stop,
rollup=rollup,
).values(),
),
clean([(timestamp, 0) for timestamp in series]),
),
clean(
tsdb.get_range(
tsdb.models.project,
[project.id],
start,
stop,
rollup=rollup,
def get(self, request, group):
data = tsdb.get_range(model=tsdb.models.group,
keys=[group.id],
**self._parse_args(request))[group.id]
return Response(data)
def test_unmerge(self):
now = datetime(2017, 5, 3, 6, 6, 6, tzinfo=pytz.utc)
def shift(i):
return timedelta(seconds=1 << i)
project = self.create_project()
source = self.create_group(project)
sequence = itertools.count(0)
tag_values = itertools.cycle(['red', 'green', 'blue'])
user_values = itertools.cycle([
{
'id': 1
},
{
'id': 2
},
])
EnvironmentProject.objects.create(
environment=Environment.objects.create(
organization_id=project.organization_id,
name='production',
),
project=project,
)
def create_message_event(template, parameters):
i = next(sequence)
event_id = uuid.UUID(
fields=(i, 0x0, 0x1000, 0x80, 0x80, 0x808080808080, ),
).hex
event = Event.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
message='%s' % (id, ),
datetime=now + shift(i),
data={
'environment':
'production',
'type':
'default',
'metadata': {
'title': template % parameters,
},
'sentry.interfaces.Message': {
'message': template,
'params': parameters,
'formatted': template % parameters,
},
'sentry.interfaces.User':
next(user_values),
'tags': [
['color', next(tag_values)],
['environment', 'production'],
['sentry:release', 'version'],
],
},
)
with self.tasks():
Group.objects.add_tags(
source,
tags=event.get_tags(),
)
EventMapping.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
date_added=event.datetime,
)
UserReport.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
name='Log Hat',
email='*****@*****.**',
comments='Quack',
)
features.record(event)
return event
events = OrderedDict()
for event in (create_message_event('This is message #%s.', i) for i in xrange(10)):
events.setdefault(get_fingerprint(event), []).append(event)
for event in (create_message_event('This is message #%s!', i) for i in xrange(10, 17)):
events.setdefault(get_fingerprint(event), []).append(event)
assert len(events) == 2
assert sum(map(len, events.values())) == 17
# XXX: This is super contrived considering that it doesn't actually go
# through the event pipeline, but them's the breaks, eh?
for fingerprint in events.keys():
GroupHash.objects.create(
project=project,
group=source,
hash=fingerprint,
)
assert set(GroupTagKey.objects.filter(group=source).values_list('key', 'values_seen')
) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
])
assert set(
GroupTagValue.objects.filter(
group_id=source.id,
).values_list('key', 'value', 'times_seen')
) == set(
[
(u'color', u'red', 6),
(u'color', u'green', 6),
(u'color', u'blue', 5),
(u'environment', u'production', 17),
(u'sentry:release', u'version', 17),
]
)
assert features.compare(source) == [
(source.id, {'message:message:character-shingles': 1.0}),
]
with self.tasks():
unmerge.delay(
source.project_id,
source.id,
None,
[events.keys()[1]],
None,
batch_size=5,
)
assert list(
Group.objects.filter(id=source.id).values_list(
'times_seen',
'first_seen',
'last_seen',
)
) == [(10, now + shift(0), now + shift(9), )]
source_activity = Activity.objects.get(
group_id=source.id,
type=Activity.UNMERGE_SOURCE,
)
destination = Group.objects.get(
id=source_activity.data['destination_id'],
)
assert list(
Group.objects.filter(id=destination.id).values_list(
'times_seen',
'first_seen',
'last_seen',
)
) == [(7, now + shift(10), now + shift(16), )]
assert source_activity.data == {
'destination_id': destination.id,
'fingerprints': [events.keys()[1]],
}
assert source.id != destination.id
assert source.project == destination.project
assert Activity.objects.get(
group_id=destination.id,
type=Activity.UNMERGE_DESTINATION,
).data == {
'source_id': source.id,
'fingerprints': [events.keys()[1]],
}
source_event_event_ids = map(
lambda event: event.event_id,
events.values()[0],
)
assert source.event_set.count() == 10
assert set(
EventMapping.objects.filter(
group_id=source.id,
).values_list('event_id', flat=True)
) == set(source_event_event_ids)
assert set(
UserReport.objects.filter(
group_id=source.id,
).values_list('event_id', flat=True)
) == set(source_event_event_ids)
assert set(GroupHash.objects.filter(
group_id=source.id,
).values_list('hash', flat=True)) == set([events.keys()[0]])
assert set(
GroupRelease.objects.filter(
group_id=source.id,
).values_list('environment', 'first_seen', 'last_seen')
) == set([
(u'production', now + shift(0), now + shift(9), ),
])
assert set(GroupTagKey.objects.filter(group=source).values_list('key', 'values_seen')
) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
])
assert set(
GroupTagValue.objects.filter(
group_id=source.id,
).values_list('key', 'value', 'times_seen', 'first_seen', 'last_seen')
) == set(
[
(u'color', u'red', 4, now + shift(0), now + shift(9), ),
(u'color', u'green', 3, now + shift(1), now + shift(7), ),
(u'color', u'blue', 3, now + shift(2), now + shift(8), ),
(u'environment', u'production', 10, now + shift(0), now + shift(9), ),
(u'sentry:release', u'version', 10, now + shift(0), now + shift(9), ),
]
)
destination_event_event_ids = map(
lambda event: event.event_id,
events.values()[1],
)
assert destination.event_set.count() == 7
assert set(
EventMapping.objects.filter(
group_id=destination.id,
).values_list('event_id', flat=True)
) == set(destination_event_event_ids)
assert set(
UserReport.objects.filter(
group_id=destination.id,
).values_list('event_id', flat=True)
) == set(destination_event_event_ids)
assert set(
GroupHash.objects.filter(
group_id=destination.id,
).values_list('hash', flat=True)
) == set([events.keys()[1]])
assert set(
GroupRelease.objects.filter(
group_id=destination.id,
).values_list('environment', 'first_seen', 'last_seen')
) == set([
(u'production', now + shift(10), now + shift(16), ),
])
assert set(GroupTagKey.objects.filter(group=destination).values_list('key', 'values_seen')
) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
])
assert set(
GroupTagValue.objects.filter(
group_id=destination.id,
).values_list('key', 'value', 'times_seen', 'first_seen', 'last_seen')
) == set(
[
(u'color', u'red', 2, now + shift(12), now + shift(15), ),
(u'color', u'green', 3, now + shift(10), now + shift(16), ),
(u'color', u'blue', 2, now + shift(11), now + shift(14), ),
(u'environment', u'production', 7, now + shift(10), now + shift(16), ),
(u'sentry:release', u'version', 7, now + shift(10), now + shift(16), ),
]
)
time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now,
now + shift(16),
)
def get_expected_series_values(rollup, events, function=None):
if function is None:
def function(aggregate, event):
return (aggregate if aggregate is not None else 0) + 1
expected = {}
for event in events:
k = float((to_timestamp(event.datetime) // rollup_duration) * rollup_duration)
expected[k] = function(expected.get(k), event)
return expected
def assert_series_contains(expected, actual, default=0):
actual = dict(actual)
for key, value in expected.items():
assert actual[key] == value
for key in set(actual.keys()) - set(expected.keys()):
assert actual[key] == default
rollup_duration = time_series.values()[0][1][0] - time_series.values()[0][0][0]
assert_series_contains(
get_expected_series_values(rollup_duration, events.values()[0]),
time_series[source.id],
0,
)
assert_series_contains(
get_expected_series_values(rollup_duration, events.values()[1]),
time_series[destination.id],
0,
)
time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now,
now + shift(16),
)
rollup_duration = time_series.values()[0][1][0] - time_series.values()[0][0][0]
def collect_by_user_tag(aggregate, event):
aggregate = aggregate if aggregate is not None else set()
aggregate.add(
get_event_user_from_interface(
event.data['sentry.interfaces.User'],
).tag_value,
)
return aggregate
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[0],
collect_by_user_tag,
).items()
},
time_series[source.id],
)
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[1],
collect_by_user_tag,
).items()
},
time_series[destination.id],
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_releases_by_group,
[source.id, destination.id],
now,
now + shift(16),
)
rollup_duration = time_series.values()[0][1][0] - time_series.values()[0][0][0]
def collect_by_release(group, aggregate, event):
aggregate = aggregate if aggregate is not None else {}
release = GroupRelease.objects.get(
group_id=group.id,
environment=event.data['environment'],
release_id=Release.objects.get(
organization_id=project.organization_id,
version=event.get_tag('sentry:release'),
).id,
).id
aggregate[release] = aggregate.get(release, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
functools.partial(
collect_by_release,
source,
),
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
functools.partial(
collect_by_release,
destination,
),
),
time_series[destination.id],
{},
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_environments_by_group,
[source.id, destination.id],
now,
now + shift(16),
)
rollup_duration = time_series.values()[0][1][0] - time_series.values()[0][0][0]
def collect_by_environment(aggregate, event):
aggregate = aggregate if aggregate is not None else {}
environment = Environment.objects.get(
organization_id=project.organization_id,
name=event.data['environment'],
).id
aggregate[environment] = aggregate.get(environment, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
collect_by_environment,
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
collect_by_environment,
),
time_series[destination.id],
{},
)
source_similar_items = features.compare(source)
assert source_similar_items[0] == (source.id, {'message:message:character-shingles': 1.0})
assert source_similar_items[1][0] == destination.id
assert source_similar_items[1][1].keys() == ['message:message:character-shingles']
assert source_similar_items[1][1]['message:message:character-shingles'] < 1.0
destination_similar_items = features.compare(destination)
assert destination_similar_items[0] == (
destination.id, {'message:message:character-shingles': 1.0})
assert destination_similar_items[1][0] == source.id
assert destination_similar_items[1][1].keys() == ['message:message:character-shingles']
assert destination_similar_items[1][1]['message:message:character-shingles'] < 1.0
def test_unmerge(self, mock_eventstream):
eventstream_state = object()
mock_eventstream.start_unmerge = Mock(return_value=eventstream_state)
def shift(i):
return timedelta(seconds=1 << i)
now = timezone.now().replace(microsecond=0) - shift(16)
project = self.create_project()
source = self.create_group(project)
sequence = itertools.count(0)
tag_values = itertools.cycle(['red', 'green', 'blue'])
user_values = itertools.cycle([
{
'id': 1
},
{
'id': 2
},
])
for environment in ('production', ''):
EnvironmentProject.objects.create(
environment=Environment.objects.create(
organization_id=project.organization_id,
name=environment,
),
project=project,
)
def create_message_event(template, parameters, environment, release):
i = next(sequence)
event_id = uuid.UUID(
fields=(i, 0x0, 0x1000, 0x80, 0x80, 0x808080808080, ),
).hex
tags = [['color', next(tag_values)]]
if environment:
tags.append(['environment', environment])
if release:
tags.append(['sentry:release', release])
event = Event.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
message='%s' % (id, ),
datetime=now + shift(i),
data={
'environment': environment,
'type': 'default',
'metadata': {
'title': template % parameters,
},
'logentry': {
'message': template,
'params': parameters,
'formatted': template % parameters,
},
'user': next(user_values),
'tags': tags,
},
)
with self.tasks():
Group.objects.add_tags(
source,
Environment.objects.get(
organization_id=project.organization_id,
name=environment
),
tags=event.get_tags(),
)
EventMapping.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
date_added=event.datetime,
)
UserReport.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
name='Log Hat',
email='*****@*****.**',
comments='Quack',
)
if release:
Release.get_or_create(
project=project,
version=event.get_tag('sentry:release'),
date_added=event.datetime,
)
features.record([event])
return event
events = OrderedDict()
for event in (create_message_event('This is message #%s.', i,
environment='production', release='version') for i in xrange(10)):
events.setdefault(get_fingerprint(event), []).append(event)
for event in (create_message_event('This is message #%s!', i,
environment='production', release='version') for i in xrange(10, 16)):
events.setdefault(get_fingerprint(event), []).append(event)
event = create_message_event('This is message #%s!', 17, environment='', release=None)
events.setdefault(get_fingerprint(event), []).append(event)
assert len(events) == 2
assert sum(map(len, events.values())) == 17
# XXX: This is super contrived considering that it doesn't actually go
# through the event pipeline, but them's the breaks, eh?
for fingerprint in events.keys():
GroupHash.objects.create(
project=project,
group=source,
hash=fingerprint,
)
production_environment = Environment.objects.get(
organization_id=project.organization_id,
name='production'
)
assert set(
[(gtk.key, gtk.values_seen)
for gtk in tagstore.get_group_tag_keys(source.project_id, source.id, production_environment.id)]
) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1)
])
if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'):
assert set(
[(gtv.key, gtv.value, gtv.times_seen, Environment.objects.get(pk=gtv._key.environment_id).name)
for gtv in
GroupTagValue.objects.filter(
project_id=source.project_id,
group_id=source.id,
).exclude(_key__environment_id=0)]
) == set([
('color', 'red', 6, 'production'),
('sentry:release', 'version', 16, 'production'),
('color', 'blue', 5, 'production'),
('color', 'green', 5, 'production'),
('environment', 'production', 16, 'production'),
('color', 'green', 1, ''),
])
else:
assert set(
[(gtv.key, gtv.value, gtv.times_seen)
for gtv in
GroupTagValue.objects.filter(
project_id=source.project_id,
group_id=source.id,
)]
) == set([
(u'color', u'red', 6),
(u'color', u'green', 6),
(u'color', u'blue', 5),
(u'environment', u'production', 16),
(u'sentry:release', u'version', 16),
])
assert features.compare(source) == [
(source.id, {
'exception:message:character-shingles': None,
'exception:stacktrace:application-chunks': None,
'exception:stacktrace:pairs': None,
'message:message:character-shingles': 1.0
}),
]
with self.tasks():
unmerge.delay(
source.project_id,
source.id,
None,
[events.keys()[1]],
None,
batch_size=5,
)
assert list(
Group.objects.filter(id=source.id).values_list(
'times_seen',
'first_seen',
'last_seen',
)
) == [(10, now + shift(0), now + shift(9), )]
source_activity = Activity.objects.get(
group_id=source.id,
type=Activity.UNMERGE_SOURCE,
)
destination = Group.objects.get(
id=source_activity.data['destination_id'],
)
mock_eventstream.start_unmerge.assert_called_once_with(
source.project_id, [events.keys()[1]], source.id, destination.id
)
mock_eventstream.end_unmerge.assert_called_once_with(eventstream_state)
assert list(
Group.objects.filter(id=destination.id).values_list(
'times_seen',
'first_seen',
'last_seen',
)
) == [(7, now + shift(10), now + shift(16), )]
assert source_activity.data == {
'destination_id': destination.id,
'fingerprints': [events.keys()[1]],
}
assert source.id != destination.id
assert source.project == destination.project
assert Activity.objects.get(
group_id=destination.id,
type=Activity.UNMERGE_DESTINATION,
).data == {
'source_id': source.id,
'fingerprints': [events.keys()[1]],
}
source_event_event_ids = map(
lambda event: event.event_id,
events.values()[0],
)
assert source.event_set.count() == 10
assert set(
EventMapping.objects.filter(
group_id=source.id,
).values_list('event_id', flat=True)
) == set(source_event_event_ids)
assert set(
UserReport.objects.filter(
group_id=source.id,
).values_list('event_id', flat=True)
) == set(source_event_event_ids)
assert set(GroupHash.objects.filter(
group_id=source.id,
).values_list('hash', flat=True)) == set([events.keys()[0]])
assert set(
GroupRelease.objects.filter(
group_id=source.id,
).values_list('environment', 'first_seen', 'last_seen')
) == set([
(u'production', now + shift(0), now + shift(9), ),
])
assert set(
[(gtk.key, gtk.values_seen)
for gtk in tagstore.get_group_tag_keys(source.project_id, source.id, production_environment.id)]
) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
])
if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'):
env_filter = {'_key__environment_id': production_environment.id}
else:
env_filter = {}
assert set(
[(gtv.key, gtv.value, gtv.times_seen,
gtv.first_seen, gtv.last_seen)
for gtv in
GroupTagValue.objects.filter(
project_id=source.project_id,
group_id=source.id,
**env_filter
)]
) == set([
(u'color', u'red', 4, now + shift(0), now + shift(9), ),
(u'color', u'green', 3, now + shift(1), now + shift(7), ),
(u'color', u'blue', 3, now + shift(2), now + shift(8), ),
(u'environment', u'production', 10, now + shift(0), now + shift(9), ),
(u'sentry:release', u'version', 10, now + shift(0), now + shift(9), ),
])
destination_event_event_ids = map(
lambda event: event.event_id,
events.values()[1],
)
assert destination.event_set.count() == 7
assert set(
EventMapping.objects.filter(
group_id=destination.id,
).values_list('event_id', flat=True)
) == set(destination_event_event_ids)
assert set(
UserReport.objects.filter(
group_id=destination.id,
).values_list('event_id', flat=True)
) == set(destination_event_event_ids)
assert set(
GroupHash.objects.filter(
group_id=destination.id,
).values_list('hash', flat=True)
) == set([events.keys()[1]])
assert set(
GroupRelease.objects.filter(
group_id=destination.id,
).values_list('environment', 'first_seen', 'last_seen')
) == set([
(u'production', now + shift(10), now + shift(15), ),
])
assert set([(gtk.key, gtk.values_seen)
for gtk in tagstore.get_group_tag_keys(source.project_id, source.id, production_environment.id)]
) == set(
[
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
]
)
if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'):
assert set(
[(gtv.key, gtv.value, gtv.times_seen,
gtv.first_seen, gtv.last_seen)
for gtv in
GroupTagValue.objects.filter(
project_id=destination.project_id,
group_id=destination.id,
**env_filter
)]
) == set([
(u'color', u'red', 2, now + shift(12), now + shift(15), ),
(u'color', u'green', 2, now + shift(10), now + shift(13), ),
(u'color', u'blue', 2, now + shift(11), now + shift(14), ),
(u'environment', u'production', 6, now + shift(10), now + shift(15), ),
(u'sentry:release', u'version', 6, now + shift(10), now + shift(15), ),
])
else:
assert set(
[(gtv.key, gtv.value, gtv.times_seen,
gtv.first_seen, gtv.last_seen)
for gtv in
GroupTagValue.objects.filter(
project_id=destination.project_id,
group_id=destination.id,
**env_filter
)]
) == set([
(u'color', u'red', 2, now + shift(12), now + shift(15), ),
(u'color', u'green', 3, now + shift(10), now + shift(16), ),
(u'color', u'blue', 2, now + shift(11), now + shift(14), ),
(u'environment', u'production', 6, now + shift(10), now + shift(15), ),
(u'sentry:release', u'version', 6, now + shift(10), now + shift(15), ),
])
rollup_duration = 3600
time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(15),
rollup_duration,
)
environment_time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(15),
rollup_duration,
environment_ids=[production_environment.id],
)
def get_expected_series_values(rollup, events, function=None):
if function is None:
def function(aggregate, event):
return (aggregate if aggregate is not None else 0) + 1
expected = {}
for event in events:
k = float((to_timestamp(event.datetime) // rollup_duration) * rollup_duration)
expected[k] = function(expected.get(k), event)
return expected
def assert_series_contains(expected, actual, default=0):
actual = dict(actual)
for key, value in expected.items():
assert actual.get(key, 0) == value
for key in set(actual.keys()) - set(expected.keys()):
assert actual.get(key, 0) == default
for series in [time_series, environment_time_series]:
assert_series_contains(
get_expected_series_values(rollup_duration, events.values()[0]),
series[source.id],
0,
)
assert_series_contains(
get_expected_series_values(rollup_duration, events.values()[1][:-1]),
series[destination.id],
0,
)
time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
environment_time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
environment_id=production_environment.id,
)
def collect_by_user_tag(aggregate, event):
aggregate = aggregate if aggregate is not None else set()
aggregate.add(
get_event_user_from_interface(
event.data['user'],
).tag_value,
)
return aggregate
for series in [time_series, environment_time_series]:
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[0],
collect_by_user_tag,
).items()
},
series[source.id],
)
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[1],
collect_by_user_tag,
).items()
},
time_series[destination.id],
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_releases_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_release(group, aggregate, event):
aggregate = aggregate if aggregate is not None else {}
release = event.get_tag('sentry:release')
if not release:
return aggregate
release = GroupRelease.objects.get(
group_id=group.id,
environment=event.data['environment'],
release_id=Release.objects.get(
organization_id=project.organization_id,
version=release,
).id,
).id
aggregate[release] = aggregate.get(release, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
functools.partial(
collect_by_release,
source,
),
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
functools.partial(
collect_by_release,
destination,
),
),
time_series[destination.id],
{},
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_environments_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_environment(aggregate, event):
aggregate = aggregate if aggregate is not None else {}
environment = Environment.objects.get(
organization_id=project.organization_id,
name=event.data['environment'],
).id
aggregate[environment] = aggregate.get(environment, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
collect_by_environment,
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
collect_by_environment,
),
time_series[destination.id],
{},
)
source_similar_items = features.compare(source)
assert source_similar_items[0] == (source.id, {
'exception:message:character-shingles': None,
'exception:stacktrace:application-chunks': None,
'exception:stacktrace:pairs': None,
'message:message:character-shingles': 1.0,
})
assert source_similar_items[1][0] == destination.id
assert source_similar_items[1][1]['message:message:character-shingles'] < 1.0
destination_similar_items = features.compare(destination)
assert destination_similar_items[0] == (
destination.id, {
'exception:message:character-shingles': None,
'exception:stacktrace:application-chunks': None,
'exception:stacktrace:pairs': None,
'message:message:character-shingles': 1.0
}
)
assert destination_similar_items[1][0] == source.id
assert destination_similar_items[1][1]['message:message:character-shingles'] < 1.0
def get(self, request, group):
"""
Retrieve an Issue
`````````````````
Return details on an individual issue. This returns the basic stats for
the issue (title, last seen, first seen), some overall numbers (number
of comments, user reports) as well as the summarized event data.
:pparam string issue_id: the ID of the issue to retrieve.
:auth: required
"""
# TODO(dcramer): handle unauthenticated/public response
data = serialize(group, request.user)
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=100)
seen_by = self._get_seen_by(request, group)
# find first seen release
if group.first_release is None:
try:
first_release = GroupTagValue.objects.filter(
group=group,
key__in=('sentry:release', 'release'),
).order_by('first_seen')[0]
except IndexError:
first_release = None
else:
first_release = first_release.value
else:
first_release = group.first_release.version
if first_release is not None:
# find last seen release
try:
last_release = GroupTagValue.objects.filter(
group=group,
key__in=('sentry:release', 'release'),
).order_by('-last_seen')[0]
except IndexError:
last_release = None
else:
last_release = last_release.value
else:
last_release = None
action_list = self._get_actions(request, group)
now = timezone.now()
hourly_stats = tsdb.rollup(tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=1),
), 3600)[group.id]
daily_stats = tsdb.rollup(tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=30),
), 3600 * 24)[group.id]
if first_release:
first_release = self._get_release_info(request, group, first_release)
if last_release:
last_release = self._get_release_info(request, group, last_release)
tags = list(GroupTagKey.objects.filter(
group=group,
)[:100])
participants = list(User.objects.filter(
groupsubscription__is_active=True,
groupsubscription__group=group,
))
data.update({
'firstRelease': first_release,
'lastRelease': last_release,
'activity': serialize(activity, request.user),
'seenBy': seen_by,
'participants': serialize(participants, request.user),
'pluginActions': action_list,
'pluginIssues': self._get_available_issue_plugins(request, group),
'userReportCount': UserReport.objects.filter(group=group).count(),
'tags': sorted(serialize(tags, request.user), key=lambda x: x['name']),
'stats': {
'24h': hourly_stats,
'30d': daily_stats,
}
})
return Response(data)
def get(self, request, group):
"""
Retrieve an aggregate
Return details on an individual aggregate.
{method} {path}
"""
# TODO(dcramer): handle unauthenticated/public response
data = serialize(group, request.user)
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=100)
seen_by = self._get_seen_by(request, group)
# find first seen release
if group.first_release is None:
try:
first_release = GroupTagValue.objects.filter(
group=group,
key__in=('sentry:release', 'release'),
).order_by('first_seen')[0]
except IndexError:
first_release = None
else:
first_release = first_release.value
else:
first_release = group.first_release.version
if first_release is not None:
# find last seen release
try:
last_release = GroupTagValue.objects.filter(
group=group,
key__in=('sentry:release', 'release'),
).order_by('-last_seen')[0]
except IndexError:
last_release = None
else:
last_release = last_release.value
else:
last_release = None
action_list = self._get_actions(request, group)
now = timezone.now()
hourly_stats = tsdb.rollup(
tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=1),
), 3600)[group.id]
daily_stats = tsdb.rollup(
tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=30),
), 3600 * 24)[group.id]
if first_release:
first_release = self._get_release_info(request, group,
first_release)
if last_release:
last_release = self._get_release_info(request, group, last_release)
data.update({
'firstRelease':
first_release,
'lastRelease':
last_release,
'activity':
serialize(activity, request.user),
'seenBy':
seen_by,
'pluginActions':
action_list,
'userReportCount':
UserReport.objects.filter(group=group).count(),
'stats': {
'24h': hourly_stats,
'30d': daily_stats,
}
})
return Response(data)
def attach_metadata(self, objects, request=None):
from sentry.templatetags.sentry_plugins import handle_before_events
attach_foreignkey(objects, Group.project, ['team'])
GroupMeta.objects.populate_cache(objects)
if request and objects:
handle_before_events(request, objects)
if request and request.user.is_authenticated() and objects:
bookmarks = set(GroupBookmark.objects.filter(
user=request.user,
group__in=objects,
).values_list('group_id', flat=True))
seen_groups = dict(GroupSeen.objects.filter(
user=request.user,
group__in=objects,
).values_list('group_id', 'last_seen'))
else:
bookmarks = set()
seen_groups = {}
if objects:
end = timezone.now()
start = end - timedelta(days=1)
historical_data = tsdb.get_range(
model=tsdb.models.group,
keys=[g.id for g in objects],
start=start,
end=end,
)
else:
historical_data = {}
project_list = set(o.project for o in objects)
tag_keys = set(['sentry:user'])
project_annotations = {}
for project in project_list:
enabled_annotations = ProjectOption.objects.get_value(
project, 'annotations', ['sentry:user'])
project_annotations[project] = enabled_annotations
tag_keys.update(enabled_annotations)
annotation_counts = defaultdict(dict)
annotation_results = GroupTagKey.objects.filter(
group__in=objects,
key__in=tag_keys,
).values_list('key', 'group', 'values_seen')
for key, group_id, values_seen in annotation_results:
annotation_counts[key][group_id] = values_seen
for g in objects:
g.is_bookmarked = g.pk in bookmarks
g.historical_data = [x[1] for x in historical_data.get(g.id, [])]
active_date = g.active_at or g.last_seen
g.has_seen = seen_groups.get(g.id, active_date) > active_date
g.annotations = []
for key in sorted(tag_keys):
if key in project_annotations[project]:
label = TAG_LABELS.get(key, key.replace('_', ' ')).lower() + 's'
try:
value = annotation_counts[key].get(g.id, 0)
except KeyError:
value = 0
g.annotations.append({
'label': label,
'count': value,
})
def get(self, request, project):
"""
Retrieve Event Counts for a Project
```````````````````````````````````
.. caution::
This endpoint may change in the future without notice.
Return a set of points representing a normalized timestamp and the
number of events seen in the period.
Query ranges are limited to Sentry's configured time-series
resolutions.
:pparam string organization_slug: the slug of the organization.
:pparam string project_slug: the slug of the project.
:qparam string stat: the name of the stat to query (``"received"``,
``"rejected"``, ``"blacklisted"``)
:qparam timestamp since: a timestamp to set the start of the query
in seconds since UNIX epoch.
:qparam timestamp until: a timestamp to set the end of the query
in seconds since UNIX epoch.
:qparam string resolution: an explicit resolution to search
for (eg: ``10s``). This should not be
used unless you are familiar with Sentry's
internals as it's restricted to pre-defined
values.
:auth: required
"""
# add by hzwangzhiwei
# if action is "stat", then get the stats category
action = request.GET.get('action', '')
if action == 'stat':
# update by hzwangzhiwei @20160816 for #860
stats = {'TOTAL': 0, 'RESOLVED': 0, 'MUTED': 0} # result
from sentry.models import Group, GroupStatus
from django.utils import timezone
from datetime import timedelta
from django.db.models import Count
# status_map = ['UNRESOLVED', 'RESOLVED', 'MUTED', 'PENDING_DELETION', 'DELETION_IN_PROGRESS', 'PENDING_MERGE']
# 1. search status == 'RESOLVED'
statsQuerySet = Group.objects.filter(project_id=project.id, status=GroupStatus.RESOLVED).aggregate(cnt=Count('id'))
stats['RESOLVED'] = stats['RESOLVED'] + int(statsQuerySet.get('cnt', 0))
# 2. search auto RESOLVED
resolve_age = project.get_option('sentry:resolve_age', None)
if resolve_age:
statsQuerySet = Group.objects.filter(project_id=project.id, status=GroupStatus.UNRESOLVED, last_seen__lte=(timezone.now()-timedelta(hours=int(resolve_age)))).aggregate(cnt=Count('id'))
stats['RESOLVED'] = stats['RESOLVED'] + int(statsQuerySet.get('cnt', 0))
# 3. search all count.
statsQuerySet = Group.objects.filter(project_id=project.id).aggregate(cnt=Count('id'))
stats['TOTAL'] = stats['TOTAL'] + int(statsQuerySet.get('cnt', 0))
# 4. ignore count, MUTED
statsQuerySet = Group.objects.filter(project_id=project.id, status=GroupStatus.MUTED).aggregate(cnt=Count('id'))
stats['MUTED'] = int(statsQuerySet.get('cnt', 0))
return Response(stats)
elif action == 'topIssueType':
cnt = 15
try:
cnt = int(request.GET.get('cnt', ''))
except:
cnt = 15
stats = []
cursor = connection.cursor()
# select
raw_sql = "select id, substring_index(message, ': ',1) as issue_type, count(id) as cnt from sentry_groupedmessage where project_id = %s group by issue_type order by cnt desc limit %s;"
cursor.execute(raw_sql, [project.id, cnt])
raw_querySet = dictFetchAll(cursor)
for s in raw_querySet:
stats.append({'name': s.get('issue_type', ''), 'value': s.get('cnt', 0)})
return Response(stats)
elif action == 'topIssuePerson':
cnt = 15
try:
cnt = int(request.GET.get('cnt', ''))
except:
cnt = 15
stats = []
cursor = connection.cursor()
# select
raw_sql = "select sentry_groupasignee.id, first_name, email, count(user_id) as cnt from sentry_groupasignee join auth_user on sentry_groupasignee.user_id = auth_user.id where project_id = %s group by user_id order by cnt desc limit %s;"
cursor.execute(raw_sql, [project.id, cnt])
raw_querySet = dictFetchAll(cursor)
for s in raw_querySet:
stats.append({'name': s.get('first_name', ''), 'value': s.get('cnt', 0), 'email': s.get('email', '')})
return Response(stats)
stat = request.GET.get('stat', 'received')
if stat == 'received':
stat_model = tsdb.models.project_total_received
elif stat == 'rejected':
stat_model = tsdb.models.project_total_rejected
elif stat == 'blacklisted':
stat_model = tsdb.models.project_total_blacklisted
else:
raise ValueError('Invalid stat: %s' % stat)
data = tsdb.get_range(
model=stat_model,
keys=[project.id],
**self._parse_args(request)
)[project.id]
return Response(data)
def test_unmerge(self, mock_eventstream):
eventstream_state = object()
mock_eventstream.start_unmerge = Mock(return_value=eventstream_state)
def shift(i):
return timedelta(seconds=1 << i)
now = timezone.now() - shift(16)
project = self.create_project()
source = self.create_group(project)
sequence = itertools.count(0)
tag_values = itertools.cycle(["red", "green", "blue"])
user_values = itertools.cycle([{"id": 1}, {"id": 2}])
for environment in ("production", ""):
EnvironmentProject.objects.create(
environment=Environment.objects.create(
organization_id=project.organization_id, name=environment),
project=project,
)
def create_message_event(template, parameters, environment, release):
i = next(sequence)
event_id = uuid.UUID(fields=(i, 0x0, 0x1000, 0x80, 0x80,
0x808080808080)).hex
tags = [["color", next(tag_values)]]
if environment:
tags.append(["environment", environment])
if release:
tags.append(["sentry:release", release])
event = Event.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
message="%s" % (id, ),
datetime=now + shift(i),
data={
"environment": environment,
"type": "default",
"metadata": {
"title": template % parameters
},
"logentry": {
"message": template,
"params": parameters,
"formatted": template % parameters,
},
"user": next(user_values),
"tags": tags,
},
)
with self.tasks():
Group.objects.add_tags(
source,
Environment.objects.get(
organization_id=project.organization_id,
name=environment),
tags=event.tags,
)
UserReport.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
name="Log Hat",
email="*****@*****.**",
comments="Quack",
)
if release:
Release.get_or_create(
project=project,
version=event.get_tag("sentry:release"),
date_added=event.datetime,
)
features.record([event])
return event
events = OrderedDict()
for event in (create_message_event("This is message #%s.",
i,
environment="production",
release="version")
for i in xrange(10)):
events.setdefault(get_fingerprint(event), []).append(event)
for event in (create_message_event("This is message #%s!",
i,
environment="production",
release="version")
for i in xrange(10, 16)):
events.setdefault(get_fingerprint(event), []).append(event)
event = create_message_event("This is message #%s!",
17,
environment="",
release=None)
events.setdefault(get_fingerprint(event), []).append(event)
assert len(events) == 2
assert sum(map(len, events.values())) == 17
# XXX: This is super contrived considering that it doesn't actually go
# through the event pipeline, but them's the breaks, eh?
for fingerprint in events.keys():
GroupHash.objects.create(project=project,
group=source,
hash=fingerprint)
production_environment = Environment.objects.get(
organization_id=project.organization_id, name="production")
assert set([
(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(
source.project_id, source.id, [production_environment.id])
]) == set([(u"color", 3), (u"environment", 1), (u"sentry:release", 1)])
if settings.SENTRY_TAGSTORE.startswith("sentry.tagstore.v2"):
assert set([(
gtv.key,
gtv.value,
gtv.times_seen,
Environment.objects.get(pk=gtv._key.environment_id).name,
) for gtv in GroupTagValue.objects.filter(
project_id=source.project_id, group_id=source.id).exclude(
_key__environment_id=0)]) == set([
("color", "red", 6, "production"),
("sentry:release", "version", 16, "production"),
("color", "blue", 5, "production"),
("color", "green", 5, "production"),
("environment", "production", 16, "production"),
("color", "green", 1, ""),
])
else:
assert set([(gtv.key, gtv.value, gtv.times_seen)
for gtv in GroupTagValue.objects.filter(
project_id=source.project_id, group_id=source.id)
]) == set([
(u"color", u"red", 6),
(u"color", u"green", 6),
(u"color", u"blue", 5),
(u"environment", u"production", 16),
(u"sentry:release", u"version", 16),
])
assert features.compare(source) == [(
source.id,
{
"exception:message:character-shingles": None,
"exception:stacktrace:application-chunks": None,
"exception:stacktrace:pairs": None,
"message:message:character-shingles": 1.0,
},
)]
with self.tasks():
unmerge.delay(source.project_id,
source.id,
None, [events.keys()[1]],
None,
batch_size=5)
assert list(
Group.objects.filter(id=source.id).values_list(
"times_seen", "first_seen",
"last_seen")) == [(10, now + shift(0), now + shift(9))]
source_activity = Activity.objects.get(group_id=source.id,
type=Activity.UNMERGE_SOURCE)
destination = Group.objects.get(
id=source_activity.data["destination_id"])
mock_eventstream.start_unmerge.assert_called_once_with(
source.project_id, [events.keys()[1]], source.id, destination.id)
mock_eventstream.end_unmerge.assert_called_once_with(eventstream_state)
assert list(
Group.objects.filter(id=destination.id).values_list(
"times_seen", "first_seen",
"last_seen")) == [(7, now + shift(10), now + shift(16))]
assert source_activity.data == {
"destination_id": destination.id,
"fingerprints": [events.keys()[1]],
}
assert source.id != destination.id
assert source.project == destination.project
assert Activity.objects.get(
group_id=destination.id,
type=Activity.UNMERGE_DESTINATION).data == {
"source_id": source.id,
"fingerprints": [events.keys()[1]]
}
source_event_event_ids = map(lambda event: event.event_id,
events.values()[0])
assert set(
UserReport.objects.filter(group_id=source.id).values_list(
"event_id", flat=True)) == set(source_event_event_ids)
assert set(
GroupHash.objects.filter(group_id=source.id).values_list(
"hash", flat=True)) == set([events.keys()[0]])
assert set(
GroupRelease.objects.filter(group_id=source.id).values_list(
"environment", "first_seen", "last_seen")) == set([
(u"production", now + shift(0), now + shift(9))
])
assert set([
(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(
source.project_id, source.id, [production_environment.id])
]) == set([(u"color", 3), (u"environment", 1), (u"sentry:release", 1)])
if settings.SENTRY_TAGSTORE.startswith("sentry.tagstore.v2"):
env_filter = {"_key__environment_id": production_environment.id}
else:
env_filter = {}
assert set([
(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen)
for gtv in GroupTagValue.objects.filter(
project_id=source.project_id, group_id=source.id, **env_filter)
]) == set([
(u"color", u"red", 4, now + shift(0), now + shift(9)),
(u"color", u"green", 3, now + shift(1), now + shift(7)),
(u"color", u"blue", 3, now + shift(2), now + shift(8)),
(u"environment", u"production", 10, now + shift(0),
now + shift(9)),
(u"sentry:release", u"version", 10, now + shift(0),
now + shift(9)),
])
destination_event_event_ids = map(lambda event: event.event_id,
events.values()[1])
assert set(
UserReport.objects.filter(group_id=destination.id).values_list(
"event_id", flat=True)) == set(destination_event_event_ids)
assert set(
GroupHash.objects.filter(group_id=destination.id).values_list(
"hash", flat=True)) == set([events.keys()[1]])
assert set(
GroupRelease.objects.filter(group_id=destination.id).values_list(
"environment", "first_seen", "last_seen")) == set([
(u"production", now + shift(10), now + shift(15))
])
assert set([
(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(
source.project_id, source.id, [production_environment.id])
]) == set([(u"color", 3), (u"environment", 1), (u"sentry:release", 1)])
if settings.SENTRY_TAGSTORE.startswith("sentry.tagstore.v2"):
assert set([
(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen,
gtv.last_seen) for gtv in GroupTagValue.objects.filter(
project_id=destination.project_id,
group_id=destination.id,
**env_filter)
]) == set([
(u"color", u"red", 2, now + shift(12), now + shift(15)),
(u"color", u"green", 2, now + shift(10), now + shift(13)),
(u"color", u"blue", 2, now + shift(11), now + shift(14)),
(u"environment", u"production", 6, now + shift(10),
now + shift(15)),
(u"sentry:release", u"version", 6, now + shift(10),
now + shift(15)),
])
else:
assert set([
(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen,
gtv.last_seen) for gtv in GroupTagValue.objects.filter(
project_id=destination.project_id,
group_id=destination.id,
**env_filter)
]) == set([
(u"color", u"red", 2, now + shift(12), now + shift(15)),
(u"color", u"green", 3, now + shift(10), now + shift(16)),
(u"color", u"blue", 2, now + shift(11), now + shift(14)),
(u"environment", u"production", 6, now + shift(10),
now + shift(15)),
(u"sentry:release", u"version", 6, now + shift(10),
now + shift(15)),
])
rollup_duration = 3600
time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(15),
rollup_duration,
)
environment_time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(15),
rollup_duration,
environment_ids=[production_environment.id],
)
def get_expected_series_values(rollup, events, function=None):
if function is None:
def function(aggregate, event):
return (aggregate if aggregate is not None else 0) + 1
expected = {}
for event in events:
k = float((to_timestamp(event.datetime) // rollup_duration) *
rollup_duration)
expected[k] = function(expected.get(k), event)
return expected
def assert_series_contains(expected, actual, default=0):
actual = dict(actual)
for key, value in expected.items():
assert actual.get(key, 0) == value
for key in set(actual.keys()) - set(expected.keys()):
assert actual.get(key, 0) == default
for series in [time_series, environment_time_series]:
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[0]),
series[source.id],
0,
)
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[1][:-1]),
series[destination.id],
0,
)
time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
environment_time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
environment_id=production_environment.id,
)
def collect_by_user_tag(aggregate, event):
aggregate = aggregate if aggregate is not None else set()
aggregate.add(
get_event_user_from_interface(event.data["user"]).tag_value)
return aggregate
for series in [time_series, environment_time_series]:
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[0], collect_by_user_tag).items()
},
series[source.id],
)
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[1], collect_by_user_tag).items()
},
time_series[destination.id],
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_releases_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_release(group, aggregate, event):
aggregate = aggregate if aggregate is not None else {}
release = event.get_tag("sentry:release")
if not release:
return aggregate
release = GroupRelease.objects.get(
group_id=group.id,
environment=event.data["environment"],
release_id=Release.objects.get(
organization_id=project.organization_id,
version=release).id,
).id
aggregate[release] = aggregate.get(release, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
functools.partial(collect_by_release, source)),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
functools.partial(collect_by_release, destination),
),
time_series[destination.id],
{},
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_environments_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_environment(aggregate, event):
aggregate = aggregate if aggregate is not None else {}
environment = Environment.objects.get(
organization_id=project.organization_id,
name=event.data["environment"]).id
aggregate[environment] = aggregate.get(environment, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[0],
collect_by_environment),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[1],
collect_by_environment),
time_series[destination.id],
{},
)
source_similar_items = features.compare(source)
assert source_similar_items[0] == (
source.id,
{
"exception:message:character-shingles": None,
"exception:stacktrace:application-chunks": None,
"exception:stacktrace:pairs": None,
"message:message:character-shingles": 1.0,
},
)
assert source_similar_items[1][0] == destination.id
assert source_similar_items[1][1][
"message:message:character-shingles"] < 1.0
destination_similar_items = features.compare(destination)
assert destination_similar_items[0] == (
destination.id,
{
"exception:message:character-shingles": None,
"exception:stacktrace:application-chunks": None,
"exception:stacktrace:pairs": None,
"message:message:character-shingles": 1.0,
},
)
assert destination_similar_items[1][0] == source.id
assert destination_similar_items[1][1][
"message:message:character-shingles"] < 1.0
def test_unmerge(self):
now = before_now(minutes=5).replace(microsecond=0, tzinfo=pytz.utc)
def time_from_now(offset=0):
return now + timedelta(seconds=offset)
project = self.create_project()
sequence = itertools.count(0)
tag_values = itertools.cycle(["red", "green", "blue"])
user_values = itertools.cycle([{"id": 1}, {"id": 2}])
def create_message_event(template, parameters, environment, release, fingerprint="group1"):
i = next(sequence)
event_id = uuid.UUID(fields=(i, 0x0, 0x1000, 0x80, 0x80, 0x808080808080)).hex
tags = [["color", next(tag_values)]]
if release:
tags.append(["sentry:release", release])
event = self.store_event(
data={
"event_id": event_id,
"message": template % parameters,
"type": "default",
"user": next(user_values),
"tags": tags,
"fingerprint": [fingerprint],
"timestamp": iso_format(now + timedelta(seconds=i)),
"environment": environment,
"release": release,
},
project_id=project.id,
)
UserReport.objects.create(
project_id=project.id,
group_id=event.group.id,
event_id=event_id,
name="Log Hat",
email="*****@*****.**",
comments="Quack",
)
features.record([event])
return event
events = OrderedDict()
for event in (
create_message_event(
"This is message #%s.", i, environment="production", release="version"
)
for i in xrange(10)
):
events.setdefault(get_fingerprint(event), []).append(event)
for event in (
create_message_event(
"This is message #%s!",
i,
environment="production",
release="version2",
fingerprint="group2",
)
for i in xrange(10, 16)
):
events.setdefault(get_fingerprint(event), []).append(event)
event = create_message_event(
"This is message #%s!",
17,
environment="staging",
release="version3",
fingerprint="group3",
)
events.setdefault(get_fingerprint(event), []).append(event)
merge_source, source, destination = list(Group.objects.all())
assert len(events) == 3
assert sum(map(len, events.values())) == 17
production_environment = Environment.objects.get(
organization_id=project.organization_id, name="production"
)
with self.tasks():
eventstream_state = eventstream.start_merge(project.id, [merge_source.id], source.id)
merge_groups.delay([merge_source.id], source.id)
eventstream.end_merge(eventstream_state)
assert set(
[
(gtv.value, gtv.times_seen)
for gtv in tagstore.get_group_tag_values(
project.id, source.id, production_environment.id, "color"
)
]
) == set([("red", 6), ("green", 5), ("blue", 5)])
similar_items = features.compare(source)
assert len(similar_items) == 2
assert similar_items[0][0] == source.id
assert similar_items[0][1]["message:message:character-shingles"] == 1.0
assert similar_items[1][0] == destination.id
assert similar_items[1][1]["message:message:character-shingles"] < 1.0
with self.tasks():
eventstream_state = eventstream.start_unmerge(
project.id, [list(events.keys())[0]], source.id, destination.id
)
unmerge.delay(
project.id, source.id, destination.id, [list(events.keys())[0]], None, batch_size=5
)
eventstream.end_unmerge(eventstream_state)
assert (
list(
Group.objects.filter(id=merge_source.id).values_list(
"times_seen", "first_seen", "last_seen"
)
)
== []
)
assert list(
Group.objects.filter(id=source.id).values_list("times_seen", "first_seen", "last_seen")
) == [(6, time_from_now(10), time_from_now(15))]
assert list(
Group.objects.filter(id=destination.id).values_list(
"times_seen", "first_seen", "last_seen"
)
) == [(11, time_from_now(0), time_from_now(16))]
assert source.id != destination.id
assert source.project == destination.project
destination_event_ids = map(lambda event: event.event_id, list(events.values())[1])
assert set(
UserReport.objects.filter(group_id=source.id).values_list("event_id", flat=True)
) == set(destination_event_ids)
assert set(
GroupHash.objects.filter(group_id=source.id).values_list("hash", flat=True)
) == set(itertools.islice(events.keys(), 2))
assert set(
GroupRelease.objects.filter(group_id=source.id).values_list(
"environment", "first_seen", "last_seen"
)
) == set([(u"production", time_from_now(10), time_from_now(15))])
assert set(
[
(gtv.value, gtv.times_seen)
for gtv in tagstore.get_group_tag_values(
project.id, destination.id, production_environment.id, "color"
)
]
) == set([(u"red", 4), (u"green", 3), (u"blue", 3)])
destination_event_ids = map(
lambda event: event.event_id, list(events.values())[0] + list(events.values())[2]
)
assert set(
UserReport.objects.filter(group_id=destination.id).values_list("event_id", flat=True)
) == set(destination_event_ids)
assert set(
GroupHash.objects.filter(group_id=destination.id).values_list("hash", flat=True)
) == set(itertools.islice(events.keys(), 2, 3))
assert set(
GroupRelease.objects.filter(group_id=destination.id).values_list(
"environment", "first_seen", "last_seen"
)
) == set(
[
("production", time_from_now(0), time_from_now(9)),
("staging", time_from_now(16), time_from_now(16)),
]
)
assert set(
[
(gtk.value, gtk.times_seen)
for gtk in tagstore.get_group_tag_values(
project.id, destination.id, production_environment.id, "color"
)
]
) == set([("red", 4), ("blue", 3), ("green", 3)])
rollup_duration = 3600
time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
time_from_now(17),
rollup_duration,
)
environment_time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
time_from_now(17),
rollup_duration,
environment_ids=[production_environment.id],
)
def get_expected_series_values(rollup, events, function=None):
if function is None:
def function(aggregate, event):
return (aggregate if aggregate is not None else 0) + 1
expected = {}
for event in events:
k = float((to_timestamp(event.datetime) // rollup_duration) * rollup_duration)
expected[k] = function(expected.get(k), event)
return expected
def assert_series_contains(expected, actual, default=0):
actual = dict(actual)
for key, value in expected.items():
assert actual.get(key, 0) == value
for key in set(actual.keys()) - set(expected.keys()):
assert actual.get(key, 0) == default
assert_series_contains(
get_expected_series_values(rollup_duration, list(events.values())[1]),
time_series[source.id],
0,
)
assert_series_contains(
get_expected_series_values(
rollup_duration, list(events.values())[0] + list(events.values())[2]
),
time_series[destination.id],
0,
)
assert_series_contains(
get_expected_series_values(rollup_duration, list(events.values())[1]),
environment_time_series[source.id],
0,
)
assert_series_contains(
get_expected_series_values(
rollup_duration, list(events.values())[0][:-1] + list(events.values())[2]
),
environment_time_series[destination.id],
0,
)
time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
time_from_now(17),
rollup_duration,
)
environment_time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
time_from_now(17),
rollup_duration,
environment_id=production_environment.id,
)
def collect_by_user_tag(aggregate, event):
aggregate = aggregate if aggregate is not None else set()
aggregate.add(get_event_user_from_interface(event.data["user"]).tag_value)
return aggregate
for series in [time_series, environment_time_series]:
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration, list(events.values())[1], collect_by_user_tag
).items()
},
series[source.id],
)
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
list(events.values())[0] + list(events.values())[2],
collect_by_user_tag,
).items()
},
time_series[destination.id],
)
def strip_zeroes(data):
for group_id, series in data.items():
for _, values in series:
for key, val in list(values.items()):
if val == 0:
values.pop(key)
return data
def collect_by_release(group, aggregate, event):
aggregate = aggregate if aggregate is not None else {}
release = event.get_tag("sentry:release")
if not release:
return aggregate
release = GroupRelease.objects.get(
group_id=group.id,
environment=event.data["environment"],
release_id=Release.objects.get(
organization_id=project.organization_id, version=release
).id,
).id
aggregate[release] = aggregate.get(release, 0) + 1
return aggregate
items = {}
for i in [source.id, destination.id]:
items[i] = list(GroupRelease.objects.filter(group_id=i).values_list("id", flat=True))
time_series = strip_zeroes(
tsdb.get_frequency_series(
tsdb.models.frequent_releases_by_group,
items,
now - timedelta(seconds=rollup_duration),
time_from_now(17),
rollup_duration,
)
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
list(events.values())[1],
functools.partial(collect_by_release, source),
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
list(events.values())[0] + list(events.values())[2],
functools.partial(collect_by_release, destination),
),
time_series[destination.id],
{},
)
items = {}
for i in [source.id, destination.id]:
items[i] = list(Environment.objects.all().values_list("id", flat=True))
time_series = strip_zeroes(
tsdb.get_frequency_series(
tsdb.models.frequent_environments_by_group,
items,
now - timedelta(seconds=rollup_duration),
time_from_now(17),
rollup_duration,
)
)
def collect_by_environment(aggregate, event):
aggregate = aggregate if aggregate is not None else {}
environment = Environment.objects.get(
organization_id=project.organization_id, name=event.data["environment"]
).id
aggregate[environment] = aggregate.get(environment, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration, list(events.values())[1], collect_by_environment
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
list(events.values())[0] + list(events.values())[2],
collect_by_environment,
),
time_series[destination.id],
{},
)
source_similar_items = features.compare(source)
assert source_similar_items[0] == (
source.id,
{
"exception:message:character-shingles": None,
"exception:stacktrace:application-chunks": None,
"exception:stacktrace:pairs": None,
"message:message:character-shingles": 1.0,
},
)
assert source_similar_items[1][0] == destination.id
assert source_similar_items[1][1]["message:message:character-shingles"] < 1.0
destination_similar_items = features.compare(destination)
assert destination_similar_items[0] == (
destination.id,
{
"exception:message:character-shingles": None,
"exception:stacktrace:application-chunks": None,
"exception:stacktrace:pairs": None,
"message:message:character-shingles": 1.0,
},
)
assert destination_similar_items[1][0] == source.id
assert destination_similar_items[1][1]["message:message:character-shingles"] < 1.0
def get(self, request, organization):
"""
Retrieve Event Counts for an Organization
`````````````````````````````````````````
.. caution::
This endpoint may change in the future without notice.
Return a set of points representing a normalized timestamp and the
number of events seen in the period.
:pparam string organization_slug: the slug of the organization for
which the stats should be
retrieved.
:qparam string stat: the name of the stat to query (``"received"``,
``"rejected"``)
:qparam timestamp since: a timestamp to set the start of the query
in seconds since UNIX epoch.
:qparam timestamp until: a timestamp to set the end of the query
in seconds since UNIX epoch.
:qparam string resolution: an explicit resolution to search
for (eg: ``10s``). This should not be
used unless you are familiar with Sentry's
internals as it's restricted to pre-defined
values.
:auth: required
"""
group = request.GET.get('group')
if not group:
keys = [organization.id]
elif group == 'project':
team_list = Team.objects.get_for_user(
organization=organization,
user=request.user,
)
project_list = []
for team in team_list:
project_list.extend(
Project.objects.get_for_user(
team=team,
user=request.user,
))
keys = [p.id for p in project_list]
else:
raise ValueError('Invalid group: %s' % group)
if not keys:
return Response([])
stat = request.GET.get('stat', 'received')
if stat == 'received':
if group == 'project':
stat_model = tsdb.models.project_total_received
else:
stat_model = tsdb.models.organization_total_received
elif stat == 'rejected':
if group == 'project':
stat_model = tsdb.models.project_total_rejected
else:
stat_model = tsdb.models.organization_total_rejected
else:
raise ValueError('Invalid stat: %s' % stat)
data = tsdb.get_range(model=stat_model,
keys=keys,
**self._parse_args(request))
if not group:
data = data[organization.id]
return Response(data)
def prepare_project_series((start, stop), project, rollup=60 * 60 * 24):
resolution, series = tsdb.get_optimal_rollup_series(start, stop, rollup)
assert resolution == rollup, 'resolution does not match requested value'
clean = functools.partial(clean_series, start, stop, rollup)
return merge_series(
reduce(
merge_series,
map(
clean,
tsdb.get_range(
tsdb.models.group,
project.group_set.filter(
status=GroupStatus.RESOLVED,
resolved_at__gte=start,
resolved_at__lt=stop,
).values_list('id', flat=True),
start,
stop,
rollup=rollup,
).values(),
),
clean([(timestamp, 0) for timestamp in series]),
),
clean(
tsdb.get_range(
tsdb.models.project,
[project.id],
start,
stop,
rollup=rollup,
def get(self, request, group):
"""
Retrieve an Issue
`````````````````
Return details on an individual issue. This returns the basic stats for
the issue (title, last seen, first seen), some overall numbers (number
of comments, user reports) as well as the summarized event data.
:pparam string issue_id: the ID of the issue to retrieve.
:auth: required
"""
# TODO(dcramer): handle unauthenticated/public response
data = serialize(group, request.user)
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=100)
seen_by = self._get_seen_by(request, group)
# find first seen release
if group.first_release is None:
try:
first_release = GroupTagValue.objects.filter(
group=group, key__in=("sentry:release", "release")
).order_by("first_seen")[0]
except IndexError:
first_release = None
else:
first_release = first_release.value
else:
first_release = group.first_release.version
if first_release is not None:
# find last seen release
try:
last_release = GroupTagValue.objects.filter(
group=group, key__in=("sentry:release", "release")
).order_by("-last_seen")[0]
except IndexError:
last_release = None
else:
last_release = last_release.value
else:
last_release = None
action_list = self._get_actions(request, group)
now = timezone.now()
hourly_stats = tsdb.rollup(
tsdb.get_range(model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=1)), 3600
)[group.id]
daily_stats = tsdb.rollup(
tsdb.get_range(model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=30)), 3600 * 24
)[group.id]
if first_release:
first_release = self._get_release_info(request, group, first_release)
if last_release:
last_release = self._get_release_info(request, group, last_release)
tags = list(GroupTagKey.objects.filter(group=group)[:100])
data.update(
{
"firstRelease": first_release,
"lastRelease": last_release,
"activity": serialize(activity, request.user),
"seenBy": seen_by,
"pluginActions": action_list,
"userReportCount": UserReport.objects.filter(group=group).count(),
"tags": sorted(serialize(tags, request.user), key=lambda x: x["name"]),
"stats": {"24h": hourly_stats, "30d": daily_stats},
}
)
return Response(data)
