def get(self, request, group):
group_tag_keys = tagstore.get_group_tag_keys(group.id)
# O(N) db access
data = []
all_top_values = []
for group_tag_key in group_tag_keys:
total_values = tagstore.get_group_tag_value_count(group.id, group_tag_key.key)
top_values = tagstore.get_top_group_tag_values(group.id, group_tag_key.key, limit=10)
all_top_values.extend(top_values)
data.append(
{
'id': six.text_type(group_tag_key.id),
'key': tagstore.get_standardized_key(group_tag_key.key),
'name': tagstore.get_tag_key_label(group_tag_key.key),
'uniqueValues': group_tag_key.values_seen,
'totalValues': total_values,
}
)
# Serialize all of the values at once to avoid O(n) serialize/db queries
top_values_by_key = defaultdict(list)
for value in serialize(all_top_values, request.user):
top_values_by_key[value['key']].append(value)
for d in data:
d['topValues'] = top_values_by_key[d['key']]
return Response(data)
def update_tag_value_counts(id_list):
instances = tagstore.get_group_tag_keys(id_list)
for instance in instances:
instance.update(
values_seen=GroupTagValue.objects.filter(
project_id=instance.project_id,
group_id=instance.group_id,
key=instance.key,
).count(),
)
def attach_metadata(self, objects, request=None):
from sentry.templatetags.sentry_plugins import handle_before_events
attach_foreignkey(objects, Group.project, ['team'])
GroupMeta.objects.populate_cache(objects)
if request and objects:
handle_before_events(request, objects)
if request and request.user.is_authenticated() and objects:
bookmarks = set(
GroupBookmark.objects.filter(
user=request.user,
group__in=objects,
).values_list('group_id', flat=True))
seen_groups = dict(
GroupSeen.objects.filter(
user=request.user,
group__in=objects,
).values_list('group_id', 'last_seen'))
else:
bookmarks = set()
seen_groups = {}
if objects:
end = timezone.now()
start = end - timedelta(days=1)
historical_data = tsdb.get_range(
model=tsdb.models.group,
keys=[g.id for g in objects],
start=start,
end=end,
)
else:
historical_data = {}
user_tagkeys = tagstore.get_group_tag_keys([o.id for o in objects],
'sentry:user')
user_counts = {}
for user_tagkey in user_tagkeys:
user_counts[user_tagkey.group_id] = user_tagkey.values_seen
for g in objects:
g.is_bookmarked = g.pk in bookmarks
g.historical_data = [x[1] for x in historical_data.get(g.id, [])]
active_date = g.active_at or g.first_seen
g.has_seen = seen_groups.get(g.id, active_date) > active_date
g.annotations = [{
'label': 'users',
'count': user_counts.get(g.id, 0),
}]
def get_tags(self):
if not hasattr(self, '_tag_cache'):
group_tags = [gtk.key for gtk in tagstore.get_group_tag_keys(self.id)]
results = []
for key in group_tags:
results.append({
'key': key,
'label': tagstore.get_tag_key_label(key),
})
self._tag_cache = sorted(results, key=lambda x: x['label'])
return self._tag_cache
def get_tags(self):
if not hasattr(self, '_tag_cache'):
group_tags = [gtk.key for gtk in tagstore.get_group_tag_keys(self.id)]
results = []
for key in group_tags:
results.append({
'key': key,
'label': tagstore.get_tag_key_label(key),
})
self._tag_cache = sorted(results, key=lambda x: x['label'])
return self._tag_cache
def test_unmerge(self, mock_eventstream):
eventstream_state = object()
mock_eventstream.start_unmerge = Mock(return_value=eventstream_state)
def shift(i):
return timedelta(seconds=1 << i)
now = timezone.now().replace(microsecond=0) - shift(16)
project = self.create_project()
source = self.create_group(project)
sequence = itertools.count(0)
tag_values = itertools.cycle(['red', 'green', 'blue'])
user_values = itertools.cycle([
{
'id': 1
},
{
'id': 2
},
])
for environment in ('production', ''):
EnvironmentProject.objects.create(
environment=Environment.objects.create(
organization_id=project.organization_id,
name=environment,
),
project=project,
)
def create_message_event(template, parameters, environment, release):
i = next(sequence)
event_id = uuid.UUID(
fields=(i, 0x0, 0x1000, 0x80, 0x80, 0x808080808080, ),
).hex
tags = [['color', next(tag_values)]]
if environment:
tags.append(['environment', environment])
if release:
tags.append(['sentry:release', release])
event = Event.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
message='%s' % (id, ),
datetime=now + shift(i),
data={
'environment': environment,
'type': 'default',
'metadata': {
'title': template % parameters,
},
'logentry': {
'message': template,
'params': parameters,
'formatted': template % parameters,
},
'user': next(user_values),
'tags': tags,
},
)
with self.tasks():
Group.objects.add_tags(
source,
Environment.objects.get(
organization_id=project.organization_id,
name=environment
),
tags=event.get_tags(),
)
EventMapping.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
date_added=event.datetime,
)
UserReport.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
name='Log Hat',
email='*****@*****.**',
comments='Quack',
)
if release:
Release.get_or_create(
project=project,
version=event.get_tag('sentry:release'),
date_added=event.datetime,
)
features.record([event])
return event
events = OrderedDict()
for event in (create_message_event('This is message #%s.', i,
environment='production', release='version') for i in xrange(10)):
events.setdefault(get_fingerprint(event), []).append(event)
for event in (create_message_event('This is message #%s!', i,
environment='production', release='version') for i in xrange(10, 16)):
events.setdefault(get_fingerprint(event), []).append(event)
event = create_message_event('This is message #%s!', 17, environment='', release=None)
events.setdefault(get_fingerprint(event), []).append(event)
assert len(events) == 2
assert sum(map(len, events.values())) == 17
# XXX: This is super contrived considering that it doesn't actually go
# through the event pipeline, but them's the breaks, eh?
for fingerprint in events.keys():
GroupHash.objects.create(
project=project,
group=source,
hash=fingerprint,
)
production_environment = Environment.objects.get(
organization_id=project.organization_id,
name='production'
)
assert set(
[(gtk.key, gtk.values_seen)
for gtk in tagstore.get_group_tag_keys(source.project_id, source.id, production_environment.id)]
) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1)
])
if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'):
assert set(
[(gtv.key, gtv.value, gtv.times_seen, Environment.objects.get(pk=gtv._key.environment_id).name)
for gtv in
GroupTagValue.objects.filter(
project_id=source.project_id,
group_id=source.id,
).exclude(_key__environment_id=0)]
) == set([
('color', 'red', 6, 'production'),
('sentry:release', 'version', 16, 'production'),
('color', 'blue', 5, 'production'),
('color', 'green', 5, 'production'),
('environment', 'production', 16, 'production'),
('color', 'green', 1, ''),
])
else:
assert set(
[(gtv.key, gtv.value, gtv.times_seen)
for gtv in
GroupTagValue.objects.filter(
project_id=source.project_id,
group_id=source.id,
)]
) == set([
(u'color', u'red', 6),
(u'color', u'green', 6),
(u'color', u'blue', 5),
(u'environment', u'production', 16),
(u'sentry:release', u'version', 16),
])
assert features.compare(source) == [
(source.id, {
'exception:message:character-shingles': None,
'exception:stacktrace:application-chunks': None,
'exception:stacktrace:pairs': None,
'message:message:character-shingles': 1.0
}),
]
with self.tasks():
unmerge.delay(
source.project_id,
source.id,
None,
[events.keys()[1]],
None,
batch_size=5,
)
assert list(
Group.objects.filter(id=source.id).values_list(
'times_seen',
'first_seen',
'last_seen',
)
) == [(10, now + shift(0), now + shift(9), )]
source_activity = Activity.objects.get(
group_id=source.id,
type=Activity.UNMERGE_SOURCE,
)
destination = Group.objects.get(
id=source_activity.data['destination_id'],
)
mock_eventstream.start_unmerge.assert_called_once_with(
source.project_id, [events.keys()[1]], source.id, destination.id
)
mock_eventstream.end_unmerge.assert_called_once_with(eventstream_state)
assert list(
Group.objects.filter(id=destination.id).values_list(
'times_seen',
'first_seen',
'last_seen',
)
) == [(7, now + shift(10), now + shift(16), )]
assert source_activity.data == {
'destination_id': destination.id,
'fingerprints': [events.keys()[1]],
}
assert source.id != destination.id
assert source.project == destination.project
assert Activity.objects.get(
group_id=destination.id,
type=Activity.UNMERGE_DESTINATION,
).data == {
'source_id': source.id,
'fingerprints': [events.keys()[1]],
}
source_event_event_ids = map(
lambda event: event.event_id,
events.values()[0],
)
assert source.event_set.count() == 10
assert set(
EventMapping.objects.filter(
group_id=source.id,
).values_list('event_id', flat=True)
) == set(source_event_event_ids)
assert set(
UserReport.objects.filter(
group_id=source.id,
).values_list('event_id', flat=True)
) == set(source_event_event_ids)
assert set(GroupHash.objects.filter(
group_id=source.id,
).values_list('hash', flat=True)) == set([events.keys()[0]])
assert set(
GroupRelease.objects.filter(
group_id=source.id,
).values_list('environment', 'first_seen', 'last_seen')
) == set([
(u'production', now + shift(0), now + shift(9), ),
])
assert set(
[(gtk.key, gtk.values_seen)
for gtk in tagstore.get_group_tag_keys(source.project_id, source.id, production_environment.id)]
) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
])
if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'):
env_filter = {'_key__environment_id': production_environment.id}
else:
env_filter = {}
assert set(
[(gtv.key, gtv.value, gtv.times_seen,
gtv.first_seen, gtv.last_seen)
for gtv in
GroupTagValue.objects.filter(
project_id=source.project_id,
group_id=source.id,
**env_filter
)]
) == set([
(u'color', u'red', 4, now + shift(0), now + shift(9), ),
(u'color', u'green', 3, now + shift(1), now + shift(7), ),
(u'color', u'blue', 3, now + shift(2), now + shift(8), ),
(u'environment', u'production', 10, now + shift(0), now + shift(9), ),
(u'sentry:release', u'version', 10, now + shift(0), now + shift(9), ),
])
destination_event_event_ids = map(
lambda event: event.event_id,
events.values()[1],
)
assert destination.event_set.count() == 7
assert set(
EventMapping.objects.filter(
group_id=destination.id,
).values_list('event_id', flat=True)
) == set(destination_event_event_ids)
assert set(
UserReport.objects.filter(
group_id=destination.id,
).values_list('event_id', flat=True)
) == set(destination_event_event_ids)
assert set(
GroupHash.objects.filter(
group_id=destination.id,
).values_list('hash', flat=True)
) == set([events.keys()[1]])
assert set(
GroupRelease.objects.filter(
group_id=destination.id,
).values_list('environment', 'first_seen', 'last_seen')
) == set([
(u'production', now + shift(10), now + shift(15), ),
])
assert set([(gtk.key, gtk.values_seen)
for gtk in tagstore.get_group_tag_keys(source.project_id, source.id, production_environment.id)]
) == set(
[
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
]
)
if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'):
assert set(
[(gtv.key, gtv.value, gtv.times_seen,
gtv.first_seen, gtv.last_seen)
for gtv in
GroupTagValue.objects.filter(
project_id=destination.project_id,
group_id=destination.id,
**env_filter
)]
) == set([
(u'color', u'red', 2, now + shift(12), now + shift(15), ),
(u'color', u'green', 2, now + shift(10), now + shift(13), ),
(u'color', u'blue', 2, now + shift(11), now + shift(14), ),
(u'environment', u'production', 6, now + shift(10), now + shift(15), ),
(u'sentry:release', u'version', 6, now + shift(10), now + shift(15), ),
])
else:
assert set(
[(gtv.key, gtv.value, gtv.times_seen,
gtv.first_seen, gtv.last_seen)
for gtv in
GroupTagValue.objects.filter(
project_id=destination.project_id,
group_id=destination.id,
**env_filter
)]
) == set([
(u'color', u'red', 2, now + shift(12), now + shift(15), ),
(u'color', u'green', 3, now + shift(10), now + shift(16), ),
(u'color', u'blue', 2, now + shift(11), now + shift(14), ),
(u'environment', u'production', 6, now + shift(10), now + shift(15), ),
(u'sentry:release', u'version', 6, now + shift(10), now + shift(15), ),
])
rollup_duration = 3600
time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(15),
rollup_duration,
)
environment_time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(15),
rollup_duration,
environment_ids=[production_environment.id],
)
def get_expected_series_values(rollup, events, function=None):
if function is None:
def function(aggregate, event):
return (aggregate if aggregate is not None else 0) + 1
expected = {}
for event in events:
k = float((to_timestamp(event.datetime) // rollup_duration) * rollup_duration)
expected[k] = function(expected.get(k), event)
return expected
def assert_series_contains(expected, actual, default=0):
actual = dict(actual)
for key, value in expected.items():
assert actual.get(key, 0) == value
for key in set(actual.keys()) - set(expected.keys()):
assert actual.get(key, 0) == default
for series in [time_series, environment_time_series]:
assert_series_contains(
get_expected_series_values(rollup_duration, events.values()[0]),
series[source.id],
0,
)
assert_series_contains(
get_expected_series_values(rollup_duration, events.values()[1][:-1]),
series[destination.id],
0,
)
time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
environment_time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
environment_id=production_environment.id,
)
def collect_by_user_tag(aggregate, event):
aggregate = aggregate if aggregate is not None else set()
aggregate.add(
get_event_user_from_interface(
event.data['user'],
).tag_value,
)
return aggregate
for series in [time_series, environment_time_series]:
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[0],
collect_by_user_tag,
).items()
},
series[source.id],
)
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[1],
collect_by_user_tag,
).items()
},
time_series[destination.id],
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_releases_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_release(group, aggregate, event):
aggregate = aggregate if aggregate is not None else {}
release = event.get_tag('sentry:release')
if not release:
return aggregate
release = GroupRelease.objects.get(
group_id=group.id,
environment=event.data['environment'],
release_id=Release.objects.get(
organization_id=project.organization_id,
version=release,
).id,
).id
aggregate[release] = aggregate.get(release, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
functools.partial(
collect_by_release,
source,
),
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
functools.partial(
collect_by_release,
destination,
),
),
time_series[destination.id],
{},
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_environments_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_environment(aggregate, event):
aggregate = aggregate if aggregate is not None else {}
environment = Environment.objects.get(
organization_id=project.organization_id,
name=event.data['environment'],
).id
aggregate[environment] = aggregate.get(environment, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
collect_by_environment,
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
collect_by_environment,
),
time_series[destination.id],
{},
)
source_similar_items = features.compare(source)
assert source_similar_items[0] == (source.id, {
'exception:message:character-shingles': None,
'exception:stacktrace:application-chunks': None,
'exception:stacktrace:pairs': None,
'message:message:character-shingles': 1.0,
})
assert source_similar_items[1][0] == destination.id
assert source_similar_items[1][1]['message:message:character-shingles'] < 1.0
destination_similar_items = features.compare(destination)
assert destination_similar_items[0] == (
destination.id, {
'exception:message:character-shingles': None,
'exception:stacktrace:application-chunks': None,
'exception:stacktrace:pairs': None,
'message:message:character-shingles': 1.0
}
)
assert destination_similar_items[1][0] == source.id
assert destination_similar_items[1][1]['message:message:character-shingles'] < 1.0
def test_merge_updates_tag_values_seen(self):
project = self.create_project()
target, other = [self.create_group(project) for _ in range(0, 2)]
data = {
'sentry:user': {
'id:1': {
target: 2,
},
'id:2': {
other: 3,
},
'id:3': {
target: 1,
other: 2,
},
},
'key': {
'foo': {
other: 3,
},
},
}
input_group_tag_keys = defaultdict(int) # [(group, key)] = values_seen
input_group_tag_values = defaultdict(
int) # [(group, key, value)] = times_seen
output_group_tag_keys = defaultdict(int) # [key] = values_seen
output_group_tag_values = defaultdict(
int) # [(key, value)] = times_seen
for key, values in data.items():
output_group_tag_keys[key] = len(values)
for value, groups in values.items():
for group, count in groups.items():
input_group_tag_keys[(group, key)] += 1
input_group_tag_values[(group, key, value)] += count
output_group_tag_values[(key, value)] += count
for ((group, key), values_seen) in input_group_tag_keys.items():
tagstore.create_group_tag_key(
project_id=project.id,
group_id=group.id,
environment_id=self.environment.id,
key=key,
values_seen=values_seen,
)
for ((group, key, value),
times_seen) in input_group_tag_values.items():
tagstore.create_group_tag_value(
project_id=project.id,
group_id=group.id,
environment_id=self.environment.id,
key=key,
value=value,
times_seen=times_seen,
)
with self.tasks():
merge_group(other.id, target.id)
assert not Group.objects.filter(id=other.id).exists()
assert len(
tagstore.get_group_tag_keys(
other.project_id, other.id,
environment_id=self.environment.id)) == 0
assert len(
GroupTagValue.objects.filter(
project_id=other.project_id,
group_id=other.id,
)) == 0
for key, values_seen in output_group_tag_keys.items():
assert tagstore.get_group_tag_key(
target.project_id,
target.id,
environment_id=self.environment.id,
key=key).values_seen == values_seen
for (key, value), times_seen in output_group_tag_values.items():
assert tagstore.get_group_tag_value(
project_id=target.project_id,
group_id=target.id,
environment_id=self.environment.id,
key=key,
value=value,
).times_seen == times_seen
def get(self, request: Request, group) -> Response:
"""
Retrieve an Issue
`````````````````
Return details on an individual issue. This returns the basic stats for
the issue (title, last seen, first seen), some overall numbers (number
of comments, user reports) as well as the summarized event data.
:pparam string issue_id: the ID of the issue to retrieve.
:auth: required
"""
from sentry.utils import snuba
try:
# TODO(dcramer): handle unauthenticated/public response
organization = group.project.organization
environments = get_environments(request, organization)
environment_ids = [e.id for e in environments]
expand = request.GET.getlist("expand", [])
collapse = request.GET.getlist("collapse", [])
# WARNING: the rest of this endpoint relies on this serializer
# populating the cache SO don't move this :)
data = serialize(
group, request.user,
GroupSerializerSnuba(environment_ids=environment_ids))
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=100)
seen_by = self._get_seen_by(request, group)
if "release" not in collapse:
first_release, last_release = get_first_last_release(
request, group)
data.update({
"firstRelease": first_release,
"lastRelease": last_release,
})
get_range = functools.partial(tsdb.get_range,
environment_ids=environment_ids)
tags = tagstore.get_group_tag_keys(group.project_id,
group.id,
environment_ids,
limit=100)
if not environment_ids:
user_reports = UserReport.objects.filter(group_id=group.id)
else:
user_reports = UserReport.objects.filter(
group_id=group.id, environment_id__in=environment_ids)
now = timezone.now()
hourly_stats = tsdb.rollup(
get_range(model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=1)),
3600,
)[group.id]
daily_stats = tsdb.rollup(
get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=30),
),
3600 * 24,
)[group.id]
participants = GroupSubscriptionManager.get_participating_users(
group)
if "inbox" in expand:
inbox_map = get_inbox_details([group])
inbox_reason = inbox_map.get(group.id)
data.update({"inbox": inbox_reason})
action_list = self._get_actions(request, group)
data.update({
"activity":
serialize(activity, request.user),
"seenBy":
seen_by,
"participants":
serialize(participants, request.user),
"pluginActions":
action_list,
"pluginIssues":
self._get_available_issue_plugins(request, group),
"pluginContexts":
self._get_context_plugins(request, group),
"userReportCount":
user_reports.count(),
"tags":
sorted(serialize(tags, request.user), key=lambda x: x["name"]),
"stats": {
"24h": hourly_stats,
"30d": daily_stats
},
})
metrics.incr(
"group.update.http_response",
sample_rate=1.0,
tags={
"status": 200,
"detail": "group_details:get:response"
},
)
return Response(data)
except snuba.RateLimitExceeded:
metrics.incr(
"group.update.http_response",
sample_rate=1.0,
tags={
"status": 429,
"detail": "group_details:get:snuba.RateLimitExceeded"
},
)
raise
except Exception:
metrics.incr(
"group.update.http_response",
sample_rate=1.0,
tags={
"status": 500,
"detail": "group_details:get:Exception"
},
)
raise
def get(self, request, group):
"""
Retrieve an Issue
`````````````````
Return details on an individual issue. This returns the basic stats for
the issue (title, last seen, first seen), some overall numbers (number
of comments, user reports) as well as the summarized event data.
:pparam string issue_id: the ID of the issue to retrieve.
:auth: required
"""
# TODO(dcramer): handle unauthenticated/public response
data = serialize(
group, request.user,
GroupSerializer(environment_func=self._get_environment_func(
request, group.project.organization_id)))
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=100)
seen_by = self._get_seen_by(request, group)
first_release = group.get_first_release()
if first_release is not None:
last_release = group.get_last_release()
else:
last_release = None
action_list = self._get_actions(request, group)
if first_release:
first_release = self._get_release_info(request, group,
first_release)
if last_release:
last_release = self._get_release_info(request, group, last_release)
try:
environment_id = self._get_environment_id_from_request(
request, group.project.organization_id)
except Environment.DoesNotExist:
get_range = lambda model, keys, start, end, **kwargs: \
{k: tsdb.make_series(0, start, end) for k in keys}
tags = []
user_reports = UserReport.objects.none()
else:
get_range = functools.partial(tsdb.get_range,
environment_id=environment_id)
tags = tagstore.get_group_tag_keys(group.project_id,
group.id,
environment_id,
limit=100)
if environment_id is None:
user_reports = UserReport.objects.filter(group=group)
else:
user_reports = UserReport.objects.filter(
group=group, environment_id=environment_id)
now = timezone.now()
hourly_stats = tsdb.rollup(
get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=1),
), 3600)[group.id]
daily_stats = tsdb.rollup(
get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=30),
), 3600 * 24)[group.id]
participants = list(
User.objects.filter(
groupsubscription__is_active=True,
groupsubscription__group=group,
))
data.update({
'firstRelease':
first_release,
'lastRelease':
last_release,
'activity':
serialize(activity, request.user),
'seenBy':
seen_by,
'participants':
serialize(participants, request.user),
'pluginActions':
action_list,
'pluginIssues':
self._get_available_issue_plugins(request, group),
'pluginContexts':
self._get_context_plugins(request, group),
'userReportCount':
user_reports.count(),
'tags':
sorted(serialize(tags, request.user), key=lambda x: x['name']),
'stats': {
'24h': hourly_stats,
'30d': daily_stats,
}
})
# the current release is the 'latest seen' release within the
# environment even if it hasnt affected this issue
try:
environment = self._get_environment_from_request(
request,
group.project.organization_id,
)
except Environment.DoesNotExist:
environment = None
if environment is not None:
try:
current_release = GroupRelease.objects.filter(
group_id=group.id,
environment=environment.name,
release_id=ReleaseEnvironment.objects.filter(
release_id__in=ReleaseProject.objects.filter(
project_id=group.project_id).values_list(
'release_id', flat=True),
organization_id=group.project.organization_id,
environment_id=environment.id,
).order_by('-first_seen').values_list('release_id',
flat=True)[:1],
)[0]
except IndexError:
current_release = None
data.update({
'currentRelease':
serialize(current_release, request.user,
GroupReleaseWithStatsSerializer())
})
return Response(data)
def get(self, request, group):
"""
Retrieve an Issue
`````````````````
Return details on an individual issue. This returns the basic stats for
the issue (title, last seen, first seen), some overall numbers (number
of comments, user reports) as well as the summarized event data.
:pparam string issue_id: the ID of the issue to retrieve.
:auth: required
"""
# TODO(dcramer): handle unauthenticated/public response
data = serialize(group, request.user)
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=100)
seen_by = self._get_seen_by(request, group)
first_release = group.get_first_release()
if first_release is not None:
last_release = group.get_last_release()
else:
last_release = None
action_list = self._get_actions(request, group)
now = timezone.now()
hourly_stats = tsdb.rollup(
tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=1),
), 3600)[group.id]
daily_stats = tsdb.rollup(
tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=30),
), 3600 * 24)[group.id]
if first_release:
first_release = self._get_release_info(request, group,
first_release)
if last_release:
last_release = self._get_release_info(request, group, last_release)
try:
environment_id = self._get_environment_id_from_request(
request, group.project.organization_id)
except Environment.DoesNotExist:
tags = []
else:
tags = tagstore.get_group_tag_keys(group.project_id,
group.id,
environment_id,
limit=100)
participants = list(
User.objects.filter(
groupsubscription__is_active=True,
groupsubscription__group=group,
))
data.update({
'firstRelease':
first_release,
'lastRelease':
last_release,
'activity':
serialize(activity, request.user),
'seenBy':
seen_by,
'participants':
serialize(participants, request.user),
'pluginActions':
action_list,
'pluginIssues':
self._get_available_issue_plugins(request, group),
'pluginContexts':
self._get_context_plugins(request, group),
'userReportCount':
UserReport.objects.filter(group=group).count(),
'tags':
sorted(serialize(tags, request.user), key=lambda x: x['name']),
'stats': {
'24h': hourly_stats,
'30d': daily_stats,
}
})
return Response(data)
def test_merge_updates_tag_values_seen(self):
project = self.create_project()
target, other = [self.create_group(project) for _ in range(0, 2)]
data = {
'sentry:user': {
'id:1': {
target: 2,
},
'id:2': {
other: 3,
},
'id:3': {
target: 1,
other: 2,
},
},
'key': {
'foo': {
other: 3,
},
},
}
input_group_tag_keys = defaultdict(int) # [(group, key)] = values_seen
input_group_tag_values = defaultdict(int) # [(group, key, value)] = times_seen
output_group_tag_keys = defaultdict(int) # [key] = values_seen
output_group_tag_values = defaultdict(int) # [(key, value)] = times_seen
for key, values in data.items():
output_group_tag_keys[key] = len(values)
for value, groups in values.items():
for group, count in groups.items():
input_group_tag_keys[(group, key)] += 1
input_group_tag_values[(group, key, value)] += count
output_group_tag_values[(key, value)] += count
for ((group, key), values_seen) in input_group_tag_keys.items():
tagstore.create_group_tag_key(
project_id=project.id,
group_id=group.id,
environment_id=self.environment.id,
key=key,
values_seen=values_seen,
)
for ((group, key, value), times_seen) in input_group_tag_values.items():
tagstore.create_group_tag_value(
project_id=project.id,
group_id=group.id,
environment_id=self.environment.id,
key=key,
value=value,
times_seen=times_seen,
)
with self.tasks():
merge_group(other.id, target.id)
assert not Group.objects.filter(id=other.id).exists()
assert len(
tagstore.get_group_tag_keys(
other.project_id,
other.id,
environment_id=self.environment.id)) == 0
assert len(
GroupTagValue.objects.filter(
project_id=other.project_id,
group_id=other.id,
)) == 0
for key, values_seen in output_group_tag_keys.items():
assert tagstore.get_group_tag_key(
target.project_id, target.id, environment_id=self.environment.id, key=key).values_seen == values_seen
for (key, value), times_seen in output_group_tag_values.items():
assert tagstore.get_group_tag_value(
project_id=target.project_id,
group_id=target.id,
environment_id=self.environment.id,
key=key,
value=value,
).times_seen == times_seen
def get(self, request, group):
"""
Retrieve an Issue
`````````````````
Return details on an individual issue. This returns the basic stats for
the issue (title, last seen, first seen), some overall numbers (number
of comments, user reports) as well as the summarized event data.
:pparam string issue_id: the ID of the issue to retrieve.
:auth: required
"""
# TODO(dcramer): handle unauthenticated/public response
data = serialize(
group,
request.user,
GroupSerializer(
environment_func=self._get_environment_func(
request, group.project.organization_id)
)
)
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=100)
seen_by = self._get_seen_by(request, group)
first_release = group.get_first_release()
if first_release is not None:
last_release = group.get_last_release()
else:
last_release = None
action_list = self._get_actions(request, group)
if first_release:
first_release = self._get_release_info(request, group, first_release)
if last_release:
last_release = self._get_release_info(request, group, last_release)
try:
environment_id = self._get_environment_id_from_request(
request, group.project.organization_id)
except Environment.DoesNotExist:
get_range = lambda model, keys, start, end, **kwargs: \
{k: tsdb.make_series(0, start, end) for k in keys}
tags = []
user_reports = UserReport.objects.none()
else:
get_range = functools.partial(tsdb.get_range,
environment_ids=environment_id and [environment_id])
tags = tagstore.get_group_tag_keys(
group.project_id, group.id, environment_id, limit=100)
if environment_id is None:
user_reports = UserReport.objects.filter(group=group)
else:
user_reports = UserReport.objects.filter(group=group, environment_id=environment_id)
now = timezone.now()
hourly_stats = tsdb.rollup(
get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=1),
), 3600
)[group.id]
daily_stats = tsdb.rollup(
get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=30),
), 3600 * 24
)[group.id]
participants = list(
User.objects.filter(
groupsubscription__is_active=True,
groupsubscription__group=group,
)
)
data.update(
{
'firstRelease': first_release,
'lastRelease': last_release,
'activity': serialize(activity, request.user),
'seenBy': seen_by,
'participants': serialize(participants, request.user),
'pluginActions': action_list,
'pluginIssues': self._get_available_issue_plugins(request, group),
'pluginContexts': self._get_context_plugins(request, group),
'userReportCount': user_reports.count(),
'tags': sorted(serialize(tags, request.user), key=lambda x: x['name']),
'stats': {
'24h': hourly_stats,
'30d': daily_stats,
}
}
)
# the current release is the 'latest seen' release within the
# environment even if it hasnt affected this issue
try:
environment = self._get_environment_from_request(
request,
group.project.organization_id,
)
except Environment.DoesNotExist:
environment = None
if environment is not None:
try:
current_release = GroupRelease.objects.filter(
group_id=group.id,
environment=environment.name,
release_id=ReleaseEnvironment.objects.filter(
release_id__in=ReleaseProject.objects.filter(project_id=group.project_id
).values_list('release_id', flat=True),
organization_id=group.project.organization_id,
environment_id=environment.id,
).order_by('-first_seen').values_list('release_id', flat=True)[:1],
)[0]
except IndexError:
current_release = None
data.update({
'currentRelease': serialize(
current_release, request.user, GroupReleaseWithStatsSerializer()
)
})
return Response(data)
def get(self, request, group):
"""
Retrieve an Issue
`````````````````
Return details on an individual issue. This returns the basic stats for
the issue (title, last seen, first seen), some overall numbers (number
of comments, user reports) as well as the summarized event data.
:pparam string issue_id: the ID of the issue to retrieve.
:auth: required
"""
# TODO(dcramer): handle unauthenticated/public response
# TODO(jess): This can be removed when tagstore v2 is deprecated
use_snuba = request.GET.get('enable_snuba') == '1'
environments = get_environments(request, group.project.organization)
environment_ids = [e.id for e in environments]
if use_snuba:
# WARNING: the rest of this endpoint relies on this serializer
# populating the cache SO don't move this :)
data = serialize(
group,
request.user,
GroupSerializerSnuba(
environment_ids=environment_ids,
)
)
else:
# TODO(jess): This is just to ensure we're not breaking the old
# issue page somehow -- non-snuba tagstore versions will raise
# if more than one env is passed
if environments:
environments = environments[:1]
environment_ids = environment_ids[:1]
data = serialize(
group,
request.user,
GroupSerializer(
# Just in case multiple envs are passed, let's make
# sure we're using the same one for all the stats
environment_func=lambda: environments[0] if environments else None
)
)
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=100)
seen_by = self._get_seen_by(request, group)
first_release = group.get_first_release()
if first_release is not None:
last_release = group.get_last_release()
else:
last_release = None
action_list = self._get_actions(request, group)
if first_release:
first_release = self._get_release_info(request, group, first_release)
if last_release:
last_release = self._get_release_info(request, group, last_release)
get_range = functools.partial(tsdb.get_range,
environment_ids=environment_ids)
tags = tagstore.get_group_tag_keys(
group.project_id, group.id, environment_ids, limit=100)
if not environment_ids:
user_reports = UserReport.objects.filter(group=group)
else:
user_reports = UserReport.objects.filter(
group=group, environment_id__in=environment_ids
)
now = timezone.now()
hourly_stats = tsdb.rollup(
get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=1),
), 3600
)[group.id]
daily_stats = tsdb.rollup(
get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=30),
), 3600 * 24
)[group.id]
participants = list(
User.objects.filter(
groupsubscription__is_active=True,
groupsubscription__group=group,
)
)
data.update(
{
'firstRelease': first_release,
'lastRelease': last_release,
'activity': serialize(activity, request.user),
'seenBy': seen_by,
'participants': serialize(participants, request.user),
'pluginActions': action_list,
'pluginIssues': self._get_available_issue_plugins(request, group),
'pluginContexts': self._get_context_plugins(request, group),
'userReportCount': user_reports.count(),
'tags': sorted(serialize(tags, request.user), key=lambda x: x['name']),
'stats': {
'24h': hourly_stats,
'30d': daily_stats,
}
}
)
# the current release is the 'latest seen' release within the
# environment even if it hasnt affected this issue
if environments:
try:
current_release = GroupRelease.objects.filter(
group_id=group.id,
environment__in=[env.name for env in environments],
release_id=ReleaseEnvironment.objects.filter(
release_id__in=ReleaseProject.objects.filter(project_id=group.project_id
).values_list('release_id', flat=True),
organization_id=group.project.organization_id,
environment_id__in=environment_ids,
).order_by('-first_seen').values_list('release_id', flat=True)[:1],
)[0]
except IndexError:
current_release = None
data.update({
'currentRelease': serialize(
current_release, request.user, GroupReleaseWithStatsSerializer()
)
})
return Response(data)
def test_unmerge(self, mock_eventstream):
eventstream_state = object()
mock_eventstream.start_unmerge = Mock(return_value=eventstream_state)
def shift(i):
return timedelta(seconds=1 << i)
now = timezone.now() - shift(16)
project = self.create_project()
source = self.create_group(project)
sequence = itertools.count(0)
tag_values = itertools.cycle(["red", "green", "blue"])
user_values = itertools.cycle([{"id": 1}, {"id": 2}])
for environment in ("production", ""):
EnvironmentProject.objects.create(
environment=Environment.objects.create(
organization_id=project.organization_id, name=environment),
project=project,
)
def create_message_event(template, parameters, environment, release):
i = next(sequence)
event_id = uuid.UUID(fields=(i, 0x0, 0x1000, 0x80, 0x80,
0x808080808080)).hex
tags = [["color", next(tag_values)]]
if environment:
tags.append(["environment", environment])
if release:
tags.append(["sentry:release", release])
event = Event.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
message="%s" % (id, ),
datetime=now + shift(i),
data={
"environment": environment,
"type": "default",
"metadata": {
"title": template % parameters
},
"logentry": {
"message": template,
"params": parameters,
"formatted": template % parameters,
},
"user": next(user_values),
"tags": tags,
},
)
with self.tasks():
Group.objects.add_tags(
source,
Environment.objects.get(
organization_id=project.organization_id,
name=environment),
tags=event.tags,
)
UserReport.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
name="Log Hat",
email="*****@*****.**",
comments="Quack",
)
if release:
Release.get_or_create(
project=project,
version=event.get_tag("sentry:release"),
date_added=event.datetime,
)
features.record([event])
return event
events = OrderedDict()
for event in (create_message_event("This is message #%s.",
i,
environment="production",
release="version")
for i in xrange(10)):
events.setdefault(get_fingerprint(event), []).append(event)
for event in (create_message_event("This is message #%s!",
i,
environment="production",
release="version")
for i in xrange(10, 16)):
events.setdefault(get_fingerprint(event), []).append(event)
event = create_message_event("This is message #%s!",
17,
environment="",
release=None)
events.setdefault(get_fingerprint(event), []).append(event)
assert len(events) == 2
assert sum(map(len, events.values())) == 17
# XXX: This is super contrived considering that it doesn't actually go
# through the event pipeline, but them's the breaks, eh?
for fingerprint in events.keys():
GroupHash.objects.create(project=project,
group=source,
hash=fingerprint)
production_environment = Environment.objects.get(
organization_id=project.organization_id, name="production")
assert set([
(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(
source.project_id, source.id, [production_environment.id])
]) == set([(u"color", 3), (u"environment", 1), (u"sentry:release", 1)])
if settings.SENTRY_TAGSTORE.startswith("sentry.tagstore.v2"):
assert set([(
gtv.key,
gtv.value,
gtv.times_seen,
Environment.objects.get(pk=gtv._key.environment_id).name,
) for gtv in GroupTagValue.objects.filter(
project_id=source.project_id, group_id=source.id).exclude(
_key__environment_id=0)]) == set([
("color", "red", 6, "production"),
("sentry:release", "version", 16, "production"),
("color", "blue", 5, "production"),
("color", "green", 5, "production"),
("environment", "production", 16, "production"),
("color", "green", 1, ""),
])
else:
assert set([(gtv.key, gtv.value, gtv.times_seen)
for gtv in GroupTagValue.objects.filter(
project_id=source.project_id, group_id=source.id)
]) == set([
(u"color", u"red", 6),
(u"color", u"green", 6),
(u"color", u"blue", 5),
(u"environment", u"production", 16),
(u"sentry:release", u"version", 16),
])
assert features.compare(source) == [(
source.id,
{
"exception:message:character-shingles": None,
"exception:stacktrace:application-chunks": None,
"exception:stacktrace:pairs": None,
"message:message:character-shingles": 1.0,
},
)]
with self.tasks():
unmerge.delay(source.project_id,
source.id,
None, [events.keys()[1]],
None,
batch_size=5)
assert list(
Group.objects.filter(id=source.id).values_list(
"times_seen", "first_seen",
"last_seen")) == [(10, now + shift(0), now + shift(9))]
source_activity = Activity.objects.get(group_id=source.id,
type=Activity.UNMERGE_SOURCE)
destination = Group.objects.get(
id=source_activity.data["destination_id"])
mock_eventstream.start_unmerge.assert_called_once_with(
source.project_id, [events.keys()[1]], source.id, destination.id)
mock_eventstream.end_unmerge.assert_called_once_with(eventstream_state)
assert list(
Group.objects.filter(id=destination.id).values_list(
"times_seen", "first_seen",
"last_seen")) == [(7, now + shift(10), now + shift(16))]
assert source_activity.data == {
"destination_id": destination.id,
"fingerprints": [events.keys()[1]],
}
assert source.id != destination.id
assert source.project == destination.project
assert Activity.objects.get(
group_id=destination.id,
type=Activity.UNMERGE_DESTINATION).data == {
"source_id": source.id,
"fingerprints": [events.keys()[1]]
}
source_event_event_ids = map(lambda event: event.event_id,
events.values()[0])
assert set(
UserReport.objects.filter(group_id=source.id).values_list(
"event_id", flat=True)) == set(source_event_event_ids)
assert set(
GroupHash.objects.filter(group_id=source.id).values_list(
"hash", flat=True)) == set([events.keys()[0]])
assert set(
GroupRelease.objects.filter(group_id=source.id).values_list(
"environment", "first_seen", "last_seen")) == set([
(u"production", now + shift(0), now + shift(9))
])
assert set([
(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(
source.project_id, source.id, [production_environment.id])
]) == set([(u"color", 3), (u"environment", 1), (u"sentry:release", 1)])
if settings.SENTRY_TAGSTORE.startswith("sentry.tagstore.v2"):
env_filter = {"_key__environment_id": production_environment.id}
else:
env_filter = {}
assert set([
(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen)
for gtv in GroupTagValue.objects.filter(
project_id=source.project_id, group_id=source.id, **env_filter)
]) == set([
(u"color", u"red", 4, now + shift(0), now + shift(9)),
(u"color", u"green", 3, now + shift(1), now + shift(7)),
(u"color", u"blue", 3, now + shift(2), now + shift(8)),
(u"environment", u"production", 10, now + shift(0),
now + shift(9)),
(u"sentry:release", u"version", 10, now + shift(0),
now + shift(9)),
])
destination_event_event_ids = map(lambda event: event.event_id,
events.values()[1])
assert set(
UserReport.objects.filter(group_id=destination.id).values_list(
"event_id", flat=True)) == set(destination_event_event_ids)
assert set(
GroupHash.objects.filter(group_id=destination.id).values_list(
"hash", flat=True)) == set([events.keys()[1]])
assert set(
GroupRelease.objects.filter(group_id=destination.id).values_list(
"environment", "first_seen", "last_seen")) == set([
(u"production", now + shift(10), now + shift(15))
])
assert set([
(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(
source.project_id, source.id, [production_environment.id])
]) == set([(u"color", 3), (u"environment", 1), (u"sentry:release", 1)])
if settings.SENTRY_TAGSTORE.startswith("sentry.tagstore.v2"):
assert set([
(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen,
gtv.last_seen) for gtv in GroupTagValue.objects.filter(
project_id=destination.project_id,
group_id=destination.id,
**env_filter)
]) == set([
(u"color", u"red", 2, now + shift(12), now + shift(15)),
(u"color", u"green", 2, now + shift(10), now + shift(13)),
(u"color", u"blue", 2, now + shift(11), now + shift(14)),
(u"environment", u"production", 6, now + shift(10),
now + shift(15)),
(u"sentry:release", u"version", 6, now + shift(10),
now + shift(15)),
])
else:
assert set([
(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen,
gtv.last_seen) for gtv in GroupTagValue.objects.filter(
project_id=destination.project_id,
group_id=destination.id,
**env_filter)
]) == set([
(u"color", u"red", 2, now + shift(12), now + shift(15)),
(u"color", u"green", 3, now + shift(10), now + shift(16)),
(u"color", u"blue", 2, now + shift(11), now + shift(14)),
(u"environment", u"production", 6, now + shift(10),
now + shift(15)),
(u"sentry:release", u"version", 6, now + shift(10),
now + shift(15)),
])
rollup_duration = 3600
time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(15),
rollup_duration,
)
environment_time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(15),
rollup_duration,
environment_ids=[production_environment.id],
)
def get_expected_series_values(rollup, events, function=None):
if function is None:
def function(aggregate, event):
return (aggregate if aggregate is not None else 0) + 1
expected = {}
for event in events:
k = float((to_timestamp(event.datetime) // rollup_duration) *
rollup_duration)
expected[k] = function(expected.get(k), event)
return expected
def assert_series_contains(expected, actual, default=0):
actual = dict(actual)
for key, value in expected.items():
assert actual.get(key, 0) == value
for key in set(actual.keys()) - set(expected.keys()):
assert actual.get(key, 0) == default
for series in [time_series, environment_time_series]:
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[0]),
series[source.id],
0,
)
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[1][:-1]),
series[destination.id],
0,
)
time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
environment_time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
environment_id=production_environment.id,
)
def collect_by_user_tag(aggregate, event):
aggregate = aggregate if aggregate is not None else set()
aggregate.add(
get_event_user_from_interface(event.data["user"]).tag_value)
return aggregate
for series in [time_series, environment_time_series]:
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[0], collect_by_user_tag).items()
},
series[source.id],
)
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[1], collect_by_user_tag).items()
},
time_series[destination.id],
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_releases_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_release(group, aggregate, event):
aggregate = aggregate if aggregate is not None else {}
release = event.get_tag("sentry:release")
if not release:
return aggregate
release = GroupRelease.objects.get(
group_id=group.id,
environment=event.data["environment"],
release_id=Release.objects.get(
organization_id=project.organization_id,
version=release).id,
).id
aggregate[release] = aggregate.get(release, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
functools.partial(collect_by_release, source)),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
functools.partial(collect_by_release, destination),
),
time_series[destination.id],
{},
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_environments_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_environment(aggregate, event):
aggregate = aggregate if aggregate is not None else {}
environment = Environment.objects.get(
organization_id=project.organization_id,
name=event.data["environment"]).id
aggregate[environment] = aggregate.get(environment, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[0],
collect_by_environment),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[1],
collect_by_environment),
time_series[destination.id],
{},
)
source_similar_items = features.compare(source)
assert source_similar_items[0] == (
source.id,
{
"exception:message:character-shingles": None,
"exception:stacktrace:application-chunks": None,
"exception:stacktrace:pairs": None,
"message:message:character-shingles": 1.0,
},
)
assert source_similar_items[1][0] == destination.id
assert source_similar_items[1][1][
"message:message:character-shingles"] < 1.0
destination_similar_items = features.compare(destination)
assert destination_similar_items[0] == (
destination.id,
{
"exception:message:character-shingles": None,
"exception:stacktrace:application-chunks": None,
"exception:stacktrace:pairs": None,
"message:message:character-shingles": 1.0,
},
)
assert destination_similar_items[1][0] == source.id
assert destination_similar_items[1][1][
"message:message:character-shingles"] < 1.0
def get(self, request, group):
"""
Retrieve an Issue
`````````````````
Return details on an individual issue. This returns the basic stats for
the issue (title, last seen, first seen), some overall numbers (number
of comments, user reports) as well as the summarized event data.
:pparam string issue_id: the ID of the issue to retrieve.
:auth: required
"""
# TODO(dcramer): handle unauthenticated/public response
# TODO(jess): This can be removed when tagstore v2 is deprecated
use_snuba = request.GET.get('enable_snuba') == '1'
environments = get_environments(request, group.project.organization)
environment_ids = [e.id for e in environments]
if use_snuba:
# WARNING: the rest of this endpoint relies on this serializer
# populating the cache SO don't move this :)
data = serialize(
group, request.user,
GroupSerializerSnuba(environment_ids=environment_ids, ))
else:
# TODO(jess): This is just to ensure we're not breaking the old
# issue page somehow -- non-snuba tagstore versions will raise
# if more than one env is passed
if environments:
environments = environments[:1]
environment_ids = environment_ids[:1]
data = serialize(
group,
request.user,
GroupSerializer(
# Just in case multiple envs are passed, let's make
# sure we're using the same one for all the stats
environment_func=lambda: environments[0]
if environments else None))
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=100)
seen_by = self._get_seen_by(request, group)
first_release = group.get_first_release()
if first_release is not None:
last_release = group.get_last_release()
else:
last_release = None
action_list = self._get_actions(request, group)
if first_release:
first_release = self._get_release_info(request, group,
first_release)
if last_release:
last_release = self._get_release_info(request, group, last_release)
get_range = functools.partial(tsdb.get_range,
environment_ids=environment_ids)
tags = tagstore.get_group_tag_keys(group.project_id,
group.id,
environment_ids,
limit=100)
if not environment_ids:
user_reports = UserReport.objects.filter(group=group)
else:
user_reports = UserReport.objects.filter(
group=group, environment_id__in=environment_ids)
now = timezone.now()
hourly_stats = tsdb.rollup(
get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=1),
), 3600)[group.id]
daily_stats = tsdb.rollup(
get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=30),
), 3600 * 24)[group.id]
participants = list(
User.objects.filter(
groupsubscription__is_active=True,
groupsubscription__group=group,
))
data.update({
'firstRelease':
first_release,
'lastRelease':
last_release,
'activity':
serialize(activity, request.user),
'seenBy':
seen_by,
'participants':
serialize(participants, request.user),
'pluginActions':
action_list,
'pluginIssues':
self._get_available_issue_plugins(request, group),
'pluginContexts':
self._get_context_plugins(request, group),
'userReportCount':
user_reports.count(),
'tags':
sorted(serialize(tags, request.user), key=lambda x: x['name']),
'stats': {
'24h': hourly_stats,
'30d': daily_stats,
}
})
# the current release is the 'latest seen' release within the
# environment even if it hasnt affected this issue
if environments:
try:
current_release = GroupRelease.objects.filter(
group_id=group.id,
environment__in=[env.name for env in environments],
release_id=ReleaseEnvironment.objects.filter(
release_id__in=ReleaseProject.objects.filter(
project_id=group.project_id).values_list(
'release_id', flat=True),
organization_id=group.project.organization_id,
environment_id__in=environment_ids,
).order_by('-first_seen').values_list('release_id',
flat=True)[:1],
)[0]
except IndexError:
current_release = None
data.update({
'currentRelease':
serialize(current_release, request.user,
GroupReleaseWithStatsSerializer())
})
return Response(data)
def get(self, request, group):
"""
Retrieve an Issue
`````````````````
Return details on an individual issue. This returns the basic stats for
the issue (title, last seen, first seen), some overall numbers (number
of comments, user reports) as well as the summarized event data.
:pparam string issue_id: the ID of the issue to retrieve.
:auth: required
"""
# TODO(dcramer): handle unauthenticated/public response
data = serialize(group, request.user)
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=100)
seen_by = self._get_seen_by(request, group)
first_release = group.get_first_release()
if first_release is not None:
last_release = group.get_last_release()
else:
last_release = None
action_list = self._get_actions(request, group)
now = timezone.now()
hourly_stats = tsdb.rollup(
tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=1),
), 3600
)[group.id]
daily_stats = tsdb.rollup(
tsdb.get_range(
model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=30),
), 3600 * 24
)[group.id]
if first_release:
first_release = self._get_release_info(request, group, first_release)
if last_release:
last_release = self._get_release_info(request, group, last_release)
tags = tagstore.get_group_tag_keys(group.id, limit=100)
participants = list(
User.objects.filter(
groupsubscription__is_active=True,
groupsubscription__group=group,
)
)
data.update(
{
'firstRelease': first_release,
'lastRelease': last_release,
'activity': serialize(activity, request.user),
'seenBy': seen_by,
'participants': serialize(participants, request.user),
'pluginActions': action_list,
'pluginIssues': self._get_available_issue_plugins(request, group),
'pluginContexts': self._get_context_plugins(request, group),
'userReportCount': UserReport.objects.filter(group=group).count(),
'tags': sorted(serialize(tags, request.user), key=lambda x: x['name']),
'stats': {
'24h': hourly_stats,
'30d': daily_stats,
}
}
)
return Response(data)
def get(self, request, group):
"""
Retrieve an Issue
`````````````````
Return details on an individual issue. This returns the basic stats for
the issue (title, last seen, first seen), some overall numbers (number
of comments, user reports) as well as the summarized event data.
:pparam string issue_id: the ID of the issue to retrieve.
:auth: required
"""
# TODO(dcramer): handle unauthenticated/public response
organization = group.project.organization
environments = get_environments(request, organization)
environment_ids = [e.id for e in environments]
# WARNING: the rest of this endpoint relies on this serializer
# populating the cache SO don't move this :)
data = serialize(group, request.user,
GroupSerializerSnuba(environment_ids=environment_ids))
# TODO: these probably should be another endpoint
activity = self._get_activity(request, group, num=100)
seen_by = self._get_seen_by(request, group)
first_release = group.get_first_release()
if first_release is not None:
last_release = group.get_last_release()
else:
last_release = None
action_list = self._get_actions(request, group)
if first_release:
first_release = self._get_release_info(request, group,
first_release)
if last_release:
last_release = self._get_release_info(request, group, last_release)
get_range = functools.partial(tsdb.get_range,
environment_ids=environment_ids)
tags = tagstore.get_group_tag_keys(group.project_id,
group.id,
environment_ids,
limit=100)
if not environment_ids:
user_reports = UserReport.objects.filter(group=group)
else:
user_reports = UserReport.objects.filter(
group=group, environment_id__in=environment_ids)
now = timezone.now()
hourly_stats = tsdb.rollup(
get_range(model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=1)),
3600,
)[group.id]
daily_stats = tsdb.rollup(
get_range(model=tsdb.models.group,
keys=[group.id],
end=now,
start=now - timedelta(days=30)),
3600 * 24,
)[group.id]
participants = list(
User.objects.filter(groupsubscription__is_active=True,
groupsubscription__group=group))
data.update({
"firstRelease":
first_release,
"lastRelease":
last_release,
"activity":
serialize(activity, request.user),
"seenBy":
seen_by,
"participants":
serialize(participants, request.user),
"pluginActions":
action_list,
"pluginIssues":
self._get_available_issue_plugins(request, group),
"pluginContexts":
self._get_context_plugins(request, group),
"userReportCount":
user_reports.count(),
"tags":
sorted(serialize(tags, request.user), key=lambda x: x["name"]),
"stats": {
"24h": hourly_stats,
"30d": daily_stats
},
})
# the current release is the 'latest seen' release within the
# environment even if it hasnt affected this issue
if environments:
try:
current_release = GroupRelease.objects.filter(
group_id=group.id,
environment__in=[env.name for env in environments],
release_id=ReleaseEnvironment.objects.filter(
release_id__in=ReleaseProject.objects.filter(
project_id=group.project_id).values_list(
"release_id", flat=True),
organization_id=group.project.organization_id,
environment_id__in=environment_ids,
).order_by("-first_seen").values_list("release_id",
flat=True)[:1],
)[0]
except IndexError:
current_release = None
data.update({
"currentRelease":
serialize(current_release, request.user,
GroupReleaseWithStatsSerializer())
})
return Response(data)
def test_unmerge(self, mock_eventstream):
eventstream_state = object()
mock_eventstream.start_unmerge = Mock(return_value=eventstream_state)
def shift(i):
return timedelta(seconds=1 << i)
now = timezone.now().replace(microsecond=0) - shift(16)
project = self.create_project()
source = self.create_group(project)
sequence = itertools.count(0)
tag_values = itertools.cycle(['red', 'green', 'blue'])
user_values = itertools.cycle([
{
'id': 1
},
{
'id': 2
},
])
for environment in ('production', ''):
EnvironmentProject.objects.create(
environment=Environment.objects.create(
organization_id=project.organization_id,
name=environment,
),
project=project,
)
def create_message_event(template, parameters, environment, release):
i = next(sequence)
event_id = uuid.UUID(fields=(
i,
0x0,
0x1000,
0x80,
0x80,
0x808080808080,
), ).hex
tags = [['color', next(tag_values)]]
if environment:
tags.append(['environment', environment])
if release:
tags.append(['sentry:release', release])
event = Event.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
message='%s' % (id, ),
datetime=now + shift(i),
data={
'environment': environment,
'type': 'default',
'metadata': {
'title': template % parameters,
},
'logentry': {
'message': template,
'params': parameters,
'formatted': template % parameters,
},
'user': next(user_values),
'tags': tags,
},
)
with self.tasks():
Group.objects.add_tags(
source,
Environment.objects.get(
organization_id=project.organization_id,
name=environment),
tags=event.tags,
)
EventMapping.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
date_added=event.datetime,
)
UserReport.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
name='Log Hat',
email='*****@*****.**',
comments='Quack',
)
if release:
Release.get_or_create(
project=project,
version=event.get_tag('sentry:release'),
date_added=event.datetime,
)
features.record([event])
return event
events = OrderedDict()
for event in (create_message_event('This is message #%s.',
i,
environment='production',
release='version')
for i in xrange(10)):
events.setdefault(get_fingerprint(event), []).append(event)
for event in (create_message_event('This is message #%s!',
i,
environment='production',
release='version')
for i in xrange(10, 16)):
events.setdefault(get_fingerprint(event), []).append(event)
event = create_message_event('This is message #%s!',
17,
environment='',
release=None)
events.setdefault(get_fingerprint(event), []).append(event)
assert len(events) == 2
assert sum(map(len, events.values())) == 17
# XXX: This is super contrived considering that it doesn't actually go
# through the event pipeline, but them's the breaks, eh?
for fingerprint in events.keys():
GroupHash.objects.create(
project=project,
group=source,
hash=fingerprint,
)
production_environment = Environment.objects.get(
organization_id=project.organization_id, name='production')
assert set([
(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(
source.project_id, source.id, [production_environment.id])
]) == set([(u'color', 3), (u'environment', 1), (u'sentry:release', 1)])
if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'):
assert set([
(gtv.key, gtv.value, gtv.times_seen,
Environment.objects.get(pk=gtv._key.environment_id).name)
for gtv in GroupTagValue.objects.filter(
project_id=source.project_id,
group_id=source.id,
).exclude(_key__environment_id=0)
]) == set([
('color', 'red', 6, 'production'),
('sentry:release', 'version', 16, 'production'),
('color', 'blue', 5, 'production'),
('color', 'green', 5, 'production'),
('environment', 'production', 16, 'production'),
('color', 'green', 1, ''),
])
else:
assert set([(gtv.key, gtv.value, gtv.times_seen)
for gtv in GroupTagValue.objects.filter(
project_id=source.project_id,
group_id=source.id,
)]) == set([
(u'color', u'red', 6),
(u'color', u'green', 6),
(u'color', u'blue', 5),
(u'environment', u'production', 16),
(u'sentry:release', u'version', 16),
])
assert features.compare(source) == [
(source.id, {
'exception:message:character-shingles': None,
'exception:stacktrace:application-chunks': None,
'exception:stacktrace:pairs': None,
'message:message:character-shingles': 1.0
}),
]
with self.tasks():
unmerge.delay(
source.project_id,
source.id,
None,
[events.keys()[1]],
None,
batch_size=5,
)
assert list(
Group.objects.filter(id=source.id).values_list(
'times_seen',
'first_seen',
'last_seen',
)) == [(
10,
now + shift(0),
now + shift(9),
)]
source_activity = Activity.objects.get(
group_id=source.id,
type=Activity.UNMERGE_SOURCE,
)
destination = Group.objects.get(
id=source_activity.data['destination_id'], )
mock_eventstream.start_unmerge.assert_called_once_with(
source.project_id, [events.keys()[1]], source.id, destination.id)
mock_eventstream.end_unmerge.assert_called_once_with(eventstream_state)
assert list(
Group.objects.filter(id=destination.id).values_list(
'times_seen',
'first_seen',
'last_seen',
)) == [(
7,
now + shift(10),
now + shift(16),
)]
assert source_activity.data == {
'destination_id': destination.id,
'fingerprints': [events.keys()[1]],
}
assert source.id != destination.id
assert source.project == destination.project
assert Activity.objects.get(
group_id=destination.id,
type=Activity.UNMERGE_DESTINATION,
).data == {
'source_id': source.id,
'fingerprints': [events.keys()[1]],
}
source_event_event_ids = map(
lambda event: event.event_id,
events.values()[0],
)
assert source.event_set.count() == 10
assert set(
EventMapping.objects.filter(group_id=source.id, ).values_list(
'event_id', flat=True)) == set(source_event_event_ids)
assert set(
UserReport.objects.filter(group_id=source.id, ).values_list(
'event_id', flat=True)) == set(source_event_event_ids)
assert set(
GroupHash.objects.filter(group_id=source.id, ).values_list(
'hash', flat=True)) == set([events.keys()[0]])
assert set(
GroupRelease.objects.filter(group_id=source.id, ).values_list(
'environment', 'first_seen', 'last_seen')) == set([
(
u'production',
now + shift(0),
now + shift(9),
),
])
assert set([
(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(
source.project_id, source.id, [production_environment.id])
]) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
])
if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'):
env_filter = {'_key__environment_id': production_environment.id}
else:
env_filter = {}
assert set([
(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen)
for gtv in GroupTagValue.objects.filter(
project_id=source.project_id, group_id=source.id, **env_filter)
]) == set([
(
u'color',
u'red',
4,
now + shift(0),
now + shift(9),
),
(
u'color',
u'green',
3,
now + shift(1),
now + shift(7),
),
(
u'color',
u'blue',
3,
now + shift(2),
now + shift(8),
),
(
u'environment',
u'production',
10,
now + shift(0),
now + shift(9),
),
(
u'sentry:release',
u'version',
10,
now + shift(0),
now + shift(9),
),
])
destination_event_event_ids = map(
lambda event: event.event_id,
events.values()[1],
)
assert destination.event_set.count() == 7
assert set(
EventMapping.objects.filter(group_id=destination.id, ).values_list(
'event_id', flat=True)) == set(destination_event_event_ids)
assert set(
UserReport.objects.filter(group_id=destination.id, ).values_list(
'event_id', flat=True)) == set(destination_event_event_ids)
assert set(
GroupHash.objects.filter(group_id=destination.id, ).values_list(
'hash', flat=True)) == set([events.keys()[1]])
assert set(
GroupRelease.objects.filter(group_id=destination.id, ).values_list(
'environment', 'first_seen', 'last_seen')) == set([
(
u'production',
now + shift(10),
now + shift(15),
),
])
assert set([
(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(
source.project_id, source.id, [production_environment.id])
]) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
])
if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'):
assert set([(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen,
gtv.last_seen)
for gtv in GroupTagValue.objects.filter(
project_id=destination.project_id,
group_id=destination.id,
**env_filter)]) == set([
(
u'color',
u'red',
2,
now + shift(12),
now + shift(15),
),
(
u'color',
u'green',
2,
now + shift(10),
now + shift(13),
),
(
u'color',
u'blue',
2,
now + shift(11),
now + shift(14),
),
(
u'environment',
u'production',
6,
now + shift(10),
now + shift(15),
),
(
u'sentry:release',
u'version',
6,
now + shift(10),
now + shift(15),
),
])
else:
assert set([(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen,
gtv.last_seen)
for gtv in GroupTagValue.objects.filter(
project_id=destination.project_id,
group_id=destination.id,
**env_filter)]) == set([
(
u'color',
u'red',
2,
now + shift(12),
now + shift(15),
),
(
u'color',
u'green',
3,
now + shift(10),
now + shift(16),
),
(
u'color',
u'blue',
2,
now + shift(11),
now + shift(14),
),
(
u'environment',
u'production',
6,
now + shift(10),
now + shift(15),
),
(
u'sentry:release',
u'version',
6,
now + shift(10),
now + shift(15),
),
])
rollup_duration = 3600
time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(15),
rollup_duration,
)
environment_time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(15),
rollup_duration,
environment_ids=[production_environment.id],
)
def get_expected_series_values(rollup, events, function=None):
if function is None:
def function(aggregate, event):
return (aggregate if aggregate is not None else 0) + 1
expected = {}
for event in events:
k = float((to_timestamp(event.datetime) // rollup_duration) *
rollup_duration)
expected[k] = function(expected.get(k), event)
return expected
def assert_series_contains(expected, actual, default=0):
actual = dict(actual)
for key, value in expected.items():
assert actual.get(key, 0) == value
for key in set(actual.keys()) - set(expected.keys()):
assert actual.get(key, 0) == default
for series in [time_series, environment_time_series]:
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[0]),
series[source.id],
0,
)
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[1][:-1]),
series[destination.id],
0,
)
time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
environment_time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
environment_id=production_environment.id,
)
def collect_by_user_tag(aggregate, event):
aggregate = aggregate if aggregate is not None else set()
aggregate.add(
get_event_user_from_interface(
event.data['user'], ).tag_value, )
return aggregate
for series in [time_series, environment_time_series]:
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[0],
collect_by_user_tag,
).items()
},
series[source.id],
)
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[1],
collect_by_user_tag,
).items()
},
time_series[destination.id],
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_releases_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_release(group, aggregate, event):
aggregate = aggregate if aggregate is not None else {}
release = event.get_tag('sentry:release')
if not release:
return aggregate
release = GroupRelease.objects.get(
group_id=group.id,
environment=event.data['environment'],
release_id=Release.objects.get(
organization_id=project.organization_id,
version=release,
).id,
).id
aggregate[release] = aggregate.get(release, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
functools.partial(
collect_by_release,
source,
),
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
functools.partial(
collect_by_release,
destination,
),
),
time_series[destination.id],
{},
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_environments_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_environment(aggregate, event):
aggregate = aggregate if aggregate is not None else {}
environment = Environment.objects.get(
organization_id=project.organization_id,
name=event.data['environment'],
).id
aggregate[environment] = aggregate.get(environment, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
collect_by_environment,
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
collect_by_environment,
),
time_series[destination.id],
{},
)
source_similar_items = features.compare(source)
assert source_similar_items[0] == (source.id, {
'exception:message:character-shingles':
None,
'exception:stacktrace:application-chunks':
None,
'exception:stacktrace:pairs':
None,
'message:message:character-shingles':
1.0,
})
assert source_similar_items[1][0] == destination.id
assert source_similar_items[1][1][
'message:message:character-shingles'] < 1.0
destination_similar_items = features.compare(destination)
assert destination_similar_items[0] == (destination.id, {
'exception:message:character-shingles':
None,
'exception:stacktrace:application-chunks':
None,
'exception:stacktrace:pairs':
None,
'message:message:character-shingles':
1.0
})
assert destination_similar_items[1][0] == source.id
assert destination_similar_items[1][1][
'message:message:character-shingles'] < 1.0
