def get(self, request, group): group_tag_keys = tagstore.get_group_tag_keys(group.id) # O(N) db access data = [] all_top_values = [] for group_tag_key in group_tag_keys: total_values = tagstore.get_group_tag_value_count(group.id, group_tag_key.key) top_values = tagstore.get_top_group_tag_values(group.id, group_tag_key.key, limit=10) all_top_values.extend(top_values) data.append( { 'id': six.text_type(group_tag_key.id), 'key': tagstore.get_standardized_key(group_tag_key.key), 'name': tagstore.get_tag_key_label(group_tag_key.key), 'uniqueValues': group_tag_key.values_seen, 'totalValues': total_values, } ) # Serialize all of the values at once to avoid O(n) serialize/db queries top_values_by_key = defaultdict(list) for value in serialize(all_top_values, request.user): top_values_by_key[value['key']].append(value) for d in data: d['topValues'] = top_values_by_key[d['key']] return Response(data)
def update_tag_value_counts(id_list): instances = tagstore.get_group_tag_keys(id_list) for instance in instances: instance.update( values_seen=GroupTagValue.objects.filter( project_id=instance.project_id, group_id=instance.group_id, key=instance.key, ).count(), )
def attach_metadata(self, objects, request=None): from sentry.templatetags.sentry_plugins import handle_before_events attach_foreignkey(objects, Group.project, ['team']) GroupMeta.objects.populate_cache(objects) if request and objects: handle_before_events(request, objects) if request and request.user.is_authenticated() and objects: bookmarks = set( GroupBookmark.objects.filter( user=request.user, group__in=objects, ).values_list('group_id', flat=True)) seen_groups = dict( GroupSeen.objects.filter( user=request.user, group__in=objects, ).values_list('group_id', 'last_seen')) else: bookmarks = set() seen_groups = {} if objects: end = timezone.now() start = end - timedelta(days=1) historical_data = tsdb.get_range( model=tsdb.models.group, keys=[g.id for g in objects], start=start, end=end, ) else: historical_data = {} user_tagkeys = tagstore.get_group_tag_keys([o.id for o in objects], 'sentry:user') user_counts = {} for user_tagkey in user_tagkeys: user_counts[user_tagkey.group_id] = user_tagkey.values_seen for g in objects: g.is_bookmarked = g.pk in bookmarks g.historical_data = [x[1] for x in historical_data.get(g.id, [])] active_date = g.active_at or g.first_seen g.has_seen = seen_groups.get(g.id, active_date) > active_date g.annotations = [{ 'label': 'users', 'count': user_counts.get(g.id, 0), }]
def get_tags(self): if not hasattr(self, '_tag_cache'): group_tags = [gtk.key for gtk in tagstore.get_group_tag_keys(self.id)] results = [] for key in group_tags: results.append({ 'key': key, 'label': tagstore.get_tag_key_label(key), }) self._tag_cache = sorted(results, key=lambda x: x['label']) return self._tag_cache
def get_tags(self): if not hasattr(self, '_tag_cache'): group_tags = [gtk.key for gtk in tagstore.get_group_tag_keys(self.id)] results = [] for key in group_tags: results.append({ 'key': key, 'label': tagstore.get_tag_key_label(key), }) self._tag_cache = sorted(results, key=lambda x: x['label']) return self._tag_cache
def test_unmerge(self, mock_eventstream): eventstream_state = object() mock_eventstream.start_unmerge = Mock(return_value=eventstream_state) def shift(i): return timedelta(seconds=1 << i) now = timezone.now().replace(microsecond=0) - shift(16) project = self.create_project() source = self.create_group(project) sequence = itertools.count(0) tag_values = itertools.cycle(['red', 'green', 'blue']) user_values = itertools.cycle([ { 'id': 1 }, { 'id': 2 }, ]) for environment in ('production', ''): EnvironmentProject.objects.create( environment=Environment.objects.create( organization_id=project.organization_id, name=environment, ), project=project, ) def create_message_event(template, parameters, environment, release): i = next(sequence) event_id = uuid.UUID( fields=(i, 0x0, 0x1000, 0x80, 0x80, 0x808080808080, ), ).hex tags = [['color', next(tag_values)]] if environment: tags.append(['environment', environment]) if release: tags.append(['sentry:release', release]) event = Event.objects.create( project_id=project.id, group_id=source.id, event_id=event_id, message='%s' % (id, ), datetime=now + shift(i), data={ 'environment': environment, 'type': 'default', 'metadata': { 'title': template % parameters, }, 'logentry': { 'message': template, 'params': parameters, 'formatted': template % parameters, }, 'user': next(user_values), 'tags': tags, }, ) with self.tasks(): Group.objects.add_tags( source, Environment.objects.get( organization_id=project.organization_id, name=environment ), tags=event.get_tags(), ) EventMapping.objects.create( project_id=project.id, group_id=source.id, event_id=event_id, date_added=event.datetime, ) UserReport.objects.create( project_id=project.id, group_id=source.id, event_id=event_id, name='Log Hat', email='*****@*****.**', comments='Quack', ) if release: Release.get_or_create( project=project, version=event.get_tag('sentry:release'), date_added=event.datetime, ) features.record([event]) return event events = OrderedDict() for event in (create_message_event('This is message #%s.', i, environment='production', release='version') for i in xrange(10)): events.setdefault(get_fingerprint(event), []).append(event) for event in (create_message_event('This is message #%s!', i, environment='production', release='version') for i in xrange(10, 16)): events.setdefault(get_fingerprint(event), []).append(event) event = create_message_event('This is message #%s!', 17, environment='', release=None) events.setdefault(get_fingerprint(event), []).append(event) assert len(events) == 2 assert sum(map(len, events.values())) == 17 # XXX: This is super contrived considering that it doesn't actually go # through the event pipeline, but them's the breaks, eh? for fingerprint in events.keys(): GroupHash.objects.create( project=project, group=source, hash=fingerprint, ) production_environment = Environment.objects.get( organization_id=project.organization_id, name='production' ) assert set( [(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(source.project_id, source.id, production_environment.id)] ) == set([ (u'color', 3), (u'environment', 1), (u'sentry:release', 1) ]) if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'): assert set( [(gtv.key, gtv.value, gtv.times_seen, Environment.objects.get(pk=gtv._key.environment_id).name) for gtv in GroupTagValue.objects.filter( project_id=source.project_id, group_id=source.id, ).exclude(_key__environment_id=0)] ) == set([ ('color', 'red', 6, 'production'), ('sentry:release', 'version', 16, 'production'), ('color', 'blue', 5, 'production'), ('color', 'green', 5, 'production'), ('environment', 'production', 16, 'production'), ('color', 'green', 1, ''), ]) else: assert set( [(gtv.key, gtv.value, gtv.times_seen) for gtv in GroupTagValue.objects.filter( project_id=source.project_id, group_id=source.id, )] ) == set([ (u'color', u'red', 6), (u'color', u'green', 6), (u'color', u'blue', 5), (u'environment', u'production', 16), (u'sentry:release', u'version', 16), ]) assert features.compare(source) == [ (source.id, { 'exception:message:character-shingles': None, 'exception:stacktrace:application-chunks': None, 'exception:stacktrace:pairs': None, 'message:message:character-shingles': 1.0 }), ] with self.tasks(): unmerge.delay( source.project_id, source.id, None, [events.keys()[1]], None, batch_size=5, ) assert list( Group.objects.filter(id=source.id).values_list( 'times_seen', 'first_seen', 'last_seen', ) ) == [(10, now + shift(0), now + shift(9), )] source_activity = Activity.objects.get( group_id=source.id, type=Activity.UNMERGE_SOURCE, ) destination = Group.objects.get( id=source_activity.data['destination_id'], ) mock_eventstream.start_unmerge.assert_called_once_with( source.project_id, [events.keys()[1]], source.id, destination.id ) mock_eventstream.end_unmerge.assert_called_once_with(eventstream_state) assert list( Group.objects.filter(id=destination.id).values_list( 'times_seen', 'first_seen', 'last_seen', ) ) == [(7, now + shift(10), now + shift(16), )] assert source_activity.data == { 'destination_id': destination.id, 'fingerprints': [events.keys()[1]], } assert source.id != destination.id assert source.project == destination.project assert Activity.objects.get( group_id=destination.id, type=Activity.UNMERGE_DESTINATION, ).data == { 'source_id': source.id, 'fingerprints': [events.keys()[1]], } source_event_event_ids = map( lambda event: event.event_id, events.values()[0], ) assert source.event_set.count() == 10 assert set( EventMapping.objects.filter( group_id=source.id, ).values_list('event_id', flat=True) ) == set(source_event_event_ids) assert set( UserReport.objects.filter( group_id=source.id, ).values_list('event_id', flat=True) ) == set(source_event_event_ids) assert set(GroupHash.objects.filter( group_id=source.id, ).values_list('hash', flat=True)) == set([events.keys()[0]]) assert set( GroupRelease.objects.filter( group_id=source.id, ).values_list('environment', 'first_seen', 'last_seen') ) == set([ (u'production', now + shift(0), now + shift(9), ), ]) assert set( [(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(source.project_id, source.id, production_environment.id)] ) == set([ (u'color', 3), (u'environment', 1), (u'sentry:release', 1), ]) if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'): env_filter = {'_key__environment_id': production_environment.id} else: env_filter = {} assert set( [(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen) for gtv in GroupTagValue.objects.filter( project_id=source.project_id, group_id=source.id, **env_filter )] ) == set([ (u'color', u'red', 4, now + shift(0), now + shift(9), ), (u'color', u'green', 3, now + shift(1), now + shift(7), ), (u'color', u'blue', 3, now + shift(2), now + shift(8), ), (u'environment', u'production', 10, now + shift(0), now + shift(9), ), (u'sentry:release', u'version', 10, now + shift(0), now + shift(9), ), ]) destination_event_event_ids = map( lambda event: event.event_id, events.values()[1], ) assert destination.event_set.count() == 7 assert set( EventMapping.objects.filter( group_id=destination.id, ).values_list('event_id', flat=True) ) == set(destination_event_event_ids) assert set( UserReport.objects.filter( group_id=destination.id, ).values_list('event_id', flat=True) ) == set(destination_event_event_ids) assert set( GroupHash.objects.filter( group_id=destination.id, ).values_list('hash', flat=True) ) == set([events.keys()[1]]) assert set( GroupRelease.objects.filter( group_id=destination.id, ).values_list('environment', 'first_seen', 'last_seen') ) == set([ (u'production', now + shift(10), now + shift(15), ), ]) assert set([(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(source.project_id, source.id, production_environment.id)] ) == set( [ (u'color', 3), (u'environment', 1), (u'sentry:release', 1), ] ) if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'): assert set( [(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen) for gtv in GroupTagValue.objects.filter( project_id=destination.project_id, group_id=destination.id, **env_filter )] ) == set([ (u'color', u'red', 2, now + shift(12), now + shift(15), ), (u'color', u'green', 2, now + shift(10), now + shift(13), ), (u'color', u'blue', 2, now + shift(11), now + shift(14), ), (u'environment', u'production', 6, now + shift(10), now + shift(15), ), (u'sentry:release', u'version', 6, now + shift(10), now + shift(15), ), ]) else: assert set( [(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen) for gtv in GroupTagValue.objects.filter( project_id=destination.project_id, group_id=destination.id, **env_filter )] ) == set([ (u'color', u'red', 2, now + shift(12), now + shift(15), ), (u'color', u'green', 3, now + shift(10), now + shift(16), ), (u'color', u'blue', 2, now + shift(11), now + shift(14), ), (u'environment', u'production', 6, now + shift(10), now + shift(15), ), (u'sentry:release', u'version', 6, now + shift(10), now + shift(15), ), ]) rollup_duration = 3600 time_series = tsdb.get_range( tsdb.models.group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(15), rollup_duration, ) environment_time_series = tsdb.get_range( tsdb.models.group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(15), rollup_duration, environment_ids=[production_environment.id], ) def get_expected_series_values(rollup, events, function=None): if function is None: def function(aggregate, event): return (aggregate if aggregate is not None else 0) + 1 expected = {} for event in events: k = float((to_timestamp(event.datetime) // rollup_duration) * rollup_duration) expected[k] = function(expected.get(k), event) return expected def assert_series_contains(expected, actual, default=0): actual = dict(actual) for key, value in expected.items(): assert actual.get(key, 0) == value for key in set(actual.keys()) - set(expected.keys()): assert actual.get(key, 0) == default for series in [time_series, environment_time_series]: assert_series_contains( get_expected_series_values(rollup_duration, events.values()[0]), series[source.id], 0, ) assert_series_contains( get_expected_series_values(rollup_duration, events.values()[1][:-1]), series[destination.id], 0, ) time_series = tsdb.get_distinct_counts_series( tsdb.models.users_affected_by_group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(16), rollup_duration, ) environment_time_series = tsdb.get_distinct_counts_series( tsdb.models.users_affected_by_group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(16), rollup_duration, environment_id=production_environment.id, ) def collect_by_user_tag(aggregate, event): aggregate = aggregate if aggregate is not None else set() aggregate.add( get_event_user_from_interface( event.data['user'], ).tag_value, ) return aggregate for series in [time_series, environment_time_series]: assert_series_contains( { timestamp: len(values) for timestamp, values in get_expected_series_values( rollup_duration, events.values()[0], collect_by_user_tag, ).items() }, series[source.id], ) assert_series_contains( { timestamp: len(values) for timestamp, values in get_expected_series_values( rollup_duration, events.values()[1], collect_by_user_tag, ).items() }, time_series[destination.id], ) time_series = tsdb.get_most_frequent_series( tsdb.models.frequent_releases_by_group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(16), rollup_duration, ) def collect_by_release(group, aggregate, event): aggregate = aggregate if aggregate is not None else {} release = event.get_tag('sentry:release') if not release: return aggregate release = GroupRelease.objects.get( group_id=group.id, environment=event.data['environment'], release_id=Release.objects.get( organization_id=project.organization_id, version=release, ).id, ).id aggregate[release] = aggregate.get(release, 0) + 1 return aggregate assert_series_contains( get_expected_series_values( rollup_duration, events.values()[0], functools.partial( collect_by_release, source, ), ), time_series[source.id], {}, ) assert_series_contains( get_expected_series_values( rollup_duration, events.values()[1], functools.partial( collect_by_release, destination, ), ), time_series[destination.id], {}, ) time_series = tsdb.get_most_frequent_series( tsdb.models.frequent_environments_by_group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(16), rollup_duration, ) def collect_by_environment(aggregate, event): aggregate = aggregate if aggregate is not None else {} environment = Environment.objects.get( organization_id=project.organization_id, name=event.data['environment'], ).id aggregate[environment] = aggregate.get(environment, 0) + 1 return aggregate assert_series_contains( get_expected_series_values( rollup_duration, events.values()[0], collect_by_environment, ), time_series[source.id], {}, ) assert_series_contains( get_expected_series_values( rollup_duration, events.values()[1], collect_by_environment, ), time_series[destination.id], {}, ) source_similar_items = features.compare(source) assert source_similar_items[0] == (source.id, { 'exception:message:character-shingles': None, 'exception:stacktrace:application-chunks': None, 'exception:stacktrace:pairs': None, 'message:message:character-shingles': 1.0, }) assert source_similar_items[1][0] == destination.id assert source_similar_items[1][1]['message:message:character-shingles'] < 1.0 destination_similar_items = features.compare(destination) assert destination_similar_items[0] == ( destination.id, { 'exception:message:character-shingles': None, 'exception:stacktrace:application-chunks': None, 'exception:stacktrace:pairs': None, 'message:message:character-shingles': 1.0 } ) assert destination_similar_items[1][0] == source.id assert destination_similar_items[1][1]['message:message:character-shingles'] < 1.0
def test_merge_updates_tag_values_seen(self): project = self.create_project() target, other = [self.create_group(project) for _ in range(0, 2)] data = { 'sentry:user': { 'id:1': { target: 2, }, 'id:2': { other: 3, }, 'id:3': { target: 1, other: 2, }, }, 'key': { 'foo': { other: 3, }, }, } input_group_tag_keys = defaultdict(int) # [(group, key)] = values_seen input_group_tag_values = defaultdict( int) # [(group, key, value)] = times_seen output_group_tag_keys = defaultdict(int) # [key] = values_seen output_group_tag_values = defaultdict( int) # [(key, value)] = times_seen for key, values in data.items(): output_group_tag_keys[key] = len(values) for value, groups in values.items(): for group, count in groups.items(): input_group_tag_keys[(group, key)] += 1 input_group_tag_values[(group, key, value)] += count output_group_tag_values[(key, value)] += count for ((group, key), values_seen) in input_group_tag_keys.items(): tagstore.create_group_tag_key( project_id=project.id, group_id=group.id, environment_id=self.environment.id, key=key, values_seen=values_seen, ) for ((group, key, value), times_seen) in input_group_tag_values.items(): tagstore.create_group_tag_value( project_id=project.id, group_id=group.id, environment_id=self.environment.id, key=key, value=value, times_seen=times_seen, ) with self.tasks(): merge_group(other.id, target.id) assert not Group.objects.filter(id=other.id).exists() assert len( tagstore.get_group_tag_keys( other.project_id, other.id, environment_id=self.environment.id)) == 0 assert len( GroupTagValue.objects.filter( project_id=other.project_id, group_id=other.id, )) == 0 for key, values_seen in output_group_tag_keys.items(): assert tagstore.get_group_tag_key( target.project_id, target.id, environment_id=self.environment.id, key=key).values_seen == values_seen for (key, value), times_seen in output_group_tag_values.items(): assert tagstore.get_group_tag_value( project_id=target.project_id, group_id=target.id, environment_id=self.environment.id, key=key, value=value, ).times_seen == times_seen
def get(self, request: Request, group) -> Response: """ Retrieve an Issue ````````````````` Return details on an individual issue. This returns the basic stats for the issue (title, last seen, first seen), some overall numbers (number of comments, user reports) as well as the summarized event data. :pparam string issue_id: the ID of the issue to retrieve. :auth: required """ from sentry.utils import snuba try: # TODO(dcramer): handle unauthenticated/public response organization = group.project.organization environments = get_environments(request, organization) environment_ids = [e.id for e in environments] expand = request.GET.getlist("expand", []) collapse = request.GET.getlist("collapse", []) # WARNING: the rest of this endpoint relies on this serializer # populating the cache SO don't move this :) data = serialize( group, request.user, GroupSerializerSnuba(environment_ids=environment_ids)) # TODO: these probably should be another endpoint activity = self._get_activity(request, group, num=100) seen_by = self._get_seen_by(request, group) if "release" not in collapse: first_release, last_release = get_first_last_release( request, group) data.update({ "firstRelease": first_release, "lastRelease": last_release, }) get_range = functools.partial(tsdb.get_range, environment_ids=environment_ids) tags = tagstore.get_group_tag_keys(group.project_id, group.id, environment_ids, limit=100) if not environment_ids: user_reports = UserReport.objects.filter(group_id=group.id) else: user_reports = UserReport.objects.filter( group_id=group.id, environment_id__in=environment_ids) now = timezone.now() hourly_stats = tsdb.rollup( get_range(model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=1)), 3600, )[group.id] daily_stats = tsdb.rollup( get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=30), ), 3600 * 24, )[group.id] participants = GroupSubscriptionManager.get_participating_users( group) if "inbox" in expand: inbox_map = get_inbox_details([group]) inbox_reason = inbox_map.get(group.id) data.update({"inbox": inbox_reason}) action_list = self._get_actions(request, group) data.update({ "activity": serialize(activity, request.user), "seenBy": seen_by, "participants": serialize(participants, request.user), "pluginActions": action_list, "pluginIssues": self._get_available_issue_plugins(request, group), "pluginContexts": self._get_context_plugins(request, group), "userReportCount": user_reports.count(), "tags": sorted(serialize(tags, request.user), key=lambda x: x["name"]), "stats": { "24h": hourly_stats, "30d": daily_stats }, }) metrics.incr( "group.update.http_response", sample_rate=1.0, tags={ "status": 200, "detail": "group_details:get:response" }, ) return Response(data) except snuba.RateLimitExceeded: metrics.incr( "group.update.http_response", sample_rate=1.0, tags={ "status": 429, "detail": "group_details:get:snuba.RateLimitExceeded" }, ) raise except Exception: metrics.incr( "group.update.http_response", sample_rate=1.0, tags={ "status": 500, "detail": "group_details:get:Exception" }, ) raise
def get(self, request, group): """ Retrieve an Issue ````````````````` Return details on an individual issue. This returns the basic stats for the issue (title, last seen, first seen), some overall numbers (number of comments, user reports) as well as the summarized event data. :pparam string issue_id: the ID of the issue to retrieve. :auth: required """ # TODO(dcramer): handle unauthenticated/public response data = serialize( group, request.user, GroupSerializer(environment_func=self._get_environment_func( request, group.project.organization_id))) # TODO: these probably should be another endpoint activity = self._get_activity(request, group, num=100) seen_by = self._get_seen_by(request, group) first_release = group.get_first_release() if first_release is not None: last_release = group.get_last_release() else: last_release = None action_list = self._get_actions(request, group) if first_release: first_release = self._get_release_info(request, group, first_release) if last_release: last_release = self._get_release_info(request, group, last_release) try: environment_id = self._get_environment_id_from_request( request, group.project.organization_id) except Environment.DoesNotExist: get_range = lambda model, keys, start, end, **kwargs: \ {k: tsdb.make_series(0, start, end) for k in keys} tags = [] user_reports = UserReport.objects.none() else: get_range = functools.partial(tsdb.get_range, environment_id=environment_id) tags = tagstore.get_group_tag_keys(group.project_id, group.id, environment_id, limit=100) if environment_id is None: user_reports = UserReport.objects.filter(group=group) else: user_reports = UserReport.objects.filter( group=group, environment_id=environment_id) now = timezone.now() hourly_stats = tsdb.rollup( get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=1), ), 3600)[group.id] daily_stats = tsdb.rollup( get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=30), ), 3600 * 24)[group.id] participants = list( User.objects.filter( groupsubscription__is_active=True, groupsubscription__group=group, )) data.update({ 'firstRelease': first_release, 'lastRelease': last_release, 'activity': serialize(activity, request.user), 'seenBy': seen_by, 'participants': serialize(participants, request.user), 'pluginActions': action_list, 'pluginIssues': self._get_available_issue_plugins(request, group), 'pluginContexts': self._get_context_plugins(request, group), 'userReportCount': user_reports.count(), 'tags': sorted(serialize(tags, request.user), key=lambda x: x['name']), 'stats': { '24h': hourly_stats, '30d': daily_stats, } }) # the current release is the 'latest seen' release within the # environment even if it hasnt affected this issue try: environment = self._get_environment_from_request( request, group.project.organization_id, ) except Environment.DoesNotExist: environment = None if environment is not None: try: current_release = GroupRelease.objects.filter( group_id=group.id, environment=environment.name, release_id=ReleaseEnvironment.objects.filter( release_id__in=ReleaseProject.objects.filter( project_id=group.project_id).values_list( 'release_id', flat=True), organization_id=group.project.organization_id, environment_id=environment.id, ).order_by('-first_seen').values_list('release_id', flat=True)[:1], )[0] except IndexError: current_release = None data.update({ 'currentRelease': serialize(current_release, request.user, GroupReleaseWithStatsSerializer()) }) return Response(data)
def get(self, request, group): """ Retrieve an Issue ````````````````` Return details on an individual issue. This returns the basic stats for the issue (title, last seen, first seen), some overall numbers (number of comments, user reports) as well as the summarized event data. :pparam string issue_id: the ID of the issue to retrieve. :auth: required """ # TODO(dcramer): handle unauthenticated/public response data = serialize(group, request.user) # TODO: these probably should be another endpoint activity = self._get_activity(request, group, num=100) seen_by = self._get_seen_by(request, group) first_release = group.get_first_release() if first_release is not None: last_release = group.get_last_release() else: last_release = None action_list = self._get_actions(request, group) now = timezone.now() hourly_stats = tsdb.rollup( tsdb.get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=1), ), 3600)[group.id] daily_stats = tsdb.rollup( tsdb.get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=30), ), 3600 * 24)[group.id] if first_release: first_release = self._get_release_info(request, group, first_release) if last_release: last_release = self._get_release_info(request, group, last_release) try: environment_id = self._get_environment_id_from_request( request, group.project.organization_id) except Environment.DoesNotExist: tags = [] else: tags = tagstore.get_group_tag_keys(group.project_id, group.id, environment_id, limit=100) participants = list( User.objects.filter( groupsubscription__is_active=True, groupsubscription__group=group, )) data.update({ 'firstRelease': first_release, 'lastRelease': last_release, 'activity': serialize(activity, request.user), 'seenBy': seen_by, 'participants': serialize(participants, request.user), 'pluginActions': action_list, 'pluginIssues': self._get_available_issue_plugins(request, group), 'pluginContexts': self._get_context_plugins(request, group), 'userReportCount': UserReport.objects.filter(group=group).count(), 'tags': sorted(serialize(tags, request.user), key=lambda x: x['name']), 'stats': { '24h': hourly_stats, '30d': daily_stats, } }) return Response(data)
def test_merge_updates_tag_values_seen(self): project = self.create_project() target, other = [self.create_group(project) for _ in range(0, 2)] data = { 'sentry:user': { 'id:1': { target: 2, }, 'id:2': { other: 3, }, 'id:3': { target: 1, other: 2, }, }, 'key': { 'foo': { other: 3, }, }, } input_group_tag_keys = defaultdict(int) # [(group, key)] = values_seen input_group_tag_values = defaultdict(int) # [(group, key, value)] = times_seen output_group_tag_keys = defaultdict(int) # [key] = values_seen output_group_tag_values = defaultdict(int) # [(key, value)] = times_seen for key, values in data.items(): output_group_tag_keys[key] = len(values) for value, groups in values.items(): for group, count in groups.items(): input_group_tag_keys[(group, key)] += 1 input_group_tag_values[(group, key, value)] += count output_group_tag_values[(key, value)] += count for ((group, key), values_seen) in input_group_tag_keys.items(): tagstore.create_group_tag_key( project_id=project.id, group_id=group.id, environment_id=self.environment.id, key=key, values_seen=values_seen, ) for ((group, key, value), times_seen) in input_group_tag_values.items(): tagstore.create_group_tag_value( project_id=project.id, group_id=group.id, environment_id=self.environment.id, key=key, value=value, times_seen=times_seen, ) with self.tasks(): merge_group(other.id, target.id) assert not Group.objects.filter(id=other.id).exists() assert len( tagstore.get_group_tag_keys( other.project_id, other.id, environment_id=self.environment.id)) == 0 assert len( GroupTagValue.objects.filter( project_id=other.project_id, group_id=other.id, )) == 0 for key, values_seen in output_group_tag_keys.items(): assert tagstore.get_group_tag_key( target.project_id, target.id, environment_id=self.environment.id, key=key).values_seen == values_seen for (key, value), times_seen in output_group_tag_values.items(): assert tagstore.get_group_tag_value( project_id=target.project_id, group_id=target.id, environment_id=self.environment.id, key=key, value=value, ).times_seen == times_seen
def get(self, request, group): """ Retrieve an Issue ````````````````` Return details on an individual issue. This returns the basic stats for the issue (title, last seen, first seen), some overall numbers (number of comments, user reports) as well as the summarized event data. :pparam string issue_id: the ID of the issue to retrieve. :auth: required """ # TODO(dcramer): handle unauthenticated/public response data = serialize( group, request.user, GroupSerializer( environment_func=self._get_environment_func( request, group.project.organization_id) ) ) # TODO: these probably should be another endpoint activity = self._get_activity(request, group, num=100) seen_by = self._get_seen_by(request, group) first_release = group.get_first_release() if first_release is not None: last_release = group.get_last_release() else: last_release = None action_list = self._get_actions(request, group) if first_release: first_release = self._get_release_info(request, group, first_release) if last_release: last_release = self._get_release_info(request, group, last_release) try: environment_id = self._get_environment_id_from_request( request, group.project.organization_id) except Environment.DoesNotExist: get_range = lambda model, keys, start, end, **kwargs: \ {k: tsdb.make_series(0, start, end) for k in keys} tags = [] user_reports = UserReport.objects.none() else: get_range = functools.partial(tsdb.get_range, environment_ids=environment_id and [environment_id]) tags = tagstore.get_group_tag_keys( group.project_id, group.id, environment_id, limit=100) if environment_id is None: user_reports = UserReport.objects.filter(group=group) else: user_reports = UserReport.objects.filter(group=group, environment_id=environment_id) now = timezone.now() hourly_stats = tsdb.rollup( get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=1), ), 3600 )[group.id] daily_stats = tsdb.rollup( get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=30), ), 3600 * 24 )[group.id] participants = list( User.objects.filter( groupsubscription__is_active=True, groupsubscription__group=group, ) ) data.update( { 'firstRelease': first_release, 'lastRelease': last_release, 'activity': serialize(activity, request.user), 'seenBy': seen_by, 'participants': serialize(participants, request.user), 'pluginActions': action_list, 'pluginIssues': self._get_available_issue_plugins(request, group), 'pluginContexts': self._get_context_plugins(request, group), 'userReportCount': user_reports.count(), 'tags': sorted(serialize(tags, request.user), key=lambda x: x['name']), 'stats': { '24h': hourly_stats, '30d': daily_stats, } } ) # the current release is the 'latest seen' release within the # environment even if it hasnt affected this issue try: environment = self._get_environment_from_request( request, group.project.organization_id, ) except Environment.DoesNotExist: environment = None if environment is not None: try: current_release = GroupRelease.objects.filter( group_id=group.id, environment=environment.name, release_id=ReleaseEnvironment.objects.filter( release_id__in=ReleaseProject.objects.filter(project_id=group.project_id ).values_list('release_id', flat=True), organization_id=group.project.organization_id, environment_id=environment.id, ).order_by('-first_seen').values_list('release_id', flat=True)[:1], )[0] except IndexError: current_release = None data.update({ 'currentRelease': serialize( current_release, request.user, GroupReleaseWithStatsSerializer() ) }) return Response(data)
def get(self, request, group): """ Retrieve an Issue ````````````````` Return details on an individual issue. This returns the basic stats for the issue (title, last seen, first seen), some overall numbers (number of comments, user reports) as well as the summarized event data. :pparam string issue_id: the ID of the issue to retrieve. :auth: required """ # TODO(dcramer): handle unauthenticated/public response # TODO(jess): This can be removed when tagstore v2 is deprecated use_snuba = request.GET.get('enable_snuba') == '1' environments = get_environments(request, group.project.organization) environment_ids = [e.id for e in environments] if use_snuba: # WARNING: the rest of this endpoint relies on this serializer # populating the cache SO don't move this :) data = serialize( group, request.user, GroupSerializerSnuba( environment_ids=environment_ids, ) ) else: # TODO(jess): This is just to ensure we're not breaking the old # issue page somehow -- non-snuba tagstore versions will raise # if more than one env is passed if environments: environments = environments[:1] environment_ids = environment_ids[:1] data = serialize( group, request.user, GroupSerializer( # Just in case multiple envs are passed, let's make # sure we're using the same one for all the stats environment_func=lambda: environments[0] if environments else None ) ) # TODO: these probably should be another endpoint activity = self._get_activity(request, group, num=100) seen_by = self._get_seen_by(request, group) first_release = group.get_first_release() if first_release is not None: last_release = group.get_last_release() else: last_release = None action_list = self._get_actions(request, group) if first_release: first_release = self._get_release_info(request, group, first_release) if last_release: last_release = self._get_release_info(request, group, last_release) get_range = functools.partial(tsdb.get_range, environment_ids=environment_ids) tags = tagstore.get_group_tag_keys( group.project_id, group.id, environment_ids, limit=100) if not environment_ids: user_reports = UserReport.objects.filter(group=group) else: user_reports = UserReport.objects.filter( group=group, environment_id__in=environment_ids ) now = timezone.now() hourly_stats = tsdb.rollup( get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=1), ), 3600 )[group.id] daily_stats = tsdb.rollup( get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=30), ), 3600 * 24 )[group.id] participants = list( User.objects.filter( groupsubscription__is_active=True, groupsubscription__group=group, ) ) data.update( { 'firstRelease': first_release, 'lastRelease': last_release, 'activity': serialize(activity, request.user), 'seenBy': seen_by, 'participants': serialize(participants, request.user), 'pluginActions': action_list, 'pluginIssues': self._get_available_issue_plugins(request, group), 'pluginContexts': self._get_context_plugins(request, group), 'userReportCount': user_reports.count(), 'tags': sorted(serialize(tags, request.user), key=lambda x: x['name']), 'stats': { '24h': hourly_stats, '30d': daily_stats, } } ) # the current release is the 'latest seen' release within the # environment even if it hasnt affected this issue if environments: try: current_release = GroupRelease.objects.filter( group_id=group.id, environment__in=[env.name for env in environments], release_id=ReleaseEnvironment.objects.filter( release_id__in=ReleaseProject.objects.filter(project_id=group.project_id ).values_list('release_id', flat=True), organization_id=group.project.organization_id, environment_id__in=environment_ids, ).order_by('-first_seen').values_list('release_id', flat=True)[:1], )[0] except IndexError: current_release = None data.update({ 'currentRelease': serialize( current_release, request.user, GroupReleaseWithStatsSerializer() ) }) return Response(data)
def test_unmerge(self, mock_eventstream): eventstream_state = object() mock_eventstream.start_unmerge = Mock(return_value=eventstream_state) def shift(i): return timedelta(seconds=1 << i) now = timezone.now() - shift(16) project = self.create_project() source = self.create_group(project) sequence = itertools.count(0) tag_values = itertools.cycle(["red", "green", "blue"]) user_values = itertools.cycle([{"id": 1}, {"id": 2}]) for environment in ("production", ""): EnvironmentProject.objects.create( environment=Environment.objects.create( organization_id=project.organization_id, name=environment), project=project, ) def create_message_event(template, parameters, environment, release): i = next(sequence) event_id = uuid.UUID(fields=(i, 0x0, 0x1000, 0x80, 0x80, 0x808080808080)).hex tags = [["color", next(tag_values)]] if environment: tags.append(["environment", environment]) if release: tags.append(["sentry:release", release]) event = Event.objects.create( project_id=project.id, group_id=source.id, event_id=event_id, message="%s" % (id, ), datetime=now + shift(i), data={ "environment": environment, "type": "default", "metadata": { "title": template % parameters }, "logentry": { "message": template, "params": parameters, "formatted": template % parameters, }, "user": next(user_values), "tags": tags, }, ) with self.tasks(): Group.objects.add_tags( source, Environment.objects.get( organization_id=project.organization_id, name=environment), tags=event.tags, ) UserReport.objects.create( project_id=project.id, group_id=source.id, event_id=event_id, name="Log Hat", email="*****@*****.**", comments="Quack", ) if release: Release.get_or_create( project=project, version=event.get_tag("sentry:release"), date_added=event.datetime, ) features.record([event]) return event events = OrderedDict() for event in (create_message_event("This is message #%s.", i, environment="production", release="version") for i in xrange(10)): events.setdefault(get_fingerprint(event), []).append(event) for event in (create_message_event("This is message #%s!", i, environment="production", release="version") for i in xrange(10, 16)): events.setdefault(get_fingerprint(event), []).append(event) event = create_message_event("This is message #%s!", 17, environment="", release=None) events.setdefault(get_fingerprint(event), []).append(event) assert len(events) == 2 assert sum(map(len, events.values())) == 17 # XXX: This is super contrived considering that it doesn't actually go # through the event pipeline, but them's the breaks, eh? for fingerprint in events.keys(): GroupHash.objects.create(project=project, group=source, hash=fingerprint) production_environment = Environment.objects.get( organization_id=project.organization_id, name="production") assert set([ (gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys( source.project_id, source.id, [production_environment.id]) ]) == set([(u"color", 3), (u"environment", 1), (u"sentry:release", 1)]) if settings.SENTRY_TAGSTORE.startswith("sentry.tagstore.v2"): assert set([( gtv.key, gtv.value, gtv.times_seen, Environment.objects.get(pk=gtv._key.environment_id).name, ) for gtv in GroupTagValue.objects.filter( project_id=source.project_id, group_id=source.id).exclude( _key__environment_id=0)]) == set([ ("color", "red", 6, "production"), ("sentry:release", "version", 16, "production"), ("color", "blue", 5, "production"), ("color", "green", 5, "production"), ("environment", "production", 16, "production"), ("color", "green", 1, ""), ]) else: assert set([(gtv.key, gtv.value, gtv.times_seen) for gtv in GroupTagValue.objects.filter( project_id=source.project_id, group_id=source.id) ]) == set([ (u"color", u"red", 6), (u"color", u"green", 6), (u"color", u"blue", 5), (u"environment", u"production", 16), (u"sentry:release", u"version", 16), ]) assert features.compare(source) == [( source.id, { "exception:message:character-shingles": None, "exception:stacktrace:application-chunks": None, "exception:stacktrace:pairs": None, "message:message:character-shingles": 1.0, }, )] with self.tasks(): unmerge.delay(source.project_id, source.id, None, [events.keys()[1]], None, batch_size=5) assert list( Group.objects.filter(id=source.id).values_list( "times_seen", "first_seen", "last_seen")) == [(10, now + shift(0), now + shift(9))] source_activity = Activity.objects.get(group_id=source.id, type=Activity.UNMERGE_SOURCE) destination = Group.objects.get( id=source_activity.data["destination_id"]) mock_eventstream.start_unmerge.assert_called_once_with( source.project_id, [events.keys()[1]], source.id, destination.id) mock_eventstream.end_unmerge.assert_called_once_with(eventstream_state) assert list( Group.objects.filter(id=destination.id).values_list( "times_seen", "first_seen", "last_seen")) == [(7, now + shift(10), now + shift(16))] assert source_activity.data == { "destination_id": destination.id, "fingerprints": [events.keys()[1]], } assert source.id != destination.id assert source.project == destination.project assert Activity.objects.get( group_id=destination.id, type=Activity.UNMERGE_DESTINATION).data == { "source_id": source.id, "fingerprints": [events.keys()[1]] } source_event_event_ids = map(lambda event: event.event_id, events.values()[0]) assert set( UserReport.objects.filter(group_id=source.id).values_list( "event_id", flat=True)) == set(source_event_event_ids) assert set( GroupHash.objects.filter(group_id=source.id).values_list( "hash", flat=True)) == set([events.keys()[0]]) assert set( GroupRelease.objects.filter(group_id=source.id).values_list( "environment", "first_seen", "last_seen")) == set([ (u"production", now + shift(0), now + shift(9)) ]) assert set([ (gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys( source.project_id, source.id, [production_environment.id]) ]) == set([(u"color", 3), (u"environment", 1), (u"sentry:release", 1)]) if settings.SENTRY_TAGSTORE.startswith("sentry.tagstore.v2"): env_filter = {"_key__environment_id": production_environment.id} else: env_filter = {} assert set([ (gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen) for gtv in GroupTagValue.objects.filter( project_id=source.project_id, group_id=source.id, **env_filter) ]) == set([ (u"color", u"red", 4, now + shift(0), now + shift(9)), (u"color", u"green", 3, now + shift(1), now + shift(7)), (u"color", u"blue", 3, now + shift(2), now + shift(8)), (u"environment", u"production", 10, now + shift(0), now + shift(9)), (u"sentry:release", u"version", 10, now + shift(0), now + shift(9)), ]) destination_event_event_ids = map(lambda event: event.event_id, events.values()[1]) assert set( UserReport.objects.filter(group_id=destination.id).values_list( "event_id", flat=True)) == set(destination_event_event_ids) assert set( GroupHash.objects.filter(group_id=destination.id).values_list( "hash", flat=True)) == set([events.keys()[1]]) assert set( GroupRelease.objects.filter(group_id=destination.id).values_list( "environment", "first_seen", "last_seen")) == set([ (u"production", now + shift(10), now + shift(15)) ]) assert set([ (gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys( source.project_id, source.id, [production_environment.id]) ]) == set([(u"color", 3), (u"environment", 1), (u"sentry:release", 1)]) if settings.SENTRY_TAGSTORE.startswith("sentry.tagstore.v2"): assert set([ (gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen) for gtv in GroupTagValue.objects.filter( project_id=destination.project_id, group_id=destination.id, **env_filter) ]) == set([ (u"color", u"red", 2, now + shift(12), now + shift(15)), (u"color", u"green", 2, now + shift(10), now + shift(13)), (u"color", u"blue", 2, now + shift(11), now + shift(14)), (u"environment", u"production", 6, now + shift(10), now + shift(15)), (u"sentry:release", u"version", 6, now + shift(10), now + shift(15)), ]) else: assert set([ (gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen) for gtv in GroupTagValue.objects.filter( project_id=destination.project_id, group_id=destination.id, **env_filter) ]) == set([ (u"color", u"red", 2, now + shift(12), now + shift(15)), (u"color", u"green", 3, now + shift(10), now + shift(16)), (u"color", u"blue", 2, now + shift(11), now + shift(14)), (u"environment", u"production", 6, now + shift(10), now + shift(15)), (u"sentry:release", u"version", 6, now + shift(10), now + shift(15)), ]) rollup_duration = 3600 time_series = tsdb.get_range( tsdb.models.group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(15), rollup_duration, ) environment_time_series = tsdb.get_range( tsdb.models.group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(15), rollup_duration, environment_ids=[production_environment.id], ) def get_expected_series_values(rollup, events, function=None): if function is None: def function(aggregate, event): return (aggregate if aggregate is not None else 0) + 1 expected = {} for event in events: k = float((to_timestamp(event.datetime) // rollup_duration) * rollup_duration) expected[k] = function(expected.get(k), event) return expected def assert_series_contains(expected, actual, default=0): actual = dict(actual) for key, value in expected.items(): assert actual.get(key, 0) == value for key in set(actual.keys()) - set(expected.keys()): assert actual.get(key, 0) == default for series in [time_series, environment_time_series]: assert_series_contains( get_expected_series_values(rollup_duration, events.values()[0]), series[source.id], 0, ) assert_series_contains( get_expected_series_values(rollup_duration, events.values()[1][:-1]), series[destination.id], 0, ) time_series = tsdb.get_distinct_counts_series( tsdb.models.users_affected_by_group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(16), rollup_duration, ) environment_time_series = tsdb.get_distinct_counts_series( tsdb.models.users_affected_by_group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(16), rollup_duration, environment_id=production_environment.id, ) def collect_by_user_tag(aggregate, event): aggregate = aggregate if aggregate is not None else set() aggregate.add( get_event_user_from_interface(event.data["user"]).tag_value) return aggregate for series in [time_series, environment_time_series]: assert_series_contains( { timestamp: len(values) for timestamp, values in get_expected_series_values( rollup_duration, events.values()[0], collect_by_user_tag).items() }, series[source.id], ) assert_series_contains( { timestamp: len(values) for timestamp, values in get_expected_series_values( rollup_duration, events.values()[1], collect_by_user_tag).items() }, time_series[destination.id], ) time_series = tsdb.get_most_frequent_series( tsdb.models.frequent_releases_by_group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(16), rollup_duration, ) def collect_by_release(group, aggregate, event): aggregate = aggregate if aggregate is not None else {} release = event.get_tag("sentry:release") if not release: return aggregate release = GroupRelease.objects.get( group_id=group.id, environment=event.data["environment"], release_id=Release.objects.get( organization_id=project.organization_id, version=release).id, ).id aggregate[release] = aggregate.get(release, 0) + 1 return aggregate assert_series_contains( get_expected_series_values( rollup_duration, events.values()[0], functools.partial(collect_by_release, source)), time_series[source.id], {}, ) assert_series_contains( get_expected_series_values( rollup_duration, events.values()[1], functools.partial(collect_by_release, destination), ), time_series[destination.id], {}, ) time_series = tsdb.get_most_frequent_series( tsdb.models.frequent_environments_by_group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(16), rollup_duration, ) def collect_by_environment(aggregate, event): aggregate = aggregate if aggregate is not None else {} environment = Environment.objects.get( organization_id=project.organization_id, name=event.data["environment"]).id aggregate[environment] = aggregate.get(environment, 0) + 1 return aggregate assert_series_contains( get_expected_series_values(rollup_duration, events.values()[0], collect_by_environment), time_series[source.id], {}, ) assert_series_contains( get_expected_series_values(rollup_duration, events.values()[1], collect_by_environment), time_series[destination.id], {}, ) source_similar_items = features.compare(source) assert source_similar_items[0] == ( source.id, { "exception:message:character-shingles": None, "exception:stacktrace:application-chunks": None, "exception:stacktrace:pairs": None, "message:message:character-shingles": 1.0, }, ) assert source_similar_items[1][0] == destination.id assert source_similar_items[1][1][ "message:message:character-shingles"] < 1.0 destination_similar_items = features.compare(destination) assert destination_similar_items[0] == ( destination.id, { "exception:message:character-shingles": None, "exception:stacktrace:application-chunks": None, "exception:stacktrace:pairs": None, "message:message:character-shingles": 1.0, }, ) assert destination_similar_items[1][0] == source.id assert destination_similar_items[1][1][ "message:message:character-shingles"] < 1.0
def get(self, request, group): """ Retrieve an Issue ````````````````` Return details on an individual issue. This returns the basic stats for the issue (title, last seen, first seen), some overall numbers (number of comments, user reports) as well as the summarized event data. :pparam string issue_id: the ID of the issue to retrieve. :auth: required """ # TODO(dcramer): handle unauthenticated/public response # TODO(jess): This can be removed when tagstore v2 is deprecated use_snuba = request.GET.get('enable_snuba') == '1' environments = get_environments(request, group.project.organization) environment_ids = [e.id for e in environments] if use_snuba: # WARNING: the rest of this endpoint relies on this serializer # populating the cache SO don't move this :) data = serialize( group, request.user, GroupSerializerSnuba(environment_ids=environment_ids, )) else: # TODO(jess): This is just to ensure we're not breaking the old # issue page somehow -- non-snuba tagstore versions will raise # if more than one env is passed if environments: environments = environments[:1] environment_ids = environment_ids[:1] data = serialize( group, request.user, GroupSerializer( # Just in case multiple envs are passed, let's make # sure we're using the same one for all the stats environment_func=lambda: environments[0] if environments else None)) # TODO: these probably should be another endpoint activity = self._get_activity(request, group, num=100) seen_by = self._get_seen_by(request, group) first_release = group.get_first_release() if first_release is not None: last_release = group.get_last_release() else: last_release = None action_list = self._get_actions(request, group) if first_release: first_release = self._get_release_info(request, group, first_release) if last_release: last_release = self._get_release_info(request, group, last_release) get_range = functools.partial(tsdb.get_range, environment_ids=environment_ids) tags = tagstore.get_group_tag_keys(group.project_id, group.id, environment_ids, limit=100) if not environment_ids: user_reports = UserReport.objects.filter(group=group) else: user_reports = UserReport.objects.filter( group=group, environment_id__in=environment_ids) now = timezone.now() hourly_stats = tsdb.rollup( get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=1), ), 3600)[group.id] daily_stats = tsdb.rollup( get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=30), ), 3600 * 24)[group.id] participants = list( User.objects.filter( groupsubscription__is_active=True, groupsubscription__group=group, )) data.update({ 'firstRelease': first_release, 'lastRelease': last_release, 'activity': serialize(activity, request.user), 'seenBy': seen_by, 'participants': serialize(participants, request.user), 'pluginActions': action_list, 'pluginIssues': self._get_available_issue_plugins(request, group), 'pluginContexts': self._get_context_plugins(request, group), 'userReportCount': user_reports.count(), 'tags': sorted(serialize(tags, request.user), key=lambda x: x['name']), 'stats': { '24h': hourly_stats, '30d': daily_stats, } }) # the current release is the 'latest seen' release within the # environment even if it hasnt affected this issue if environments: try: current_release = GroupRelease.objects.filter( group_id=group.id, environment__in=[env.name for env in environments], release_id=ReleaseEnvironment.objects.filter( release_id__in=ReleaseProject.objects.filter( project_id=group.project_id).values_list( 'release_id', flat=True), organization_id=group.project.organization_id, environment_id__in=environment_ids, ).order_by('-first_seen').values_list('release_id', flat=True)[:1], )[0] except IndexError: current_release = None data.update({ 'currentRelease': serialize(current_release, request.user, GroupReleaseWithStatsSerializer()) }) return Response(data)
def get(self, request, group): """ Retrieve an Issue ````````````````` Return details on an individual issue. This returns the basic stats for the issue (title, last seen, first seen), some overall numbers (number of comments, user reports) as well as the summarized event data. :pparam string issue_id: the ID of the issue to retrieve. :auth: required """ # TODO(dcramer): handle unauthenticated/public response data = serialize(group, request.user) # TODO: these probably should be another endpoint activity = self._get_activity(request, group, num=100) seen_by = self._get_seen_by(request, group) first_release = group.get_first_release() if first_release is not None: last_release = group.get_last_release() else: last_release = None action_list = self._get_actions(request, group) now = timezone.now() hourly_stats = tsdb.rollup( tsdb.get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=1), ), 3600 )[group.id] daily_stats = tsdb.rollup( tsdb.get_range( model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=30), ), 3600 * 24 )[group.id] if first_release: first_release = self._get_release_info(request, group, first_release) if last_release: last_release = self._get_release_info(request, group, last_release) tags = tagstore.get_group_tag_keys(group.id, limit=100) participants = list( User.objects.filter( groupsubscription__is_active=True, groupsubscription__group=group, ) ) data.update( { 'firstRelease': first_release, 'lastRelease': last_release, 'activity': serialize(activity, request.user), 'seenBy': seen_by, 'participants': serialize(participants, request.user), 'pluginActions': action_list, 'pluginIssues': self._get_available_issue_plugins(request, group), 'pluginContexts': self._get_context_plugins(request, group), 'userReportCount': UserReport.objects.filter(group=group).count(), 'tags': sorted(serialize(tags, request.user), key=lambda x: x['name']), 'stats': { '24h': hourly_stats, '30d': daily_stats, } } ) return Response(data)
def get(self, request, group): """ Retrieve an Issue ````````````````` Return details on an individual issue. This returns the basic stats for the issue (title, last seen, first seen), some overall numbers (number of comments, user reports) as well as the summarized event data. :pparam string issue_id: the ID of the issue to retrieve. :auth: required """ # TODO(dcramer): handle unauthenticated/public response organization = group.project.organization environments = get_environments(request, organization) environment_ids = [e.id for e in environments] # WARNING: the rest of this endpoint relies on this serializer # populating the cache SO don't move this :) data = serialize(group, request.user, GroupSerializerSnuba(environment_ids=environment_ids)) # TODO: these probably should be another endpoint activity = self._get_activity(request, group, num=100) seen_by = self._get_seen_by(request, group) first_release = group.get_first_release() if first_release is not None: last_release = group.get_last_release() else: last_release = None action_list = self._get_actions(request, group) if first_release: first_release = self._get_release_info(request, group, first_release) if last_release: last_release = self._get_release_info(request, group, last_release) get_range = functools.partial(tsdb.get_range, environment_ids=environment_ids) tags = tagstore.get_group_tag_keys(group.project_id, group.id, environment_ids, limit=100) if not environment_ids: user_reports = UserReport.objects.filter(group=group) else: user_reports = UserReport.objects.filter( group=group, environment_id__in=environment_ids) now = timezone.now() hourly_stats = tsdb.rollup( get_range(model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=1)), 3600, )[group.id] daily_stats = tsdb.rollup( get_range(model=tsdb.models.group, keys=[group.id], end=now, start=now - timedelta(days=30)), 3600 * 24, )[group.id] participants = list( User.objects.filter(groupsubscription__is_active=True, groupsubscription__group=group)) data.update({ "firstRelease": first_release, "lastRelease": last_release, "activity": serialize(activity, request.user), "seenBy": seen_by, "participants": serialize(participants, request.user), "pluginActions": action_list, "pluginIssues": self._get_available_issue_plugins(request, group), "pluginContexts": self._get_context_plugins(request, group), "userReportCount": user_reports.count(), "tags": sorted(serialize(tags, request.user), key=lambda x: x["name"]), "stats": { "24h": hourly_stats, "30d": daily_stats }, }) # the current release is the 'latest seen' release within the # environment even if it hasnt affected this issue if environments: try: current_release = GroupRelease.objects.filter( group_id=group.id, environment__in=[env.name for env in environments], release_id=ReleaseEnvironment.objects.filter( release_id__in=ReleaseProject.objects.filter( project_id=group.project_id).values_list( "release_id", flat=True), organization_id=group.project.organization_id, environment_id__in=environment_ids, ).order_by("-first_seen").values_list("release_id", flat=True)[:1], )[0] except IndexError: current_release = None data.update({ "currentRelease": serialize(current_release, request.user, GroupReleaseWithStatsSerializer()) }) return Response(data)
def test_unmerge(self, mock_eventstream): eventstream_state = object() mock_eventstream.start_unmerge = Mock(return_value=eventstream_state) def shift(i): return timedelta(seconds=1 << i) now = timezone.now().replace(microsecond=0) - shift(16) project = self.create_project() source = self.create_group(project) sequence = itertools.count(0) tag_values = itertools.cycle(['red', 'green', 'blue']) user_values = itertools.cycle([ { 'id': 1 }, { 'id': 2 }, ]) for environment in ('production', ''): EnvironmentProject.objects.create( environment=Environment.objects.create( organization_id=project.organization_id, name=environment, ), project=project, ) def create_message_event(template, parameters, environment, release): i = next(sequence) event_id = uuid.UUID(fields=( i, 0x0, 0x1000, 0x80, 0x80, 0x808080808080, ), ).hex tags = [['color', next(tag_values)]] if environment: tags.append(['environment', environment]) if release: tags.append(['sentry:release', release]) event = Event.objects.create( project_id=project.id, group_id=source.id, event_id=event_id, message='%s' % (id, ), datetime=now + shift(i), data={ 'environment': environment, 'type': 'default', 'metadata': { 'title': template % parameters, }, 'logentry': { 'message': template, 'params': parameters, 'formatted': template % parameters, }, 'user': next(user_values), 'tags': tags, }, ) with self.tasks(): Group.objects.add_tags( source, Environment.objects.get( organization_id=project.organization_id, name=environment), tags=event.tags, ) EventMapping.objects.create( project_id=project.id, group_id=source.id, event_id=event_id, date_added=event.datetime, ) UserReport.objects.create( project_id=project.id, group_id=source.id, event_id=event_id, name='Log Hat', email='*****@*****.**', comments='Quack', ) if release: Release.get_or_create( project=project, version=event.get_tag('sentry:release'), date_added=event.datetime, ) features.record([event]) return event events = OrderedDict() for event in (create_message_event('This is message #%s.', i, environment='production', release='version') for i in xrange(10)): events.setdefault(get_fingerprint(event), []).append(event) for event in (create_message_event('This is message #%s!', i, environment='production', release='version') for i in xrange(10, 16)): events.setdefault(get_fingerprint(event), []).append(event) event = create_message_event('This is message #%s!', 17, environment='', release=None) events.setdefault(get_fingerprint(event), []).append(event) assert len(events) == 2 assert sum(map(len, events.values())) == 17 # XXX: This is super contrived considering that it doesn't actually go # through the event pipeline, but them's the breaks, eh? for fingerprint in events.keys(): GroupHash.objects.create( project=project, group=source, hash=fingerprint, ) production_environment = Environment.objects.get( organization_id=project.organization_id, name='production') assert set([ (gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys( source.project_id, source.id, [production_environment.id]) ]) == set([(u'color', 3), (u'environment', 1), (u'sentry:release', 1)]) if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'): assert set([ (gtv.key, gtv.value, gtv.times_seen, Environment.objects.get(pk=gtv._key.environment_id).name) for gtv in GroupTagValue.objects.filter( project_id=source.project_id, group_id=source.id, ).exclude(_key__environment_id=0) ]) == set([ ('color', 'red', 6, 'production'), ('sentry:release', 'version', 16, 'production'), ('color', 'blue', 5, 'production'), ('color', 'green', 5, 'production'), ('environment', 'production', 16, 'production'), ('color', 'green', 1, ''), ]) else: assert set([(gtv.key, gtv.value, gtv.times_seen) for gtv in GroupTagValue.objects.filter( project_id=source.project_id, group_id=source.id, )]) == set([ (u'color', u'red', 6), (u'color', u'green', 6), (u'color', u'blue', 5), (u'environment', u'production', 16), (u'sentry:release', u'version', 16), ]) assert features.compare(source) == [ (source.id, { 'exception:message:character-shingles': None, 'exception:stacktrace:application-chunks': None, 'exception:stacktrace:pairs': None, 'message:message:character-shingles': 1.0 }), ] with self.tasks(): unmerge.delay( source.project_id, source.id, None, [events.keys()[1]], None, batch_size=5, ) assert list( Group.objects.filter(id=source.id).values_list( 'times_seen', 'first_seen', 'last_seen', )) == [( 10, now + shift(0), now + shift(9), )] source_activity = Activity.objects.get( group_id=source.id, type=Activity.UNMERGE_SOURCE, ) destination = Group.objects.get( id=source_activity.data['destination_id'], ) mock_eventstream.start_unmerge.assert_called_once_with( source.project_id, [events.keys()[1]], source.id, destination.id) mock_eventstream.end_unmerge.assert_called_once_with(eventstream_state) assert list( Group.objects.filter(id=destination.id).values_list( 'times_seen', 'first_seen', 'last_seen', )) == [( 7, now + shift(10), now + shift(16), )] assert source_activity.data == { 'destination_id': destination.id, 'fingerprints': [events.keys()[1]], } assert source.id != destination.id assert source.project == destination.project assert Activity.objects.get( group_id=destination.id, type=Activity.UNMERGE_DESTINATION, ).data == { 'source_id': source.id, 'fingerprints': [events.keys()[1]], } source_event_event_ids = map( lambda event: event.event_id, events.values()[0], ) assert source.event_set.count() == 10 assert set( EventMapping.objects.filter(group_id=source.id, ).values_list( 'event_id', flat=True)) == set(source_event_event_ids) assert set( UserReport.objects.filter(group_id=source.id, ).values_list( 'event_id', flat=True)) == set(source_event_event_ids) assert set( GroupHash.objects.filter(group_id=source.id, ).values_list( 'hash', flat=True)) == set([events.keys()[0]]) assert set( GroupRelease.objects.filter(group_id=source.id, ).values_list( 'environment', 'first_seen', 'last_seen')) == set([ ( u'production', now + shift(0), now + shift(9), ), ]) assert set([ (gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys( source.project_id, source.id, [production_environment.id]) ]) == set([ (u'color', 3), (u'environment', 1), (u'sentry:release', 1), ]) if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'): env_filter = {'_key__environment_id': production_environment.id} else: env_filter = {} assert set([ (gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen) for gtv in GroupTagValue.objects.filter( project_id=source.project_id, group_id=source.id, **env_filter) ]) == set([ ( u'color', u'red', 4, now + shift(0), now + shift(9), ), ( u'color', u'green', 3, now + shift(1), now + shift(7), ), ( u'color', u'blue', 3, now + shift(2), now + shift(8), ), ( u'environment', u'production', 10, now + shift(0), now + shift(9), ), ( u'sentry:release', u'version', 10, now + shift(0), now + shift(9), ), ]) destination_event_event_ids = map( lambda event: event.event_id, events.values()[1], ) assert destination.event_set.count() == 7 assert set( EventMapping.objects.filter(group_id=destination.id, ).values_list( 'event_id', flat=True)) == set(destination_event_event_ids) assert set( UserReport.objects.filter(group_id=destination.id, ).values_list( 'event_id', flat=True)) == set(destination_event_event_ids) assert set( GroupHash.objects.filter(group_id=destination.id, ).values_list( 'hash', flat=True)) == set([events.keys()[1]]) assert set( GroupRelease.objects.filter(group_id=destination.id, ).values_list( 'environment', 'first_seen', 'last_seen')) == set([ ( u'production', now + shift(10), now + shift(15), ), ]) assert set([ (gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys( source.project_id, source.id, [production_environment.id]) ]) == set([ (u'color', 3), (u'environment', 1), (u'sentry:release', 1), ]) if settings.SENTRY_TAGSTORE.startswith('sentry.tagstore.v2'): assert set([(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen) for gtv in GroupTagValue.objects.filter( project_id=destination.project_id, group_id=destination.id, **env_filter)]) == set([ ( u'color', u'red', 2, now + shift(12), now + shift(15), ), ( u'color', u'green', 2, now + shift(10), now + shift(13), ), ( u'color', u'blue', 2, now + shift(11), now + shift(14), ), ( u'environment', u'production', 6, now + shift(10), now + shift(15), ), ( u'sentry:release', u'version', 6, now + shift(10), now + shift(15), ), ]) else: assert set([(gtv.key, gtv.value, gtv.times_seen, gtv.first_seen, gtv.last_seen) for gtv in GroupTagValue.objects.filter( project_id=destination.project_id, group_id=destination.id, **env_filter)]) == set([ ( u'color', u'red', 2, now + shift(12), now + shift(15), ), ( u'color', u'green', 3, now + shift(10), now + shift(16), ), ( u'color', u'blue', 2, now + shift(11), now + shift(14), ), ( u'environment', u'production', 6, now + shift(10), now + shift(15), ), ( u'sentry:release', u'version', 6, now + shift(10), now + shift(15), ), ]) rollup_duration = 3600 time_series = tsdb.get_range( tsdb.models.group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(15), rollup_duration, ) environment_time_series = tsdb.get_range( tsdb.models.group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(15), rollup_duration, environment_ids=[production_environment.id], ) def get_expected_series_values(rollup, events, function=None): if function is None: def function(aggregate, event): return (aggregate if aggregate is not None else 0) + 1 expected = {} for event in events: k = float((to_timestamp(event.datetime) // rollup_duration) * rollup_duration) expected[k] = function(expected.get(k), event) return expected def assert_series_contains(expected, actual, default=0): actual = dict(actual) for key, value in expected.items(): assert actual.get(key, 0) == value for key in set(actual.keys()) - set(expected.keys()): assert actual.get(key, 0) == default for series in [time_series, environment_time_series]: assert_series_contains( get_expected_series_values(rollup_duration, events.values()[0]), series[source.id], 0, ) assert_series_contains( get_expected_series_values(rollup_duration, events.values()[1][:-1]), series[destination.id], 0, ) time_series = tsdb.get_distinct_counts_series( tsdb.models.users_affected_by_group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(16), rollup_duration, ) environment_time_series = tsdb.get_distinct_counts_series( tsdb.models.users_affected_by_group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(16), rollup_duration, environment_id=production_environment.id, ) def collect_by_user_tag(aggregate, event): aggregate = aggregate if aggregate is not None else set() aggregate.add( get_event_user_from_interface( event.data['user'], ).tag_value, ) return aggregate for series in [time_series, environment_time_series]: assert_series_contains( { timestamp: len(values) for timestamp, values in get_expected_series_values( rollup_duration, events.values()[0], collect_by_user_tag, ).items() }, series[source.id], ) assert_series_contains( { timestamp: len(values) for timestamp, values in get_expected_series_values( rollup_duration, events.values()[1], collect_by_user_tag, ).items() }, time_series[destination.id], ) time_series = tsdb.get_most_frequent_series( tsdb.models.frequent_releases_by_group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(16), rollup_duration, ) def collect_by_release(group, aggregate, event): aggregate = aggregate if aggregate is not None else {} release = event.get_tag('sentry:release') if not release: return aggregate release = GroupRelease.objects.get( group_id=group.id, environment=event.data['environment'], release_id=Release.objects.get( organization_id=project.organization_id, version=release, ).id, ).id aggregate[release] = aggregate.get(release, 0) + 1 return aggregate assert_series_contains( get_expected_series_values( rollup_duration, events.values()[0], functools.partial( collect_by_release, source, ), ), time_series[source.id], {}, ) assert_series_contains( get_expected_series_values( rollup_duration, events.values()[1], functools.partial( collect_by_release, destination, ), ), time_series[destination.id], {}, ) time_series = tsdb.get_most_frequent_series( tsdb.models.frequent_environments_by_group, [source.id, destination.id], now - timedelta(seconds=rollup_duration), now + shift(16), rollup_duration, ) def collect_by_environment(aggregate, event): aggregate = aggregate if aggregate is not None else {} environment = Environment.objects.get( organization_id=project.organization_id, name=event.data['environment'], ).id aggregate[environment] = aggregate.get(environment, 0) + 1 return aggregate assert_series_contains( get_expected_series_values( rollup_duration, events.values()[0], collect_by_environment, ), time_series[source.id], {}, ) assert_series_contains( get_expected_series_values( rollup_duration, events.values()[1], collect_by_environment, ), time_series[destination.id], {}, ) source_similar_items = features.compare(source) assert source_similar_items[0] == (source.id, { 'exception:message:character-shingles': None, 'exception:stacktrace:application-chunks': None, 'exception:stacktrace:pairs': None, 'message:message:character-shingles': 1.0, }) assert source_similar_items[1][0] == destination.id assert source_similar_items[1][1][ 'message:message:character-shingles'] < 1.0 destination_similar_items = features.compare(destination) assert destination_similar_items[0] == (destination.id, { 'exception:message:character-shingles': None, 'exception:stacktrace:application-chunks': None, 'exception:stacktrace:pairs': None, 'message:message:character-shingles': 1.0 }) assert destination_similar_items[1][0] == source.id assert destination_similar_items[1][1][ 'message:message:character-shingles'] < 1.0