def test_add_tags(self):
event = Group.objects.from_kwargs(1, message='rrr')
group = event.group
environment = self.create_environment()
with self.tasks():
Group.objects.add_tags(
group,
environment,
tags=[
('foo', 'bar'),
('foo', 'baz'),
('biz', 'boz'),
],
)
results = sorted(
tagstore.get_group_tag_values(
group.project_id,
group.id,
environment_id=None,
key='foo',
),
key=lambda x: x.value,
)
assert len(results) == 2
res = results[0]
self.assertEquals(res.value, 'bar')
self.assertEquals(res.times_seen, 1)
res = results[1]
self.assertEquals(res.value, 'baz')
self.assertEquals(res.times_seen, 1)
results = tagstore.get_group_tag_values(
group.project_id,
group.id,
environment_id=None,
key='biz'
)
assert len(results) == 1
res = list(results)[0]
self.assertEquals(res.value, 'boz')
self.assertEquals(res.times_seen, 1)
def test_add_tags(self):
event = Group.objects.from_kwargs(1, message='rrr')
group = event.group
with self.tasks():
Group.objects.add_tags(group, tags=(('foo', 'bar'), ('foo', 'baz'), ('biz', 'boz')))
results = sorted(tagstore.get_group_tag_values(group.id, 'foo'), key=lambda x: x.id)
assert len(results) == 2
res = results[0]
self.assertEquals(res.value, 'bar')
self.assertEquals(res.times_seen, 1)
res = results[1]
self.assertEquals(res.value, 'baz')
self.assertEquals(res.times_seen, 1)
results = sorted(tagstore.get_group_tag_values(group.id, 'biz'), key=lambda x: x.id)
assert len(results) == 1
res = results[0]
self.assertEquals(res.value, 'boz')
self.assertEquals(res.times_seen, 1)
def merge_to(self, project):
from sentry.models import (Group, Event)
if not isinstance(project, Project):
project = Project.objects.get_from_cache(pk=project)
for group in Group.objects.filter(project=self):
try:
other = Group.objects.get(
project=project,
)
except Group.DoesNotExist:
group.update(project=project)
tagstore.update_project_for_group(
group_id=group.id,
old_project_id=self.id,
new_project_id=project.id)
else:
Event.objects.filter(
group_id=group.id,
).update(group_id=other.id)
for obj in tagstore.get_group_tag_values(group_id=group.id):
obj2, created = tagstore.get_or_create_group_tag_value(
project_id=project.id,
group_id=group.id,
key=obj.key,
value=obj.value,
defaults={'times_seen': obj.times_seen}
)
if not created:
obj2.update(times_seen=F('times_seen') + obj.times_seen)
for fv in tagstore.get_tag_values(self.id):
tagstore.get_or_create_tag_value(project_id=project.id, key=fv.key, value=fv.value)
fv.delete()
self.delete()
def test_unmerge(self):
def shift(i):
return timedelta(seconds=1 << i)
now = timezone.now().replace(microsecond=0) - shift(16)
project = self.create_project()
source = self.create_group(project)
sequence = itertools.count(0)
tag_values = itertools.cycle(['red', 'green', 'blue'])
user_values = itertools.cycle([
{
'id': 1
},
{
'id': 2
},
])
EnvironmentProject.objects.create(
environment=Environment.objects.create(
organization_id=project.organization_id,
name='production',
),
project=project,
)
def create_message_event(template, parameters):
i = next(sequence)
event_id = uuid.UUID(
fields=(i, 0x0, 0x1000, 0x80, 0x80, 0x808080808080, ),
).hex
event = Event.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
message='%s' % (id, ),
datetime=now + shift(i),
data={
'environment':
'production',
'type':
'default',
'metadata': {
'title': template % parameters,
},
'sentry.interfaces.Message': {
'message': template,
'params': parameters,
'formatted': template % parameters,
},
'sentry.interfaces.User':
next(user_values),
'tags': [
['color', next(tag_values)],
['environment', 'production'],
['sentry:release', 'version'],
],
},
)
with self.tasks():
Group.objects.add_tags(
source,
tags=event.get_tags(),
)
EventMapping.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
date_added=event.datetime,
)
UserReport.objects.create(
project_id=project.id,
group_id=source.id,
event_id=event_id,
name='Log Hat',
email='*****@*****.**',
comments='Quack',
)
features.record([event])
return event
events = OrderedDict()
for event in (create_message_event('This is message #%s.', i) for i in xrange(10)):
events.setdefault(get_fingerprint(event), []).append(event)
for event in (create_message_event('This is message #%s!', i) for i in xrange(10, 17)):
events.setdefault(get_fingerprint(event), []).append(event)
assert len(events) == 2
assert sum(map(len, events.values())) == 17
# XXX: This is super contrived considering that it doesn't actually go
# through the event pipeline, but them's the breaks, eh?
for fingerprint in events.keys():
GroupHash.objects.create(
project=project,
group=source,
hash=fingerprint,
)
assert set(
[(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(source.id)]
) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1)
])
assert set(
[(gtv.key, gtv.value, gtv.times_seen)
for gtv in tagstore.get_group_tag_values(source.id)]
) == set([
(u'color', u'red', 6),
(u'color', u'green', 6),
(u'color', u'blue', 5),
(u'environment', u'production', 17),
(u'sentry:release', u'version', 17),
])
assert features.compare(source) == [
(source.id, {
'exception:message:character-shingles': None,
'exception:stacktrace:application-chunks': None,
'exception:stacktrace:pairs': None,
'message:message:character-shingles': 1.0
}),
]
with self.tasks():
unmerge.delay(
source.project_id,
source.id,
None,
[events.keys()[1]],
None,
batch_size=5,
)
assert list(
Group.objects.filter(id=source.id).values_list(
'times_seen',
'first_seen',
'last_seen',
)
) == [(10, now + shift(0), now + shift(9), )]
source_activity = Activity.objects.get(
group_id=source.id,
type=Activity.UNMERGE_SOURCE,
)
destination = Group.objects.get(
id=source_activity.data['destination_id'],
)
assert list(
Group.objects.filter(id=destination.id).values_list(
'times_seen',
'first_seen',
'last_seen',
)
) == [(7, now + shift(10), now + shift(16), )]
assert source_activity.data == {
'destination_id': destination.id,
'fingerprints': [events.keys()[1]],
}
assert source.id != destination.id
assert source.project == destination.project
assert Activity.objects.get(
group_id=destination.id,
type=Activity.UNMERGE_DESTINATION,
).data == {
'source_id': source.id,
'fingerprints': [events.keys()[1]],
}
source_event_event_ids = map(
lambda event: event.event_id,
events.values()[0],
)
assert source.event_set.count() == 10
assert set(
EventMapping.objects.filter(
group_id=source.id,
).values_list('event_id', flat=True)
) == set(source_event_event_ids)
assert set(
UserReport.objects.filter(
group_id=source.id,
).values_list('event_id', flat=True)
) == set(source_event_event_ids)
assert set(GroupHash.objects.filter(
group_id=source.id,
).values_list('hash', flat=True)) == set([events.keys()[0]])
assert set(
GroupRelease.objects.filter(
group_id=source.id,
).values_list('environment', 'first_seen', 'last_seen')
) == set([
(u'production', now + shift(0), now + shift(9), ),
])
assert set(
[(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(source.id)]
) == set([
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
])
assert set(
[(gtv.key, gtv.value, gtv.times_seen,
gtv.first_seen, gtv.last_seen) for gtv in tagstore.get_group_tag_values(source.id)]
) == set([
(u'color', u'red', 4, now + shift(0), now + shift(9), ),
(u'color', u'green', 3, now + shift(1), now + shift(7), ),
(u'color', u'blue', 3, now + shift(2), now + shift(8), ),
(u'environment', u'production', 10, now + shift(0), now + shift(9), ),
(u'sentry:release', u'version', 10, now + shift(0), now + shift(9), ),
])
destination_event_event_ids = map(
lambda event: event.event_id,
events.values()[1],
)
assert destination.event_set.count() == 7
assert set(
EventMapping.objects.filter(
group_id=destination.id,
).values_list('event_id', flat=True)
) == set(destination_event_event_ids)
assert set(
UserReport.objects.filter(
group_id=destination.id,
).values_list('event_id', flat=True)
) == set(destination_event_event_ids)
assert set(
GroupHash.objects.filter(
group_id=destination.id,
).values_list('hash', flat=True)
) == set([events.keys()[1]])
assert set(
GroupRelease.objects.filter(
group_id=destination.id,
).values_list('environment', 'first_seen', 'last_seen')
) == set([
(u'production', now + shift(10), now + shift(16), ),
])
assert set([(gtk.key, gtk.values_seen) for gtk in tagstore.get_group_tag_keys(source.id)]
) == set(
[
(u'color', 3),
(u'environment', 1),
(u'sentry:release', 1),
]
)
assert set(
[(gtv.key, gtv.value, gtv.times_seen,
gtv.first_seen, gtv.last_seen) for gtv in tagstore.get_group_tag_values(destination.id)]
) == set([
(u'color', u'red', 2, now + shift(12), now + shift(15), ),
(u'color', u'green', 3, now + shift(10), now + shift(16), ),
(u'color', u'blue', 2, now + shift(11), now + shift(14), ),
(u'environment', u'production', 7, now + shift(10), now + shift(16), ),
(u'sentry:release', u'version', 7, now + shift(10), now + shift(16), ),
])
rollup_duration = 3600
time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def get_expected_series_values(rollup, events, function=None):
if function is None:
def function(aggregate, event):
return (aggregate if aggregate is not None else 0) + 1
expected = {}
for event in events:
k = float((to_timestamp(event.datetime) // rollup_duration) * rollup_duration)
expected[k] = function(expected.get(k), event)
return expected
def assert_series_contains(expected, actual, default=0):
actual = dict(actual)
for key, value in expected.items():
assert actual.get(key, 0) == value
for key in set(actual.keys()) - set(expected.keys()):
assert actual.get(key, 0) == default
assert_series_contains(
get_expected_series_values(rollup_duration, events.values()[0]),
time_series[source.id],
0,
)
assert_series_contains(
get_expected_series_values(rollup_duration, events.values()[1]),
time_series[destination.id],
0,
)
time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_user_tag(aggregate, event):
aggregate = aggregate if aggregate is not None else set()
aggregate.add(
get_event_user_from_interface(
event.data['sentry.interfaces.User'],
).tag_value,
)
return aggregate
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[0],
collect_by_user_tag,
).items()
},
time_series[source.id],
)
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[1],
collect_by_user_tag,
).items()
},
time_series[destination.id],
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_releases_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_release(group, aggregate, event):
aggregate = aggregate if aggregate is not None else {}
release = GroupRelease.objects.get(
group_id=group.id,
environment=event.data['environment'],
release_id=Release.objects.get(
organization_id=project.organization_id,
version=event.get_tag('sentry:release'),
).id,
).id
aggregate[release] = aggregate.get(release, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
functools.partial(
collect_by_release,
source,
),
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
functools.partial(
collect_by_release,
destination,
),
),
time_series[destination.id],
{},
)
time_series = tsdb.get_most_frequent_series(
tsdb.models.frequent_environments_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
now + shift(16),
rollup_duration,
)
def collect_by_environment(aggregate, event):
aggregate = aggregate if aggregate is not None else {}
environment = Environment.objects.get(
organization_id=project.organization_id,
name=event.data['environment'],
).id
aggregate[environment] = aggregate.get(environment, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0],
collect_by_environment,
),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
collect_by_environment,
),
time_series[destination.id],
{},
)
source_similar_items = features.compare(source)
assert source_similar_items[0] == (source.id, {
'exception:message:character-shingles': None,
'exception:stacktrace:application-chunks': None,
'exception:stacktrace:pairs': None,
'message:message:character-shingles': 1.0,
})
assert source_similar_items[1][0] == destination.id
assert source_similar_items[1][1]['message:message:character-shingles'] < 1.0
destination_similar_items = features.compare(destination)
assert destination_similar_items[0] == (
destination.id, {
'exception:message:character-shingles': None,
'exception:stacktrace:application-chunks': None,
'exception:stacktrace:pairs': None,
'message:message:character-shingles': 1.0
}
)
assert destination_similar_items[1][0] == source.id
assert destination_similar_items[1][1]['message:message:character-shingles'] < 1.0
def test_merge_updates_tag_values_seen(self):
project = self.create_project()
target, other = [self.create_group(project) for _ in range(0, 2)]
data = {
'sentry:user': {
'id:1': {
target: 2,
},
'id:2': {
other: 3,
},
'id:3': {
target: 1,
other: 2,
},
},
'key': {
'foo': {
other: 3,
},
},
}
input_group_tag_keys = defaultdict(int) # [(group, key)] = values_seen
input_group_tag_values = defaultdict(int) # [(group, key, value)] = times_seen
output_group_tag_keys = defaultdict(int) # [key] = values_seen
output_group_tag_values = defaultdict(int) # [(key, value)] = times_seen
for key, values in data.items():
output_group_tag_keys[key] = len(values)
for value, groups in values.items():
for group, count in groups.items():
input_group_tag_keys[(group, key)] += 1
input_group_tag_values[(group, key, value)] += count
output_group_tag_values[(key, value)] += count
for ((group, key), values_seen) in input_group_tag_keys.items():
tagstore.create_group_tag_key(
project_id=project.id,
group_id=group.id,
environment_id=self.environment.id,
key=key,
values_seen=values_seen,
)
for ((group, key, value), times_seen) in input_group_tag_values.items():
tagstore.create_group_tag_value(
project_id=project.id,
group_id=group.id,
environment_id=self.environment.id,
key=key,
value=value,
times_seen=times_seen,
)
with self.tasks():
merge_group(other.id, target.id)
assert not Group.objects.filter(id=other.id).exists()
assert len(tagstore.get_group_tag_keys(other.id)) == 0
assert len(tagstore.get_group_tag_values(other.id)) == 0
for key, values_seen in output_group_tag_keys.items():
assert tagstore.get_group_tag_key(target.id, key).values_seen == values_seen
for (key, value), times_seen in output_group_tag_values.items():
assert tagstore.get_group_tag_value(
group_id=target.id,
key=key,
value=value,
).times_seen == times_seen