def get(self, request, organization, key):
if not TAG_KEY_RE.match(key):
return Response(
{'detail': 'Invalid tag key format for "%s"' % (key, )},
status=400)
try:
filter_params = self.get_filter_params(request, organization)
except OrganizationEventsError as exc:
return Response({'detail': exc.message}, status=400)
except NoProjects:
paginator = SequencePaginator([])
else:
# TODO(jess): update this when snuba tagstore is the primary backend for us
tagstore = SnubaTagStorage()
paginator = tagstore.get_tag_value_paginator_for_projects(
filter_params['project_id'],
filter_params.get('environment'),
key,
filter_params['start'],
filter_params['end'],
query=request.GET.get('query'),
)
return self.paginate(
request=request,
paginator=paginator,
on_results=lambda results: serialize(results, request.user),
)
class TagStorage(TestCase):
def setUp(self):
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
self.proj1env1 = self.create_environment(project=self.proj1, name='test')
self.proj1env2 = self.create_environment(project=self.proj1, name='prod')
self.proj1group1 = self.create_group(self.proj1)
self.proj1group2 = self.create_group(self.proj1)
GroupHash.objects.create(project=self.proj1, group=self.proj1group1, hash='1' * 32)
GroupHash.objects.create(project=self.proj1, group=self.proj1group2, hash='2' * 32)
@responses.activate
def test_get_group_ids_for_search_filter(self):
from sentry.search.base import ANY
tags = {
'foo': 'bar',
'baz': 'quux',
}
with responses.RequestsMock() as rsps:
def snuba_response(request):
body = json.loads(request.body)
assert body['project'] == [self.proj1.id]
assert body['groupby'] == ['issue']
assert body['issues']
assert ['tags[foo]', '=', 'bar'] in body['conditions']
assert ['tags[baz]', '=', 'quux'] in body['conditions']
return (200, {}, json.dumps({
'meta': [{'name': 'issue'}, {'name': 'aggregate'}],
'data': [{'issue': self.proj1group1.id, 'aggregate': 1}],
}))
rsps.add_callback(responses.POST, snuba.SNUBA + '/query', callback=snuba_response)
result = self.ts.get_group_ids_for_search_filter(self.proj1.id, self.proj1env1.id, tags)
assert result == [self.proj1group1.id]
tags = {
'foo': ANY,
}
with responses.RequestsMock() as rsps:
def snuba_response_2(request):
body = json.loads(request.body)
assert body['project'] == [self.proj1.id]
assert body['groupby'] == ['issue']
assert body['issues']
assert ['tags[foo]', 'IS NOT NULL', None] in body['conditions']
return (200, {}, json.dumps({
'meta': [{'name': 'issue'}, {'name': 'aggregate'}],
'data': [{'issue': self.proj1group2.id, 'aggregate': 1}],
}))
rsps.add_callback(responses.POST, snuba.SNUBA + '/query', callback=snuba_response_2)
result = self.ts.get_group_ids_for_search_filter(self.proj1.id, self.proj1env1.id, tags)
assert result == [self.proj1group2.id]
def setUp(self):
super(TagStorageTest, self).setUp()
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
self.proj1env1 = self.create_environment(project=self.proj1, name='test')
self.proj1group1 = self.create_group(self.proj1)
self.proj1group2 = self.create_group(self.proj1)
hash1 = '1' * 32
hash2 = '2' * 32
GroupHash.objects.create(project=self.proj1, group=self.proj1group1, hash=hash1)
GroupHash.objects.create(project=self.proj1, group=self.proj1group2, hash=hash2)
self.now = timezone.now().replace(microsecond=0)
data = json.dumps([{
'event_id': six.text_type(r) * 32,
'primary_hash': hash1,
'project_id': self.proj1.id,
'message': 'message 1',
'platform': 'python',
'datetime': (self.now - timedelta(seconds=r)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': {
'received': calendar.timegm(self.now.timetuple()) - r,
'tags': {
'foo': 'bar',
'baz': 'quux',
'environment': self.proj1env1.name,
'sentry:release': 100 * r,
'sentry:user': "******".format(r),
},
'sentry.interfaces.User': {
'id': "user{}".format(r),
'email': "user{}@sentry.io".format(r)
}
},
} for r in [1, 2]] + [{
'event_id': '3' * 32,
'primary_hash': hash2,
'project_id': self.proj1.id,
'message': 'message 2',
'platform': 'python',
'datetime': (self.now - timedelta(seconds=2)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': {
'received': calendar.timegm(self.now.timetuple()) - 2,
'tags': {
'browser': 'chrome',
'environment': self.proj1env1.name,
'sentry:user': "******",
},
'sentry.interfaces.User': {
'id': "user1"
}
},
}])
assert requests.post(settings.SENTRY_SNUBA + '/tests/insert', data=data).status_code == 200
def setUp(self):
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
self.proj1env1 = self.create_environment(project=self.proj1, name='test')
self.proj1env2 = self.create_environment(project=self.proj1, name='prod')
self.proj1group1 = self.create_group(self.proj1)
self.proj1group2 = self.create_group(self.proj1)
GroupHash.objects.create(project=self.proj1, group=self.proj1group1, hash='1' * 32)
GroupHash.objects.create(project=self.proj1, group=self.proj1group2, hash='2' * 32)
class BaseSemverTest:
KEY = None
def setUp(self):
super().setUp()
self.ts = SnubaTagStorage()
def run_test(self,
query,
expected_versions,
environment=None,
project=None):
if project is None:
project = self.project
assert list(
self.ts.get_tag_value_paginator(
project.id,
environment.id if environment else None,
self.KEY,
query=query,
).get_result(10)) == [
TagValue(
key=self.KEY,
value=v,
times_seen=None,
first_seen=None,
last_seen=None,
) for v in expected_versions
]
def get(self, request, organization):
try:
filter_params = self.get_filter_params(request, organization)
except OrganizationEventsError as exc:
return Response({'detail': exc.message}, status=400)
# TODO(jess): update this when snuba tagstore is the primary backend for us
tagstore = SnubaTagStorage()
results = tagstore.get_tag_keys_for_projects(
filter_params['project_id'],
filter_params.get('environment'),
filter_params['start'],
filter_params['end'],
)
return Response(serialize(results, request.user))
def setUp(self):
super(TagStorageTest, self).setUp()
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
self.proj1env1 = self.create_environment(project=self.proj1, name='test')
self.proj1group1 = self.create_group(self.proj1)
self.proj1group2 = self.create_group(self.proj1)
hash1 = '1' * 32
hash2 = '2' * 32
GroupHash.objects.create(project=self.proj1, group=self.proj1group1, hash=hash1)
GroupHash.objects.create(project=self.proj1, group=self.proj1group2, hash=hash2)
self.now = timezone.now().replace(microsecond=0)
data = json.dumps([{
'event_id': six.text_type(r) * 32,
'primary_hash': hash1,
'project_id': self.proj1.id,
'message': 'message 1',
'platform': 'python',
'datetime': (self.now - timedelta(seconds=r)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': {
'received': calendar.timegm(self.now.timetuple()) - r,
'tags': {
'foo': 'bar',
'baz': 'quux',
'environment': self.proj1env1.name,
'sentry:release': 100 * r,
'sentry:user': u"id:user{}".format(r),
},
'sentry.interfaces.User': {
'id': u"user{}".format(r),
'email': u"user{}@sentry.io".format(r)
}
},
} for r in [1, 2]] + [{
'event_id': '3' * 32,
'primary_hash': hash2,
'project_id': self.proj1.id,
'message': 'message 2',
'platform': 'python',
'datetime': (self.now - timedelta(seconds=2)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': {
'received': calendar.timegm(self.now.timetuple()) - 2,
'tags': {
'browser': 'chrome',
'environment': self.proj1env1.name,
'sentry:user': "******",
},
'sentry.interfaces.User': {
'id': "user1"
}
},
}])
assert requests.post(settings.SENTRY_SNUBA + '/tests/insert', data=data).status_code == 200
class GetTagValuePaginatorForProjectsReleaseStageTest(TestCase, SnubaTestCase):
def setUp(self):
super().setUp()
self.ts = SnubaTagStorage()
def run_test(self,
query,
expected_releases,
environment=None,
project=None):
if project is None:
project = self.project
assert list(
self.ts.get_tag_value_paginator(
project.id,
environment.id if environment else None,
RELEASE_STAGE_ALIAS,
query=query,
).get_result(10)) == [
TagValue(
key=RELEASE_STAGE_ALIAS,
value=r.version,
times_seen=None,
first_seen=None,
last_seen=None,
) for r in expected_releases
]
def test_release_stage(self):
replaced_release = self.create_release(
version="replaced_release",
environments=[self.environment],
adopted=timezone.now(),
unadopted=timezone.now(),
)
adopted_release = self.create_release(version="adopted_release",
environments=[self.environment],
adopted=timezone.now())
not_adopted_release = self.create_release(
version="not_adopted_release", environments=[self.environment])
env_2 = self.create_environment()
project_2 = self.create_project()
self.run_test(ReleaseStages.ADOPTED, [adopted_release],
environment=self.environment)
self.run_test(ReleaseStages.LOW_ADOPTION, [not_adopted_release],
environment=self.environment)
self.run_test(ReleaseStages.REPLACED, [replaced_release],
environment=self.environment)
self.run_test(ReleaseStages.ADOPTED, [], environment=env_2)
self.run_test(ReleaseStages.ADOPTED, [],
project=project_2,
environment=self.environment)
def _get_seen_stats(self, item_list, user):
tagstore = SnubaTagStorage()
project_ids = list(set([item.project_id for item in item_list]))
group_ids = [item.id for item in item_list]
user_counts = tagstore.get_groups_user_counts(
project_ids,
group_ids,
environment_ids=self.environment_ids,
start=self.start,
end=self.end,
)
first_seen = {}
last_seen = {}
times_seen = {}
if not self.environment_ids:
# use issue fields
for item in item_list:
first_seen[item.id] = item.first_seen
last_seen[item.id] = item.last_seen
times_seen[item.id] = item.times_seen
else:
seen_data = tagstore.get_group_seen_values_for_environments(
project_ids,
group_ids,
self.environment_ids,
start=self.start,
end=self.end,
)
first_seen_data = {
ge['group_id']: ge['first_seen__min']
for ge in GroupEnvironment.objects.filter(
group_id__in=[item.id for item in item_list],
environment_id__in=self.environment_ids,
).values('group_id').annotate(Min('first_seen'))
}
for item_id, value in seen_data.items():
first_seen[item_id] = first_seen_data.get(item_id)
last_seen[item_id] = value['last_seen']
times_seen[item_id] = value['times_seen']
attrs = {}
for item in item_list:
attrs[item] = {
'times_seen': times_seen.get(item.id, 0),
'first_seen': first_seen.get(item.id),
'last_seen': last_seen.get(item.id),
'user_count': user_counts.get(item.id, 0),
}
return attrs
def _get_seen_stats(self, item_list, user):
tagstore = SnubaTagStorage()
project_ids = list(set([item.project_id for item in item_list]))
group_ids = [item.id for item in item_list]
user_counts = tagstore.get_groups_user_counts(
project_ids,
group_ids,
environment_ids=self.environment_ids,
)
first_seen = {}
last_seen = {}
times_seen = {}
if self.environment_ids is None:
# use issue fields
for item in item_list:
first_seen[item.id] = item.first_seen
last_seen[item.id] = item.last_seen
times_seen[item.id] = item.times_seen
else:
seen_data = tagstore.get_group_seen_values_for_environments(
project_ids,
group_ids,
self.environment_ids,
)
for item_id, value in seen_data.items():
first_seen[item_id] = value['first_seen']
last_seen[item_id] = value['last_seen']
times_seen[item_id] = value['times_seen']
attrs = {}
for item in item_list:
attrs[item] = {
'times_seen': times_seen.get(item.id, 0),
'first_seen': first_seen.get(item.id),
'last_seen': last_seen.get(item.id),
'user_count': user_counts.get(item.id, 0),
}
return attrs
class TagStorageTest(SnubaTestCase):
def setUp(self):
super(TagStorageTest, self).setUp()
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
self.proj1env1 = self.create_environment(project=self.proj1,
name='test')
self.proj1group1 = self.create_group(self.proj1)
self.proj1group2 = self.create_group(self.proj1)
hash1 = '1' * 32
hash2 = '2' * 32
GroupHash.objects.create(project=self.proj1,
group=self.proj1group1,
hash=hash1)
GroupHash.objects.create(project=self.proj1,
group=self.proj1group2,
hash=hash2)
self.now = timezone.now().replace(microsecond=0)
data = json.dumps([{
'event_id':
six.text_type(r) * 32,
'primary_hash':
hash1,
'project_id':
self.proj1.id,
'message':
'message 1',
'platform':
'python',
'datetime': (self.now - timedelta(seconds=r)
).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': {
'received': calendar.timegm(self.now.timetuple()) - r,
'tags': {
'foo': 'bar',
'baz': 'quux',
'environment': self.proj1env1.name,
'sentry:release': 100 * r,
'sentry:user': "******".format(r),
},
'sentry.interfaces.User': {
'id': "user{}".format(r),
'email': "user{}@sentry.io".format(r)
}
},
} for r in [1, 2]] + [{
'event_id':
'3' * 32,
'primary_hash':
hash2,
'project_id':
self.proj1.id,
'message':
'message 2',
'platform':
'python',
'datetime': (self.now - timedelta(seconds=2)
).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': {
'received': calendar.timegm(self.now.timetuple()) - 2,
'tags': {
'browser': 'chrome',
'environment': self.proj1env1.name,
'sentry:user': "******",
},
'sentry.interfaces.User': {
'id': "user1"
}
},
}])
assert requests.post(settings.SENTRY_SNUBA + '/tests/insert',
data=data).status_code == 200
def test_get_group_tag_keys_and_top_values(self):
# TODO: `release` should be `sentry:release`
result = self.ts.get_group_tag_keys_and_top_values(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
)
tags = [r['key'] for r in result]
assert set(tags) == set(
['foo', 'baz', 'environment', 'release', 'user'])
result.sort(key=lambda r: r['key'])
assert result[0]['key'] == 'baz'
assert result[0]['uniqueValues'] == 1
assert result[0]['totalValues'] == 2
assert result[0]['topValues'][0]['value'] == 'quux'
assert result[3]['key'] == 'release'
assert result[3]['uniqueValues'] == 2
assert result[3]['totalValues'] == 2
top_release_values = result[3]['topValues']
assert len(top_release_values) == 2
assert set(v['value']
for v in top_release_values) == set(['100', '200'])
assert all(v['count'] == 1 for v in top_release_values)
def test_get_top_group_tag_values(self):
resp = self.ts.get_top_group_tag_values(self.proj1.id,
self.proj1group1.id,
self.proj1env1.id, 'foo', 1)
assert len(resp) == 1
assert resp[0].times_seen == 2
assert resp[0].key == 'foo'
assert resp[0].value == 'bar'
assert resp[0].group_id == self.proj1group1.id
def test_get_group_tag_value_count(self):
assert self.ts.get_group_tag_value_count(self.proj1.id,
self.proj1group1.id,
self.proj1env1.id, 'foo') == 2
def test_get_group_tag_key(self):
with pytest.raises(GroupTagKeyNotFound):
self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='notreal',
)
assert self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
).key == 'foo'
keys = {
k.key: k
for k in self.ts.get_group_tag_keys(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
)
}
assert set(keys) == set(
['baz', 'environment', 'foo', 'sentry:release', 'sentry:user'])
for k in keys.values():
if k.key not in set(['sentry:release', 'sentry:user']):
assert k.values_seen == 1, 'expected {!r} to have 1 unique value'.format(
k.key)
else:
assert k.values_seen == 2
def test_get_group_tag_value(self):
with pytest.raises(GroupTagValueNotFound):
self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
value='notreal',
)
assert self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='notreal',
) == set([])
assert list(
self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
))[0].value == 'bar'
assert self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
value='bar',
).value == 'bar'
def test_get_tag_key(self):
with pytest.raises(TagKeyNotFound):
self.ts.get_tag_key(project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key='notreal')
def test_get_tag_value(self):
with pytest.raises(TagValueNotFound):
self.ts.get_tag_value(
project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key='foo',
value='notreal',
)
def test_get_groups_user_counts(self):
assert self.ts.get_groups_user_counts(
project_id=self.proj1.id,
group_ids=[self.proj1group1.id, self.proj1group2.id],
environment_id=self.proj1env1.id) == {
self.proj1group1.id: 2,
self.proj1group2.id: 1,
}
def test_get_releases(self):
assert self.ts.get_first_release(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
) == '200'
assert self.ts.get_first_release(
project_id=self.proj1.id,
group_id=self.proj1group2.id,
) is None
assert self.ts.get_last_release(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
) == '100'
assert self.ts.get_last_release(
project_id=self.proj1.id,
group_id=self.proj1group2.id,
) is None
def test_get_group_ids_for_users(self):
assert set(
self.ts.get_group_ids_for_users(
[self.proj1.id],
[EventUser(project_id=self.proj1.id, ident='user1')])) == set(
[self.proj1group1.id, self.proj1group2.id])
assert set(
self.ts.get_group_ids_for_users(
[self.proj1.id],
[EventUser(project_id=self.proj1.id, ident='user2')])) == set(
[self.proj1group1.id])
def test_get_group_tag_values_for_users(self):
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident='user1')])
assert len(result) == 2
assert set(v.group_id for v in result) == set([
self.proj1group1.id,
self.proj1group2.id,
])
assert set(v.last_seen for v in result) == \
set([self.now - timedelta(seconds=1), self.now - timedelta(seconds=2)])
result.sort(key=lambda x: x.last_seen)
assert result[0].last_seen == self.now - timedelta(seconds=2)
assert result[1].last_seen == self.now - timedelta(seconds=1)
for v in result:
assert v.value == 'user1'
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident='user2')])
assert len(result) == 1
assert result[0].value == 'user2'
assert result[0].last_seen == self.now - timedelta(seconds=2)
# Test that users identified by different means are collected.
# (effectively tests OR conditions in snuba API)
result = self.ts.get_group_tag_values_for_users([
EventUser(project_id=self.proj1.id, email='*****@*****.**'),
EventUser(project_id=self.proj1.id, ident='user2')
])
assert len(result) == 2
result.sort(key=lambda x: x.value)
assert result[0].value == 'user1'
assert result[0].last_seen == self.now - timedelta(seconds=1)
assert result[1].value == 'user2'
assert result[1].last_seen == self.now - timedelta(seconds=2)
def test_get_release_tags(self):
tags = list(self.ts.get_release_tags([self.proj1.id], None, ['100']))
assert len(tags) == 1
one_second_ago = self.now - timedelta(seconds=1)
assert tags[0].last_seen == one_second_ago
assert tags[0].first_seen == one_second_ago
assert tags[0].times_seen == 1
def test_get_group_event_ids(self):
assert set(
self.ts.get_group_event_ids(self.proj1.id, self.proj1group1.id,
self.proj1env1.id, {
'foo': 'bar',
})) == set(["1" * 32, "2" * 32])
assert set(
self.ts.get_group_event_ids(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
{
'foo': 'bar', # OR
'release': '200'
})) == set(["1" * 32, "2" * 32])
assert set(
self.ts.get_group_event_ids(
self.proj1.id, self.proj1group2.id, self.proj1env1.id,
{'browser': 'chrome'})) == set(["3" * 32])
assert set(
self.ts.get_group_event_ids(self.proj1.id, self.proj1group2.id,
self.proj1env1.id,
{'browser': 'ie'})) == set([])
class TagStorage(TestCase):
def setUp(self):
assert requests.post(snuba.SNUBA + '/tests/drop').status_code == 200
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
self.proj1env1 = self.create_environment(project=self.proj1,
name='test')
self.proj1group1 = self.create_group(self.proj1)
self.proj1group2 = self.create_group(self.proj1)
hash1 = '1' * 32
hash2 = '2' * 32
GroupHash.objects.create(project=self.proj1,
group=self.proj1group1,
hash=hash1)
GroupHash.objects.create(project=self.proj1,
group=self.proj1group2,
hash=hash2)
self.now = datetime.utcnow().replace(microsecond=0)
data = json.dumps([{
'event_id':
six.text_type(r) * 32,
'primary_hash':
hash1,
'project_id':
self.proj1.id,
'message':
'message 1',
'platform':
'python',
'datetime': (self.now - timedelta(seconds=r)
).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': {
'received': calendar.timegm(self.now.timetuple()) - r,
'tags': {
'foo': 'bar',
'baz': 'quux',
'environment': self.proj1env1.name,
'sentry:release': 100 * r,
},
'sentry.interfaces.User': {
'id': "user{}".format(r),
'email': "user{}@sentry.io".format(r)
}
},
} for r in range(1, 3)] + [{
'event_id':
'3' * 32,
'primary_hash':
hash2,
'project_id':
self.proj1.id,
'message':
'message 2',
'platform':
'python',
'datetime': (self.now - timedelta(seconds=r)
).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': {
'received': calendar.timegm(self.now.timetuple()) - r,
'tags': {
'browser': 'chrome',
'environment': self.proj1env1.name,
},
'sentry.interfaces.User': {
'id': "user1"
}
},
}])
assert requests.post(snuba.SNUBA + '/tests/insert',
data=data).status_code == 200
@responses.activate
def test_get_group_ids_for_search_filter(self):
from sentry.search.base import ANY
tags = {
'foo': 'bar',
'baz': 'quux',
}
with responses.RequestsMock() as rsps:
def snuba_response(request):
body = json.loads(request.body)
assert body['project'] == [self.proj1.id]
assert body['groupby'] == ['issue']
assert body['issues']
assert ['tags[foo]', '=', 'bar'] in body['conditions']
assert ['tags[baz]', '=', 'quux'] in body['conditions']
return (200, {},
json.dumps({
'meta': [{
'name': 'issue'
}, {
'name': 'aggregate'
}],
'data': [{
'issue': self.proj1group1.id,
'aggregate': 1
}],
}))
rsps.add_callback(responses.POST,
snuba.SNUBA + '/query',
callback=snuba_response)
result = self.ts.get_group_ids_for_search_filter(
self.proj1.id, self.proj1env1.id, tags)
assert result == [self.proj1group1.id]
tags = {
'foo': ANY,
}
with responses.RequestsMock() as rsps:
def snuba_response_2(request):
body = json.loads(request.body)
assert body['project'] == [self.proj1.id]
assert body['groupby'] == ['issue']
assert body['issues']
assert ['tags[foo]', 'IS NOT NULL', None] in body['conditions']
return (200, {},
json.dumps({
'meta': [{
'name': 'issue'
}, {
'name': 'aggregate'
}],
'data': [{
'issue': self.proj1group2.id,
'aggregate': 1
}],
}))
rsps.add_callback(responses.POST,
snuba.SNUBA + '/query',
callback=snuba_response_2)
result = self.ts.get_group_ids_for_search_filter(
self.proj1.id, self.proj1env1.id, tags)
assert result == [self.proj1group2.id]
def test_get_group_tag_keys_and_top_values(self):
result = self.ts.get_group_tag_keys_and_top_values(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
)
result.sort(key=lambda r: r['id'])
assert result[0]['key'] == 'baz'
assert result[1]['key'] == 'foo'
assert result[0]['uniqueValues'] == 1
assert result[0]['totalValues'] == 2
assert result[0]['topValues'][0]['value'] == 'quux'
assert result[1]['topValues'][0]['value'] == 'bar'
def test_get_top_group_tag_values(self):
resp = self.ts.get_top_group_tag_values(self.proj1.id,
self.proj1group1.id,
self.proj1env1.id, 'foo', 1)
assert len(resp) == 1
assert resp[0].times_seen == 2
assert resp[0].key == 'foo'
assert resp[0].value == 'bar'
assert resp[0].group_id == self.proj1group1.id
def test_get_group_tag_value_count(self):
assert self.ts.get_group_tag_value_count(self.proj1.id,
self.proj1group1.id,
self.proj1env1.id, 'foo') == 2
def test_get_group_tag_key(self):
with pytest.raises(GroupTagKeyNotFound):
self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='notreal',
)
assert self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
).key == 'foo'
keys = self.ts.get_group_tag_keys(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
)
keys.sort(key=lambda x: x.key)
assert len(keys) == 2
assert keys[0].key == 'baz'
assert keys[0].times_seen == 2
assert keys[1].key == 'foo'
assert keys[1].times_seen == 2
def test_get_group_tag_value(self):
with pytest.raises(GroupTagValueNotFound):
self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
value='notreal',
)
assert self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='notreal',
) == []
assert self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
)[0].value == 'bar'
assert self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
value='bar',
).value == 'bar'
def test_get_tag_key(self):
with pytest.raises(TagKeyNotFound):
self.ts.get_tag_key(project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key='notreal')
def test_get_tag_value(self):
with pytest.raises(TagValueNotFound):
self.ts.get_tag_value(
project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key='foo',
value='notreal',
)
def test_get_groups_user_counts(self):
assert self.ts.get_groups_user_counts(
project_id=self.proj1.id,
group_ids=[self.proj1group1.id, self.proj1group2.id],
environment_id=self.proj1env1.id) == {
self.proj1group1.id: 2,
self.proj1group2.id: 1,
}
def test_get_releases(self):
assert self.ts.get_first_release(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
) == '200'
assert self.ts.get_first_release(
project_id=self.proj1.id,
group_id=self.proj1group2.id,
) is None
assert self.ts.get_last_release(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
) == '100'
assert self.ts.get_last_release(
project_id=self.proj1.id,
group_id=self.proj1group2.id,
) is None
def test_get_group_ids_for_users(self):
assert set(
self.ts.get_group_ids_for_users(
[self.proj1.id],
[EventUser(project_id=self.proj1.id, ident='user1')])) == set(
[self.proj1group1.id, self.proj1group2.id])
assert set(
self.ts.get_group_ids_for_users(
[self.proj1.id],
[EventUser(project_id=self.proj1.id, ident='user2')])) == set(
[self.proj1group1.id])
def test_get_group_tag_values_for_users(self):
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident='user1')])
one_second_ago = (
self.now -
timedelta(seconds=1)).strftime('%Y-%m-%dT%H:%M:%S+00:00')
assert len(result) == 1
assert result[0].value == 'user1'
assert result[0].last_seen == one_second_ago
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident='user2')])
two_seconds_ago = (
self.now -
timedelta(seconds=2)).strftime('%Y-%m-%dT%H:%M:%S+00:00')
assert len(result) == 1
assert result[0].value == 'user2'
assert result[0].last_seen == two_seconds_ago
# Test that users identified by different means are collected.
# (effectively tests OR conditions in snuba API)
result = self.ts.get_group_tag_values_for_users([
EventUser(project_id=self.proj1.id, email='*****@*****.**'),
EventUser(project_id=self.proj1.id, ident='user2')
])
two_seconds_ago = (
self.now -
timedelta(seconds=2)).strftime('%Y-%m-%dT%H:%M:%S+00:00')
assert len(result) == 2
result.sort(key=lambda x: x.value)
assert result[0].value == 'user1'
assert result[0].last_seen == one_second_ago
assert result[1].value == 'user2'
assert result[1].last_seen == two_seconds_ago
def test_get_release_tags(self):
tags = self.ts.get_release_tags([self.proj1.id], None, ['100'])
assert len(tags) == 1
one_second_ago = (
self.now -
timedelta(seconds=1)).strftime('%Y-%m-%dT%H:%M:%S+00:00')
assert tags[0].last_seen == one_second_ago
assert tags[0].first_seen == one_second_ago
assert tags[0].times_seen == 1
def test_get_group_event_ids(self):
assert sorted(
self.ts.get_group_event_ids(self.proj1.id, self.proj1group1.id,
self.proj1env1.id, {
'foo': 'bar',
})) == ["1" * 32, "2" * 32]
assert sorted(
self.ts.get_group_event_ids(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
{
'foo': 'bar', # OR
'release': '200'
})) == ["1" * 32, "2" * 32]
assert self.ts.get_group_event_ids(
self.proj1.id, self.proj1group2.id, self.proj1env1.id,
{'browser': 'chrome'}) == ["3" * 32]
assert self.ts.get_group_event_ids(self.proj1.id, self.proj1group2.id,
self.proj1env1.id,
{'browser': 'ie'}) == []
class TagStorageTest(TestCase, SnubaTestCase):
def setUp(self):
super(TagStorageTest, self).setUp()
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
self.proj1env1 = self.create_environment(project=self.proj1, name="test")
self.proj1env2 = self.create_environment(project=self.proj1, name="test2")
self.proj1group1 = self.create_group(self.proj1)
self.proj1group2 = self.create_group(self.proj1)
hash1 = "1" * 32
hash2 = "2" * 32
GroupHash.objects.create(project=self.proj1, group=self.proj1group1, hash=hash1)
GroupHash.objects.create(project=self.proj1, group=self.proj1group2, hash=hash2)
self.now = timezone.now().replace(microsecond=0)
data = json.dumps(
[
{
"event_id": six.text_type(r) * 32,
"primary_hash": hash1,
"group_id": self.proj1group1.id,
"project_id": self.proj1.id,
"message": "message 1",
"platform": "python",
"datetime": (self.now - timedelta(seconds=r)).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
"data": {
"received": calendar.timegm(self.now.timetuple()) - r,
"tags": {
"foo": "bar",
"baz": "quux",
"environment": self.proj1env1.name,
"sentry:release": 100 * r,
"sentry:user": u"id:user{}".format(r),
},
"user": {"id": u"user{}".format(r), "email": u"user{}@sentry.io".format(r)},
"exception": {"values": [{"stacktrace": {"frames": [{"lineno": 29}]}}]},
},
}
for r in [1, 2]
]
+ [
{
"event_id": "3" * 32,
"primary_hash": hash2,
"group_id": self.proj1group2.id,
"project_id": self.proj1.id,
"message": "message 2",
"platform": "python",
"datetime": (self.now - timedelta(seconds=2)).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
"data": {
"received": calendar.timegm(self.now.timetuple()) - 2,
"tags": {
"browser": "chrome",
"environment": self.proj1env1.name,
"sentry:user": "******",
},
"user": {"id": "user1"},
},
}
]
+ [
{
"event_id": "4" * 32,
"primary_hash": hash2,
"group_id": self.proj1group1.id,
"project_id": self.proj1.id,
"message": "message 2",
"platform": "python",
"datetime": (self.now - timedelta(seconds=2)).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
"data": {
"received": calendar.timegm(self.now.timetuple()) - 2,
"tags": {"foo": "bar", "environment": self.proj1env2.name},
"user": {"id": "user1"},
},
}
]
)
assert (
requests.post(settings.SENTRY_SNUBA + "/tests/events/insert", data=data).status_code
== 200
)
def test_get_group_tag_keys_and_top_values(self):
result = list(
self.ts.get_group_tag_keys_and_top_values(
self.proj1.id, self.proj1group1.id, [self.proj1env1.id]
)
)
tags = [r.key for r in result]
assert set(tags) == set(["foo", "baz", "environment", "sentry:release", "sentry:user"])
result.sort(key=lambda r: r.key)
assert result[0].key == "baz"
assert result[0].top_values[0].value == "quux"
assert result[0].count == 2
assert result[3].key == "sentry:release"
assert result[3].count == 2
top_release_values = result[3].top_values
assert len(top_release_values) == 2
assert set(v.value for v in top_release_values) == set(["100", "200"])
assert all(v.times_seen == 1 for v in top_release_values)
# Now with only a specific set of keys,
result = list(
self.ts.get_group_tag_keys_and_top_values(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
keys=["environment", "sentry:release"],
)
)
tags = [r.key for r in result]
assert set(tags) == set(["environment", "sentry:release"])
result.sort(key=lambda r: r.key)
assert result[0].key == "environment"
assert result[0].top_values[0].value == "test"
assert result[1].key == "sentry:release"
top_release_values = result[1].top_values
assert len(top_release_values) == 2
assert set(v.value for v in top_release_values) == set(["100", "200"])
assert all(v.times_seen == 1 for v in top_release_values)
def test_get_top_group_tag_values(self):
resp = self.ts.get_top_group_tag_values(
self.proj1.id, self.proj1group1.id, self.proj1env1.id, "foo", 1
)
assert len(resp) == 1
assert resp[0].times_seen == 2
assert resp[0].key == "foo"
assert resp[0].value == "bar"
assert resp[0].group_id == self.proj1group1.id
def test_get_group_tag_value_count(self):
assert (
self.ts.get_group_tag_value_count(
self.proj1.id, self.proj1group1.id, self.proj1env1.id, "foo"
)
== 2
)
def test_get_tag_keys(self):
expected_keys = set(
["baz", "browser", "environment", "foo", "sentry:release", "sentry:user"]
)
keys = {
k.key: k
for k in self.ts.get_tag_keys(
project_id=self.proj1.id, environment_id=self.proj1env1.id
)
}
assert set(keys) == expected_keys
keys = {
k.key: k
for k in self.ts.get_tag_keys(
project_id=self.proj1.id, environment_id=self.proj1env1.id, include_values_seen=True
)
}
assert set(keys) == expected_keys
def test_get_group_tag_key(self):
with pytest.raises(GroupTagKeyNotFound):
self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="notreal",
)
assert (
self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="foo",
).key
== "foo"
)
keys = {
k.key: k
for k in self.ts.get_group_tag_keys(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_ids=[self.proj1env1.id],
)
}
assert set(keys) == set(["baz", "environment", "foo", "sentry:release", "sentry:user"])
def test_get_group_tag_value(self):
with pytest.raises(GroupTagValueNotFound):
self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="foo",
value="notreal",
)
assert self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="notreal",
) == set([])
assert (
list(
self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="foo",
)
)[0].value
== "bar"
)
assert (
self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="foo",
value="bar",
).value
== "bar"
)
def test_get_tag_key(self):
with pytest.raises(TagKeyNotFound):
self.ts.get_tag_key(
project_id=self.proj1.id, environment_id=self.proj1env1.id, key="notreal"
)
def test_get_tag_value(self):
with pytest.raises(TagValueNotFound):
self.ts.get_tag_value(
project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key="foo",
value="notreal",
)
def test_get_tag_value_label(self):
assert self.ts.get_tag_value_label("foo", "notreal") == "notreal"
assert self.ts.get_tag_value_label("sentry:user", None) is None
assert self.ts.get_tag_value_label("sentry:user", "id:stuff") == "stuff"
assert self.ts.get_tag_value_label("sentry:user", "email:stuff") == "stuff"
assert self.ts.get_tag_value_label("sentry:user", "username:stuff") == "stuff"
assert self.ts.get_tag_value_label("sentry:user", "ip:stuff") == "stuff"
def test_get_groups_user_counts(self):
assert self.ts.get_groups_user_counts(
project_ids=[self.proj1.id],
group_ids=[self.proj1group1.id, self.proj1group2.id],
environment_ids=[self.proj1env1.id],
) == {self.proj1group1.id: 2, self.proj1group2.id: 1}
# test filtering by date range where there shouldn't be results
assert (
self.ts.get_groups_user_counts(
project_ids=[self.proj1.id],
group_ids=[self.proj1group1.id, self.proj1group2.id],
environment_ids=[self.proj1env1.id],
start=self.now - timedelta(days=5),
end=self.now - timedelta(days=4),
)
== {}
)
def test_get_releases(self):
assert (
self.ts.get_first_release(project_id=self.proj1.id, group_id=self.proj1group1.id)
== "200"
)
assert (
self.ts.get_first_release(project_id=self.proj1.id, group_id=self.proj1group2.id)
is None
)
assert (
self.ts.get_last_release(project_id=self.proj1.id, group_id=self.proj1group1.id)
== "100"
)
assert (
self.ts.get_last_release(project_id=self.proj1.id, group_id=self.proj1group2.id) is None
)
def test_get_group_ids_for_users(self):
assert self.ts.get_group_ids_for_users(
[self.proj1.id], [EventUser(project_id=self.proj1.id, ident="user1")]
) == set([self.proj1group1.id, self.proj1group2.id])
assert self.ts.get_group_ids_for_users(
[self.proj1.id], [EventUser(project_id=self.proj1.id, ident="user2")]
) == set([self.proj1group1.id])
def test_get_group_tag_values_for_users(self):
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident="user1")]
)
assert len(result) == 2
assert set(v.group_id for v in result) == set([self.proj1group1.id, self.proj1group2.id])
assert set(v.last_seen for v in result) == set(
[self.now - timedelta(seconds=1), self.now - timedelta(seconds=2)]
)
result.sort(key=lambda x: x.last_seen)
assert result[0].last_seen == self.now - timedelta(seconds=2)
assert result[1].last_seen == self.now - timedelta(seconds=1)
for v in result:
assert v.value == "user1"
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident="user2")]
)
assert len(result) == 1
assert result[0].value == "user2"
assert result[0].last_seen == self.now - timedelta(seconds=2)
def test_get_release_tags(self):
tags = list(self.ts.get_release_tags([self.proj1.id], None, ["100"]))
assert len(tags) == 1
one_second_ago = self.now - timedelta(seconds=1)
assert tags[0].last_seen == one_second_ago
assert tags[0].first_seen == one_second_ago
assert tags[0].times_seen == 1
assert tags[0].key == "sentry:release"
def test_get_group_event_filter(self):
assert self.ts.get_group_event_filter(
self.proj1.id, self.proj1group1.id, [self.proj1env1.id], {"foo": "bar"}, None, None
) == {"event_id__in": set(["1" * 32, "2" * 32])}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
{"foo": "bar"},
(self.now - timedelta(seconds=1)),
None,
) == {"event_id__in": set(["1" * 32])}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
{"foo": "bar"},
None,
(self.now - timedelta(seconds=1)),
) == {"event_id__in": set(["2" * 32])}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id, self.proj1env2.id],
{"foo": "bar"},
None,
None,
) == {"event_id__in": set(["1" * 32, "2" * 32, "4" * 32])}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
{"foo": "bar", "sentry:release": "200"}, # AND
None,
None,
) == {"event_id__in": set(["2" * 32])}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group2.id,
[self.proj1env1.id],
{"browser": "chrome"},
None,
None,
) == {"event_id__in": set(["3" * 32])}
assert (
self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group2.id,
[self.proj1env1.id],
{"browser": "ie"},
None,
None,
)
is None
)
def test_get_tag_value_paginator(self):
from sentry.tagstore.types import TagValue
assert list(
self.ts.get_tag_value_paginator(
self.proj1.id, self.proj1env1.id, "sentry:user"
).get_result(10)
) == [
TagValue(
key="sentry:user",
value="id:user1",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1),
),
TagValue(
key="sentry:user",
value="id:user2",
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2),
),
]
assert list(
self.ts.get_tag_value_paginator(
self.proj1.id, self.proj1env1.id, "sentry:user", query="user1"
).get_result(10)
) == [
TagValue(
key="sentry:user",
value="id:user1",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1),
)
]
def test_get_tag_value_paginator_with_dates(self):
from sentry.tagstore.types import TagValue
day_ago = self.now - timedelta(days=1)
two_days_ago = self.now - timedelta(days=2)
assert list(
self.ts.get_tag_value_paginator(
self.proj1.id, self.proj1env1.id, "sentry:user", start=day_ago, end=self.now
).get_result(10)
) == [
TagValue(
key="sentry:user",
value="id:user1",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1),
),
TagValue(
key="sentry:user",
value="id:user2",
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2),
),
]
day_ago = self.now - timedelta(days=1)
assert (
list(
self.ts.get_tag_value_paginator(
self.proj1.id, self.proj1env1.id, "sentry:user", start=two_days_ago, end=day_ago
).get_result(10)
)
== []
)
def test_numeric_tag_value_paginator(self):
from sentry.tagstore.types import TagValue
assert list(
self.ts.get_tag_value_paginator(
self.proj1.id, self.proj1env1.id, "stack.lineno"
).get_result(10)
) == [
TagValue(
key="stack.lineno",
value="29",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1),
)
]
assert list(
self.ts.get_tag_value_paginator(
self.proj1.id, self.proj1env1.id, "stack.lineno", query="30"
).get_result(10)
) == [
TagValue(
key="stack.lineno",
value="29",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1),
)
]
def test_get_group_tag_value_iter(self):
from sentry.tagstore.types import GroupTagValue
assert list(
self.ts.get_group_tag_value_iter(
self.proj1.id, self.proj1group1.id, self.proj1env1.id, "sentry:user"
)
) == [
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user1",
times_seen=1,
first_seen=self.now - timedelta(seconds=1),
last_seen=self.now - timedelta(seconds=1),
),
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user2",
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2),
),
]
def test_get_group_tag_value_paginator(self):
from sentry.tagstore.types import GroupTagValue
assert list(
self.ts.get_group_tag_value_paginator(
self.proj1.id, self.proj1group1.id, self.proj1env1.id, "sentry:user"
).get_result(10)
) == [
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user1",
times_seen=1,
first_seen=self.now - timedelta(seconds=1),
last_seen=self.now - timedelta(seconds=1),
),
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user2",
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2),
),
]
def test_get_group_seen_values_for_environments(self):
assert self.ts.get_group_seen_values_for_environments(
[self.proj1.id], [self.proj1group1.id], [self.proj1env1.id]
) == {
self.proj1group1.id: {
"first_seen": self.now - timedelta(seconds=2),
"last_seen": self.now - timedelta(seconds=1),
"times_seen": 2,
}
}
# test where there should be no results because of time filters
assert (
self.ts.get_group_seen_values_for_environments(
[self.proj1.id],
[self.proj1group1.id],
[self.proj1env1.id],
start=self.now - timedelta(hours=5),
end=self.now - timedelta(hours=4),
)
== {}
)
def test_cache_suffix_time(self):
starting_key = cache_suffix_time(self.now, 0)
finishing_key = cache_suffix_time(self.now + timedelta(seconds=300), 0)
assert starting_key != finishing_key
def test_cache_suffix_hour_edges(self):
""" a suffix should still behave correctly around the end of the hour
At a duration of 10 only one key between 0-10 should flip on the hour, the other 9
should flip at different times.
"""
before = datetime(2019, 9, 5, 17, 59, 59)
on_hour = datetime(2019, 9, 5, 18, 0, 0)
changed_on_hour = 0
# Check multiple keyhashes so that this test doesn't depend on implementation
for key_hash in range(10):
before_key = cache_suffix_time(before, key_hash, duration=10)
on_key = cache_suffix_time(on_hour, key_hash, duration=10)
if before_key != on_key:
changed_on_hour += 1
assert changed_on_hour == 1
def test_cache_suffix_day_edges(self):
""" a suffix should still behave correctly around the end of a day
This test is nearly identical to test_cache_suffix_hour_edges, but is to confirm that date changes don't
cause a different behaviour
"""
before = datetime(2019, 9, 5, 23, 59, 59)
next_day = datetime(2019, 9, 6, 0, 0, 0)
changed_on_hour = 0
for key_hash in range(10):
before_key = cache_suffix_time(before, key_hash, duration=10)
next_key = cache_suffix_time(next_day, key_hash, duration=10)
if before_key != next_key:
changed_on_hour += 1
assert changed_on_hour == 1
def test_cache_suffix_time_matches_duration(self):
""" The number of seconds between keys changing should match duration """
previous_key = cache_suffix_time(self.now, 0, duration=10)
changes = []
for i in range(21):
current_time = self.now + timedelta(seconds=i)
current_key = cache_suffix_time(current_time, 0, duration=10)
if current_key != previous_key:
changes.append(current_time)
previous_key = current_key
assert len(changes) == 2
assert (changes[1] - changes[0]).total_seconds() == 10
def test_cache_suffix_time_jitter(self):
""" Different key hashes should change keys at different times
While starting_key and other_key might begin as the same values they should change at different times
"""
starting_key = cache_suffix_time(self.now, 0, duration=10)
for i in range(11):
current_key = cache_suffix_time(self.now + timedelta(seconds=i), 0, duration=10)
if current_key != starting_key:
break
other_key = cache_suffix_time(self.now, 5, duration=10)
for j in range(11):
current_key = cache_suffix_time(self.now + timedelta(seconds=j), 5, duration=10)
if current_key != other_key:
break
assert i != j
def setUp(self):
super().setUp()
self.ts = SnubaTagStorage()
class TagStorageTest(TestCase, SnubaTestCase):
def setUp(self):
super(TagStorageTest, self).setUp()
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
self.proj1env1 = self.create_environment(project=self.proj1,
name="test")
self.proj1env2 = self.create_environment(project=self.proj1,
name="test2")
self.proj1group1 = self.create_group(self.proj1)
self.proj1group2 = self.create_group(self.proj1)
hash1 = "1" * 32
hash2 = "2" * 32
GroupHash.objects.create(project=self.proj1,
group=self.proj1group1,
hash=hash1)
GroupHash.objects.create(project=self.proj1,
group=self.proj1group2,
hash=hash2)
self.now = timezone.now().replace(microsecond=0)
data = json.dumps([{
"event_id":
six.text_type(r) * 32,
"primary_hash":
hash1,
"group_id":
self.proj1group1.id,
"project_id":
self.proj1.id,
"message":
"message 1",
"platform":
"python",
"datetime": (self.now - timedelta(seconds=r)
).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
"data": {
"received": calendar.timegm(self.now.timetuple()) - r,
"tags": {
"foo": "bar",
"baz": "quux",
"environment": self.proj1env1.name,
"sentry:release": 100 * r,
"sentry:user": u"id:user{}".format(r),
},
"user": {
"id": u"user{}".format(r),
"email": u"user{}@sentry.io".format(r)
},
},
} for r in [1, 2]] + [{
"event_id":
"3" * 32,
"primary_hash":
hash2,
"group_id":
self.proj1group2.id,
"project_id":
self.proj1.id,
"message":
"message 2",
"platform":
"python",
"datetime": (self.now - timedelta(seconds=2)
).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
"data": {
"received": calendar.timegm(self.now.timetuple()) - 2,
"tags": {
"browser": "chrome",
"environment": self.proj1env1.name,
"sentry:user": "******",
},
"user": {
"id": "user1"
},
},
}] + [{
"event_id":
"4" * 32,
"primary_hash":
hash2,
"group_id":
self.proj1group1.id,
"project_id":
self.proj1.id,
"message":
"message 2",
"platform":
"python",
"datetime": (self.now - timedelta(seconds=2)
).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
"data": {
"received": calendar.timegm(self.now.timetuple()) - 2,
"tags": {
"foo": "bar",
"environment": self.proj1env2.name
},
"user": {
"id": "user1"
},
},
}])
assert (requests.post(settings.SENTRY_SNUBA + "/tests/events/insert",
data=data).status_code == 200)
def test_get_group_tag_keys_and_top_values(self):
result = list(
self.ts.get_group_tag_keys_and_top_values(self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id]))
tags = [r.key for r in result]
assert set(tags) == set(
["foo", "baz", "environment", "sentry:release", "sentry:user"])
result.sort(key=lambda r: r.key)
assert result[0].key == "baz"
assert result[0].top_values[0].value == "quux"
assert result[0].count == 2
assert result[3].key == "sentry:release"
assert result[3].count == 2
top_release_values = result[3].top_values
assert len(top_release_values) == 2
assert set(v.value for v in top_release_values) == set(["100", "200"])
assert all(v.times_seen == 1 for v in top_release_values)
# Now with only a specific set of keys,
result = list(
self.ts.get_group_tag_keys_and_top_values(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
keys=["environment", "sentry:release"],
))
tags = [r.key for r in result]
assert set(tags) == set(["environment", "sentry:release"])
result.sort(key=lambda r: r.key)
assert result[0].key == "environment"
assert result[0].top_values[0].value == "test"
assert result[1].key == "sentry:release"
top_release_values = result[1].top_values
assert len(top_release_values) == 2
assert set(v.value for v in top_release_values) == set(["100", "200"])
assert all(v.times_seen == 1 for v in top_release_values)
def test_get_top_group_tag_values(self):
resp = self.ts.get_top_group_tag_values(self.proj1.id,
self.proj1group1.id,
self.proj1env1.id, "foo", 1)
assert len(resp) == 1
assert resp[0].times_seen == 2
assert resp[0].key == "foo"
assert resp[0].value == "bar"
assert resp[0].group_id == self.proj1group1.id
def test_get_group_tag_value_count(self):
assert (self.ts.get_group_tag_value_count(self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
"foo") == 2)
def test_get_tag_keys(self):
expected_keys = set([
"baz", "browser", "environment", "foo", "sentry:release",
"sentry:user"
])
keys = {
k.key: k
for k in self.ts.get_tag_keys(project_id=self.proj1.id,
environment_id=self.proj1env1.id)
}
assert set(keys) == expected_keys
keys = {
k.key: k
for k in self.ts.get_tag_keys(project_id=self.proj1.id,
environment_id=self.proj1env1.id,
include_values_seen=True)
}
assert set(keys) == expected_keys
def test_get_group_tag_key(self):
with pytest.raises(GroupTagKeyNotFound):
self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="notreal",
)
assert (self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="foo",
).key == "foo")
keys = {
k.key: k
for k in self.ts.get_group_tag_keys(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_ids=[self.proj1env1.id],
)
}
assert set(keys) == set(
["baz", "environment", "foo", "sentry:release", "sentry:user"])
def test_get_group_tag_value(self):
with pytest.raises(GroupTagValueNotFound):
self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="foo",
value="notreal",
)
assert self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="notreal",
) == set([])
assert (list(
self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="foo",
))[0].value == "bar")
assert (self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="foo",
value="bar",
).value == "bar")
def test_get_tag_key(self):
with pytest.raises(TagKeyNotFound):
self.ts.get_tag_key(project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key="notreal")
def test_get_tag_value(self):
with pytest.raises(TagValueNotFound):
self.ts.get_tag_value(
project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key="foo",
value="notreal",
)
def test_get_groups_user_counts(self):
assert self.ts.get_groups_user_counts(
project_ids=[self.proj1.id],
group_ids=[self.proj1group1.id, self.proj1group2.id],
environment_ids=[self.proj1env1.id],
) == {
self.proj1group1.id: 2,
self.proj1group2.id: 1
}
# test filtering by date range where there shouldn't be results
assert (self.ts.get_groups_user_counts(
project_ids=[self.proj1.id],
group_ids=[self.proj1group1.id, self.proj1group2.id],
environment_ids=[self.proj1env1.id],
start=self.now - timedelta(days=5),
end=self.now - timedelta(days=4),
) == {})
def test_get_releases(self):
assert (self.ts.get_first_release(
project_id=self.proj1.id, group_id=self.proj1group1.id) == "200")
assert (self.ts.get_first_release(
project_id=self.proj1.id, group_id=self.proj1group2.id) is None)
assert (self.ts.get_last_release(
project_id=self.proj1.id, group_id=self.proj1group1.id) == "100")
assert (self.ts.get_last_release(project_id=self.proj1.id,
group_id=self.proj1group2.id) is None)
def test_get_group_ids_for_users(self):
assert self.ts.get_group_ids_for_users(
[self.proj1.id],
[EventUser(project_id=self.proj1.id, ident="user1")]) == set(
[self.proj1group1.id, self.proj1group2.id])
assert self.ts.get_group_ids_for_users(
[self.proj1.id],
[EventUser(project_id=self.proj1.id, ident="user2")]) == set(
[self.proj1group1.id])
def test_get_group_tag_values_for_users(self):
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident="user1")])
assert len(result) == 2
assert set(v.group_id for v in result) == set(
[self.proj1group1.id, self.proj1group2.id])
assert set(v.last_seen for v in result) == set(
[self.now - timedelta(seconds=1), self.now - timedelta(seconds=2)])
result.sort(key=lambda x: x.last_seen)
assert result[0].last_seen == self.now - timedelta(seconds=2)
assert result[1].last_seen == self.now - timedelta(seconds=1)
for v in result:
assert v.value == "user1"
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident="user2")])
assert len(result) == 1
assert result[0].value == "user2"
assert result[0].last_seen == self.now - timedelta(seconds=2)
def test_get_release_tags(self):
tags = list(self.ts.get_release_tags([self.proj1.id], None, ["100"]))
assert len(tags) == 1
one_second_ago = self.now - timedelta(seconds=1)
assert tags[0].last_seen == one_second_ago
assert tags[0].first_seen == one_second_ago
assert tags[0].times_seen == 1
assert tags[0].key == "sentry:release"
def test_get_group_event_filter(self):
assert self.ts.get_group_event_filter(self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
{"foo": "bar"}, None, None) == {
"event_id__in":
set(["1" * 32, "2" * 32])
}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
{"foo": "bar"},
(self.now - timedelta(seconds=1)),
None,
) == {
"event_id__in": set(["1" * 32])
}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
{"foo": "bar"},
None,
(self.now - timedelta(seconds=1)),
) == {
"event_id__in": set(["2" * 32])
}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id, self.proj1env2.id],
{"foo": "bar"},
None,
None,
) == {
"event_id__in": set(["1" * 32, "2" * 32, "4" * 32])
}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
{
"foo": "bar",
"sentry:release": "200"
}, # AND
None,
None,
) == {
"event_id__in": set(["2" * 32])
}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group2.id,
[self.proj1env1.id],
{"browser": "chrome"},
None,
None,
) == {
"event_id__in": set(["3" * 32])
}
assert (self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group2.id,
[self.proj1env1.id],
{"browser": "ie"},
None,
None,
) is None)
def test_get_tag_value_paginator(self):
from sentry.tagstore.types import TagValue
assert list(
self.ts.get_tag_value_paginator(
self.proj1.id, self.proj1env1.id,
"sentry:user").get_result(10)) == [
TagValue(
key="sentry:user",
value="id:user1",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1),
),
TagValue(
key="sentry:user",
value="id:user2",
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2),
),
]
assert list(
self.ts.get_tag_value_paginator(
self.proj1.id, self.proj1env1.id, "sentry:user",
query="user1").get_result(10)) == [
TagValue(
key="sentry:user",
value="id:user1",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1),
)
]
def test_get_group_tag_value_iter(self):
from sentry.tagstore.types import GroupTagValue
assert list(
self.ts.get_group_tag_value_iter(
self.proj1.id, self.proj1group1.id, self.proj1env1.id,
"sentry:user")) == [
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user1",
times_seen=1,
first_seen=self.now - timedelta(seconds=1),
last_seen=self.now - timedelta(seconds=1),
),
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user2",
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2),
),
]
def test_get_group_tag_value_paginator(self):
from sentry.tagstore.types import GroupTagValue
assert list(
self.ts.get_group_tag_value_paginator(
self.proj1.id, self.proj1group1.id, self.proj1env1.id,
"sentry:user").get_result(10)) == [
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user1",
times_seen=1,
first_seen=self.now - timedelta(seconds=1),
last_seen=self.now - timedelta(seconds=1),
),
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user2",
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2),
),
]
def test_get_group_seen_values_for_environments(self):
assert self.ts.get_group_seen_values_for_environments(
[self.proj1.id], [self.proj1group1.id], [self.proj1env1.id]) == {
self.proj1group1.id: {
"first_seen": self.now - timedelta(seconds=2),
"last_seen": self.now - timedelta(seconds=1),
"times_seen": 2,
}
}
# test where there should be no results because of time filters
assert (self.ts.get_group_seen_values_for_environments(
[self.proj1.id],
[self.proj1group1.id],
[self.proj1env1.id],
start=self.now - timedelta(hours=5),
end=self.now - timedelta(hours=4),
) == {})
def setUp(self):
super().setUp()
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
env1 = "test"
env2 = "test2"
self.env3 = Environment.objects.create(
organization_id=self.proj1.organization_id, name="test3")
self.now = timezone.now().replace(microsecond=0)
exception = {
"values": [{
"type": "ValidationError",
"value": "Bad request",
"stacktrace": {
"frames": [{
"function": "?",
"filename": "http://localhost:1337/error.js",
"lineno": 29,
"colno": 3,
"in_app": False,
}]
},
}]
}
self.store_event(
data={
"event_id": "1" * 32,
"message": "message 1",
"platform": "python",
"environment": env1,
"fingerprint": ["group-1"],
"timestamp": iso_format(self.now - timedelta(seconds=1)),
"tags": {
"foo": "bar",
"baz": "quux",
"sentry:release": 100,
"sentry:user": "******",
},
"user": {
"id": "user1"
},
"exception": exception,
},
project_id=self.proj1.id,
)
self.proj1group1 = self.store_event(
data={
"event_id": "2" * 32,
"message": "message 1",
"platform": "python",
"environment": env1,
"fingerprint": ["group-1"],
"timestamp": iso_format(self.now - timedelta(seconds=2)),
"tags": {
"foo": "bar",
"baz": "quux",
"sentry:release": 200,
"sentry:user": "******",
},
"user": {
"id": "user2"
},
"exception": exception,
},
project_id=self.proj1.id,
).group
self.proj1group2 = self.store_event(
data={
"event_id": "3" * 32,
"message": "message 2",
"platform": "python",
"environment": env1,
"fingerprint": ["group-2"],
"timestamp": iso_format(self.now - timedelta(seconds=2)),
"tags": {
"browser": "chrome",
"sentry:user": "******"
},
"user": {
"id": "user1"
},
},
project_id=self.proj1.id,
).group
self.store_event(
data={
"event_id": "4" * 32,
"message": "message2",
"platform": "python",
"environment": env2,
"fingerprint": ["group-1"],
"timestamp": iso_format(self.now - timedelta(seconds=2)),
"tags": {
"foo": "bar"
},
},
project_id=self.proj1.id,
)
self.proj1env1 = Environment.objects.get(name=env1)
self.proj1env2 = Environment.objects.get(name=env2)
class TagStorage(TestCase):
def setUp(self):
assert requests.post(settings.SENTRY_SNUBA + '/tests/drop').status_code == 200
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
self.proj1env1 = self.create_environment(project=self.proj1, name='test')
self.proj1group1 = self.create_group(self.proj1)
self.proj1group2 = self.create_group(self.proj1)
hash1 = '1' * 32
hash2 = '2' * 32
GroupHash.objects.create(project=self.proj1, group=self.proj1group1, hash=hash1)
GroupHash.objects.create(project=self.proj1, group=self.proj1group2, hash=hash2)
self.now = datetime.utcnow().replace(microsecond=0)
data = json.dumps([{
'event_id': six.text_type(r) * 32,
'primary_hash': hash1,
'project_id': self.proj1.id,
'message': 'message 1',
'platform': 'python',
'datetime': (self.now - timedelta(seconds=r)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': {
'received': calendar.timegm(self.now.timetuple()) - r,
'tags': {
'foo': 'bar',
'baz': 'quux',
'environment': self.proj1env1.name,
'sentry:release': 100 * r,
},
'sentry.interfaces.User': {
'id': "user{}".format(r),
'email': "user{}@sentry.io".format(r)
}
},
} for r in range(1, 3)] + [{
'event_id': '3' * 32,
'primary_hash': hash2,
'project_id': self.proj1.id,
'message': 'message 2',
'platform': 'python',
'datetime': (self.now - timedelta(seconds=r)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': {
'received': calendar.timegm(self.now.timetuple()) - r,
'tags': {
'browser': 'chrome',
'environment': self.proj1env1.name,
},
'sentry.interfaces.User': {
'id': "user1"
}
},
}])
assert requests.post(settings.SENTRY_SNUBA + '/tests/insert', data=data).status_code == 200
def test_get_group_tag_keys_and_top_values(self):
result = self.ts.get_group_tag_keys_and_top_values(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
)
tags = [r['id'] for r in result]
assert set(tags) == set(['foo', 'baz', 'environment', 'release'])
result.sort(key=lambda r: r['id'])
assert result[0]['id'] == 'baz'
assert result[0]['uniqueValues'] == 1
assert result[0]['totalValues'] == 2
assert result[0]['topValues'][0]['value'] == 'quux'
assert result[3]['id'] == 'release'
assert result[3]['uniqueValues'] == 2
assert result[3]['totalValues'] == 2
assert result[3]['topValues'][0]['value'] == '100'
def test_get_top_group_tag_values(self):
resp = self.ts.get_top_group_tag_values(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
'foo',
1
)
assert len(resp) == 1
assert resp[0].times_seen == 2
assert resp[0].key == 'foo'
assert resp[0].value == 'bar'
assert resp[0].group_id == self.proj1group1.id
def test_get_group_tag_value_count(self):
assert self.ts.get_group_tag_value_count(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
'foo'
) == 2
def test_get_group_tag_key(self):
with pytest.raises(GroupTagKeyNotFound):
self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='notreal',
)
assert self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
).key == 'foo'
keys = self.ts.get_group_tag_keys(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
)
keys.sort(key=lambda x: x.key)
assert len(keys) == 2
assert keys[0].key == 'baz'
assert keys[0].times_seen == 2
assert keys[1].key == 'foo'
assert keys[1].times_seen == 2
def test_get_group_tag_value(self):
with pytest.raises(GroupTagValueNotFound):
self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
value='notreal',
)
assert self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='notreal',
) == []
assert self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
)[0].value == 'bar'
assert self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
value='bar',
).value == 'bar'
def test_get_tag_key(self):
with pytest.raises(TagKeyNotFound):
self.ts.get_tag_key(
project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key='notreal'
)
def test_get_tag_value(self):
with pytest.raises(TagValueNotFound):
self.ts.get_tag_value(
project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key='foo',
value='notreal',
)
def test_get_groups_user_counts(self):
assert self.ts.get_groups_user_counts(
project_id=self.proj1.id,
group_ids=[self.proj1group1.id, self.proj1group2.id],
environment_id=self.proj1env1.id
) == {
self.proj1group1.id: 2,
self.proj1group2.id: 1,
}
def test_get_releases(self):
assert self.ts.get_first_release(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
) == '200'
assert self.ts.get_first_release(
project_id=self.proj1.id,
group_id=self.proj1group2.id,
) is None
assert self.ts.get_last_release(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
) == '100'
assert self.ts.get_last_release(
project_id=self.proj1.id,
group_id=self.proj1group2.id,
) is None
def test_get_group_ids_for_users(self):
assert set(self.ts.get_group_ids_for_users(
[self.proj1.id],
[EventUser(project_id=self.proj1.id, ident='user1')]
)) == set([self.proj1group1.id, self.proj1group2.id])
assert set(self.ts.get_group_ids_for_users(
[self.proj1.id],
[EventUser(project_id=self.proj1.id, ident='user2')]
)) == set([self.proj1group1.id])
def test_get_group_tag_values_for_users(self):
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident='user1')]
)
one_second_ago = (self.now - timedelta(seconds=1)).strftime('%Y-%m-%dT%H:%M:%S+00:00')
assert len(result) == 1
assert result[0].value == 'user1'
assert result[0].last_seen == one_second_ago
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident='user2')]
)
two_seconds_ago = (self.now - timedelta(seconds=2)).strftime('%Y-%m-%dT%H:%M:%S+00:00')
assert len(result) == 1
assert result[0].value == 'user2'
assert result[0].last_seen == two_seconds_ago
# Test that users identified by different means are collected.
# (effectively tests OR conditions in snuba API)
result = self.ts.get_group_tag_values_for_users([
EventUser(project_id=self.proj1.id, email='*****@*****.**'),
EventUser(project_id=self.proj1.id, ident='user2')
])
two_seconds_ago = (self.now - timedelta(seconds=2)).strftime('%Y-%m-%dT%H:%M:%S+00:00')
assert len(result) == 2
result.sort(key=lambda x: x.value)
assert result[0].value == 'user1'
assert result[0].last_seen == one_second_ago
assert result[1].value == 'user2'
assert result[1].last_seen == two_seconds_ago
def test_get_release_tags(self):
tags = self.ts.get_release_tags(
[self.proj1.id],
None,
['100']
)
assert len(tags) == 1
one_second_ago = (self.now - timedelta(seconds=1)).strftime('%Y-%m-%dT%H:%M:%S+00:00')
assert tags[0].last_seen == one_second_ago
assert tags[0].first_seen == one_second_ago
assert tags[0].times_seen == 1
def test_get_group_event_ids(self):
assert sorted(self.ts.get_group_event_ids(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
{
'foo': 'bar',
}
)) == ["1" * 32, "2" * 32]
assert sorted(self.ts.get_group_event_ids(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
{
'foo': 'bar', # OR
'release': '200'
}
)) == ["1" * 32, "2" * 32]
assert self.ts.get_group_event_ids(
self.proj1.id,
self.proj1group2.id,
self.proj1env1.id,
{
'browser': 'chrome'
}
) == ["3" * 32]
assert self.ts.get_group_event_ids(
self.proj1.id,
self.proj1group2.id,
self.proj1env1.id,
{
'browser': 'ie'
}
) == []
def test_unmerge(self):
tagstore = SnubaTagStorage(
) # Snuba is not the default tag storage for tests yet
now = before_now(seconds=20).replace(microsecond=0, tzinfo=pytz.utc)
def time_from_now(offset=0):
return now + timedelta(seconds=offset)
project = self.create_project()
sequence = itertools.count(0)
tag_values = itertools.cycle(["red", "green", "blue"])
user_values = itertools.cycle([{"id": 1}, {"id": 2}])
def create_message_event(template,
parameters,
environment,
release,
fingerprint="group1"):
i = next(sequence)
event_id = uuid.UUID(fields=(i, 0x0, 0x1000, 0x80, 0x80,
0x808080808080)).hex
tags = [["color", next(tag_values)]]
if release:
tags.append(["sentry:release", release])
event = self.store_event(
data={
"event_id": event_id,
"message": template % parameters,
"type": "default",
"user": next(user_values),
"tags": tags,
"fingerprint": [fingerprint],
"timestamp": iso_format(now + timedelta(seconds=i)),
"environment": environment,
"release": release,
},
project_id=project.id,
)
UserReport.objects.create(
project_id=project.id,
group_id=event.group.id,
event_id=event_id,
name="Log Hat",
email="*****@*****.**",
comments="Quack",
)
features.record([event])
return event
events = OrderedDict()
for event in (create_message_event("This is message #%s.",
i,
environment="production",
release="version")
for i in xrange(10)):
events.setdefault(get_fingerprint(event), []).append(event)
for event in (create_message_event(
"This is message #%s!",
i,
environment="production",
release="version2",
fingerprint="group2",
) for i in xrange(10, 16)):
events.setdefault(get_fingerprint(event), []).append(event)
event = create_message_event(
"This is message #%s!",
17,
environment="staging",
release="version3",
fingerprint="group3",
)
events.setdefault(get_fingerprint(event), []).append(event)
merge_source, source, destination = list(Group.objects.all())
assert len(events) == 3
assert sum(map(len, events.values())) == 17
production_environment = Environment.objects.get(
organization_id=project.organization_id, name="production")
with self.tasks():
eventstream_state = eventstream.start_merge(
project.id, [merge_source.id], source.id)
merge_groups.delay([merge_source.id], source.id)
eventstream.end_merge(eventstream_state)
assert set([
(gtv.value, gtv.times_seen)
for gtv in tagstore.get_group_tag_values(
project.id, source.id, production_environment.id, "color")
]) == set([("red", 6), ("green", 5), ("blue", 5)])
similar_items = features.compare(source)
assert len(similar_items) == 2
assert similar_items[0][0] == source.id
assert similar_items[0][1]["message:message:character-shingles"] == 1.0
assert similar_items[1][0] == destination.id
assert similar_items[1][1]["message:message:character-shingles"] < 1.0
with self.tasks():
eventstream_state = eventstream.start_unmerge(
project.id, [events.keys()[0]], source.id, destination.id)
unmerge.delay(project.id,
source.id,
destination.id, [events.keys()[0]],
None,
batch_size=5)
eventstream.end_unmerge(eventstream_state)
assert (list(
Group.objects.filter(id=merge_source.id).values_list(
"times_seen", "first_seen", "last_seen")) == [])
assert list(
Group.objects.filter(id=source.id).values_list(
"times_seen", "first_seen",
"last_seen")) == [(6, time_from_now(10), time_from_now(15))]
assert list(
Group.objects.filter(id=destination.id).values_list(
"times_seen", "first_seen",
"last_seen")) == [(11, time_from_now(0), time_from_now(16))]
assert source.id != destination.id
assert source.project == destination.project
destination_event_ids = map(lambda event: event.event_id,
events.values()[1])
assert set(
UserReport.objects.filter(group_id=source.id).values_list(
"event_id", flat=True)) == set(destination_event_ids)
assert set(
GroupHash.objects.filter(group_id=source.id).values_list(
"hash",
flat=True)) == set([events.keys()[0],
events.keys()[1]])
assert set(
GroupRelease.objects.filter(group_id=source.id).values_list(
"environment", "first_seen",
"last_seen")) == set([(u"production", time_from_now(10),
time_from_now(15))])
assert set([
(gtv.value, gtv.times_seen)
for gtv in tagstore.get_group_tag_values(
project.id, destination.id, production_environment.id, "color")
]) == set([(u"red", 4), (u"green", 3), (u"blue", 3)])
destination_event_ids = map(lambda event: event.event_id,
events.values()[0] + events.values()[2])
assert set(
UserReport.objects.filter(group_id=destination.id).values_list(
"event_id", flat=True)) == set(destination_event_ids)
assert set(
GroupHash.objects.filter(group_id=destination.id).values_list(
"hash", flat=True)) == set([events.keys()[2]])
assert set(
GroupRelease.objects.filter(group_id=destination.id).values_list(
"environment", "first_seen", "last_seen")) == set([
("production", time_from_now(0), time_from_now(9)),
("staging", time_from_now(16), time_from_now(16)),
])
assert set([
(gtk.value, gtk.times_seen)
for gtk in tagstore.get_group_tag_values(
project.id, destination.id, production_environment.id, "color")
]) == set([("red", 4), ("blue", 3), ("green", 3)])
rollup_duration = 3600
time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
time_from_now(17),
rollup_duration,
)
environment_time_series = tsdb.get_range(
tsdb.models.group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
time_from_now(17),
rollup_duration,
environment_ids=[production_environment.id],
)
def get_expected_series_values(rollup, events, function=None):
if function is None:
def function(aggregate, event):
return (aggregate if aggregate is not None else 0) + 1
expected = {}
for event in events:
k = float((to_timestamp(event.datetime) // rollup_duration) *
rollup_duration)
expected[k] = function(expected.get(k), event)
return expected
def assert_series_contains(expected, actual, default=0):
actual = dict(actual)
for key, value in expected.items():
assert actual.get(key, 0) == value
for key in set(actual.keys()) - set(expected.keys()):
assert actual.get(key, 0) == default
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[1]),
time_series[source.id],
0,
)
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[0] +
events.values()[2]),
time_series[destination.id],
0,
)
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[1]),
environment_time_series[source.id],
0,
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0][:-1] + events.values()[2]),
environment_time_series[destination.id],
0,
)
time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
time_from_now(17),
rollup_duration,
)
environment_time_series = tsdb.get_distinct_counts_series(
tsdb.models.users_affected_by_group,
[source.id, destination.id],
now - timedelta(seconds=rollup_duration),
time_from_now(17),
rollup_duration,
environment_id=production_environment.id,
)
def collect_by_user_tag(aggregate, event):
aggregate = aggregate if aggregate is not None else set()
aggregate.add(
get_event_user_from_interface(event.data["user"]).tag_value)
return aggregate
for series in [time_series, environment_time_series]:
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[1], collect_by_user_tag).items()
},
series[source.id],
)
assert_series_contains(
{
timestamp: len(values)
for timestamp, values in get_expected_series_values(
rollup_duration,
events.values()[0] + events.values()[2],
collect_by_user_tag,
).items()
},
time_series[destination.id],
)
def strip_zeroes(data):
for group_id, series in data.items():
for _, values in series:
for key, val in values.items():
if val == 0:
values.pop(key)
return data
def collect_by_release(group, aggregate, event):
aggregate = aggregate if aggregate is not None else {}
release = event.get_tag("sentry:release")
if not release:
return aggregate
release = GroupRelease.objects.get(
group_id=group.id,
environment=event.data["environment"],
release_id=Release.objects.get(
organization_id=project.organization_id,
version=release).id,
).id
aggregate[release] = aggregate.get(release, 0) + 1
return aggregate
items = {}
for i in [source.id, destination.id]:
items[i] = list(
GroupRelease.objects.filter(group_id=i).values_list("id",
flat=True))
time_series = strip_zeroes(
tsdb.get_frequency_series(
tsdb.models.frequent_releases_by_group,
items,
now - timedelta(seconds=rollup_duration),
time_from_now(17),
rollup_duration,
))
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[1],
functools.partial(collect_by_release, source)),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(
rollup_duration,
events.values()[0] + events.values()[2],
functools.partial(collect_by_release, destination),
),
time_series[destination.id],
{},
)
items = {}
for i in [source.id, destination.id]:
items[i] = list(Environment.objects.all().values_list("id",
flat=True))
time_series = strip_zeroes(
tsdb.get_frequency_series(
tsdb.models.frequent_environments_by_group,
items,
now - timedelta(seconds=rollup_duration),
time_from_now(17),
rollup_duration,
))
def collect_by_environment(aggregate, event):
aggregate = aggregate if aggregate is not None else {}
environment = Environment.objects.get(
organization_id=project.organization_id,
name=event.data["environment"]).id
aggregate[environment] = aggregate.get(environment, 0) + 1
return aggregate
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[1],
collect_by_environment),
time_series[source.id],
{},
)
assert_series_contains(
get_expected_series_values(rollup_duration,
events.values()[0] + events.values()[2],
collect_by_environment),
time_series[destination.id],
{},
)
source_similar_items = features.compare(source)
assert source_similar_items[0] == (
source.id,
{
"exception:message:character-shingles": None,
"exception:stacktrace:application-chunks": None,
"exception:stacktrace:pairs": None,
"message:message:character-shingles": 1.0,
},
)
assert source_similar_items[1][0] == destination.id
assert source_similar_items[1][1][
"message:message:character-shingles"] < 1.0
destination_similar_items = features.compare(destination)
assert destination_similar_items[0] == (
destination.id,
{
"exception:message:character-shingles": None,
"exception:stacktrace:application-chunks": None,
"exception:stacktrace:pairs": None,
"message:message:character-shingles": 1.0,
},
)
assert destination_similar_items[1][0] == source.id
assert destination_similar_items[1][1][
"message:message:character-shingles"] < 1.0
class TagStorageTest(SnubaTestCase):
def setUp(self):
super(TagStorageTest, self).setUp()
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
self.proj1env1 = self.create_environment(project=self.proj1, name='test')
self.proj1group1 = self.create_group(self.proj1)
self.proj1group2 = self.create_group(self.proj1)
hash1 = '1' * 32
hash2 = '2' * 32
GroupHash.objects.create(project=self.proj1, group=self.proj1group1, hash=hash1)
GroupHash.objects.create(project=self.proj1, group=self.proj1group2, hash=hash2)
self.now = timezone.now().replace(microsecond=0)
data = json.dumps([{
'event_id': six.text_type(r) * 32,
'primary_hash': hash1,
'group_id': self.proj1group1.id,
'project_id': self.proj1.id,
'message': 'message 1',
'platform': 'python',
'datetime': (self.now - timedelta(seconds=r)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': {
'received': calendar.timegm(self.now.timetuple()) - r,
'tags': {
'foo': 'bar',
'baz': 'quux',
'environment': self.proj1env1.name,
'sentry:release': 100 * r,
'sentry:user': u"id:user{}".format(r),
},
'user': {
'id': u"user{}".format(r),
'email': u"user{}@sentry.io".format(r)
}
},
} for r in [1, 2]] + [{
'event_id': '3' * 32,
'primary_hash': hash2,
'group_id': self.proj1group2.id,
'project_id': self.proj1.id,
'message': 'message 2',
'platform': 'python',
'datetime': (self.now - timedelta(seconds=2)).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': {
'received': calendar.timegm(self.now.timetuple()) - 2,
'tags': {
'browser': 'chrome',
'environment': self.proj1env1.name,
'sentry:user': "******",
},
'user': {
'id': "user1"
}
},
}])
assert requests.post(settings.SENTRY_SNUBA + '/tests/insert', data=data).status_code == 200
def test_get_group_tag_keys_and_top_values(self):
result = list(self.ts.get_group_tag_keys_and_top_values(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
))
tags = [r.key for r in result]
assert set(tags) == set(['foo', 'baz', 'environment', 'sentry:release', 'sentry:user'])
result.sort(key=lambda r: r.key)
assert result[0].key == 'baz'
assert result[0].top_values[0].value == 'quux'
assert result[0].values_seen == 1
assert result[0].count == 2
assert result[3].key == 'sentry:release'
assert result[3].values_seen == 2
assert result[3].count == 2
top_release_values = result[3].top_values
assert len(top_release_values) == 2
assert set(v.value for v in top_release_values) == set(['100', '200'])
assert all(v.times_seen == 1 for v in top_release_values)
# Now with only a specific set of keys,
result = list(self.ts.get_group_tag_keys_and_top_values(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
keys=['environment', 'sentry:release'],
))
tags = [r.key for r in result]
assert set(tags) == set(['environment', 'sentry:release'])
result.sort(key=lambda r: r.key)
assert result[0].key == 'environment'
assert result[0].top_values[0].value == 'test'
assert result[1].key == 'sentry:release'
top_release_values = result[1].top_values
assert len(top_release_values) == 2
assert set(v.value for v in top_release_values) == set(['100', '200'])
assert all(v.times_seen == 1 for v in top_release_values)
def test_get_top_group_tag_values(self):
resp = self.ts.get_top_group_tag_values(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
'foo',
1
)
assert len(resp) == 1
assert resp[0].times_seen == 2
assert resp[0].key == 'foo'
assert resp[0].value == 'bar'
assert resp[0].group_id == self.proj1group1.id
def test_get_group_tag_value_count(self):
assert self.ts.get_group_tag_value_count(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
'foo'
) == 2
def test_get_group_tag_key(self):
with pytest.raises(GroupTagKeyNotFound):
self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='notreal',
)
assert self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
).key == 'foo'
keys = {
k.key: k for k in self.ts.get_group_tag_keys(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
)
}
assert set(keys) == set(['baz', 'environment', 'foo', 'sentry:release', 'sentry:user'])
for k in keys.values():
if k.key not in set(['sentry:release', 'sentry:user']):
assert k.values_seen == 1, u'expected {!r} to have 1 unique value'.format(k.key)
else:
assert k.values_seen == 2
def test_get_group_tag_value(self):
with pytest.raises(GroupTagValueNotFound):
self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
value='notreal',
)
assert self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='notreal',
) == set([])
assert list(self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
))[0].value == 'bar'
assert self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key='foo',
value='bar',
).value == 'bar'
def test_get_tag_key(self):
with pytest.raises(TagKeyNotFound):
self.ts.get_tag_key(
project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key='notreal'
)
def test_get_tag_value(self):
with pytest.raises(TagValueNotFound):
self.ts.get_tag_value(
project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key='foo',
value='notreal',
)
def test_get_groups_user_counts(self):
assert self.ts.get_groups_user_counts(
project_ids=[self.proj1.id],
group_ids=[self.proj1group1.id, self.proj1group2.id],
environment_ids=[self.proj1env1.id]
) == {
self.proj1group1.id: 2,
self.proj1group2.id: 1,
}
def test_get_releases(self):
assert self.ts.get_first_release(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
) == '200'
assert self.ts.get_first_release(
project_id=self.proj1.id,
group_id=self.proj1group2.id,
) is None
assert self.ts.get_last_release(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
) == '100'
assert self.ts.get_last_release(
project_id=self.proj1.id,
group_id=self.proj1group2.id,
) is None
def test_get_group_ids_for_users(self):
assert self.ts.get_group_ids_for_users(
[self.proj1.id],
[EventUser(project_id=self.proj1.id, ident='user1')]
) == set([self.proj1group1.id, self.proj1group2.id])
assert self.ts.get_group_ids_for_users(
[self.proj1.id],
[EventUser(project_id=self.proj1.id, ident='user2')]
) == set([self.proj1group1.id])
def test_get_group_tag_values_for_users(self):
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident='user1')]
)
assert len(result) == 2
assert set(v.group_id for v in result) == set([
self.proj1group1.id,
self.proj1group2.id,
])
assert set(v.last_seen for v in result) == \
set([self.now - timedelta(seconds=1), self.now - timedelta(seconds=2)])
result.sort(key=lambda x: x.last_seen)
assert result[0].last_seen == self.now - timedelta(seconds=2)
assert result[1].last_seen == self.now - timedelta(seconds=1)
for v in result:
assert v.value == 'user1'
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident='user2')]
)
assert len(result) == 1
assert result[0].value == 'user2'
assert result[0].last_seen == self.now - timedelta(seconds=2)
def test_get_release_tags(self):
tags = list(
self.ts.get_release_tags(
[self.proj1.id],
None,
['100']
)
)
assert len(tags) == 1
one_second_ago = self.now - timedelta(seconds=1)
assert tags[0].last_seen == one_second_ago
assert tags[0].first_seen == one_second_ago
assert tags[0].times_seen == 1
assert tags[0].key == 'sentry:release'
def test_get_group_event_filter(self):
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
{
'foo': 'bar',
}
) == {'event_id__in': set(["1" * 32, "2" * 32])}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
{
'foo': 'bar', # AND
'sentry:release': '200'
}
) == {'event_id__in': set(["2" * 32])}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group2.id,
self.proj1env1.id,
{
'browser': 'chrome'
}
) == {'event_id__in': set(["3" * 32])}
assert self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group2.id,
self.proj1env1.id,
{
'browser': 'ie'
}
) is None
def test_get_tag_value_paginator(self):
from sentry.tagstore.types import TagValue
assert list(self.ts.get_tag_value_paginator(
self.proj1.id,
self.proj1env1.id,
'sentry:user',
).get_result(10)) == [
TagValue(
key='sentry:user',
value='id:user1',
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1)
),
TagValue(
key='sentry:user',
value='id:user2',
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2)
)
]
assert list(self.ts.get_tag_value_paginator(
self.proj1.id,
self.proj1env1.id,
'sentry:user',
query='user1',
).get_result(10)) == [
TagValue(
key='sentry:user',
value='id:user1',
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1)
),
]
def test_get_group_tag_value_iter(self):
from sentry.tagstore.types import GroupTagValue
assert list(self.ts.get_group_tag_value_iter(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
'sentry:user',
)) == [
GroupTagValue(
group_id=self.proj1group1.id,
key='sentry:user',
value='id:user1',
times_seen=1,
first_seen=self.now - timedelta(seconds=1),
last_seen=self.now - timedelta(seconds=1)
),
GroupTagValue(
group_id=self.proj1group1.id,
key='sentry:user',
value='id:user2',
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2)
)
]
def test_get_group_tag_value_paginator(self):
from sentry.tagstore.types import GroupTagValue
assert list(self.ts.get_group_tag_value_paginator(
self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
'sentry:user',
).get_result(10)) == [
GroupTagValue(
group_id=self.proj1group1.id,
key='sentry:user',
value='id:user1',
times_seen=1,
first_seen=self.now - timedelta(seconds=1),
last_seen=self.now - timedelta(seconds=1)
),
GroupTagValue(
group_id=self.proj1group1.id,
key='sentry:user',
value='id:user2',
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2)
)
]
class TagStorageTest(TestCase, SnubaTestCase):
def setUp(self):
super().setUp()
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
env1 = "test"
env2 = "test2"
self.env3 = Environment.objects.create(
organization_id=self.proj1.organization_id, name="test3")
self.now = timezone.now().replace(microsecond=0)
self.store_event(
data={
"event_id": "1" * 32,
"message": "message 1",
"platform": "python",
"environment": env1,
"fingerprint": ["group-1"],
"timestamp": iso_format(self.now - timedelta(seconds=1)),
"tags": {
"foo": "bar",
"baz": "quux",
"sentry:release": 100,
"sentry:user": "******",
},
"user": {
"id": "user1"
},
"exception": exception,
},
project_id=self.proj1.id,
)
self.proj1group1 = self.store_event(
data={
"event_id": "2" * 32,
"message": "message 1",
"platform": "python",
"environment": env1,
"fingerprint": ["group-1"],
"timestamp": iso_format(self.now - timedelta(seconds=2)),
"tags": {
"foo": "bar",
"baz": "quux",
"sentry:release": 200,
"sentry:user": "******",
},
"user": {
"id": "user2"
},
"exception": exception,
},
project_id=self.proj1.id,
).group
self.proj1group2 = self.store_event(
data={
"event_id": "3" * 32,
"message": "message 2",
"platform": "python",
"environment": env1,
"fingerprint": ["group-2"],
"timestamp": iso_format(self.now - timedelta(seconds=2)),
"tags": {
"browser": "chrome",
"sentry:user": "******"
},
"user": {
"id": "user1"
},
},
project_id=self.proj1.id,
).group
self.store_event(
data={
"event_id": "4" * 32,
"message": "message2",
"platform": "python",
"environment": env2,
"fingerprint": ["group-1"],
"timestamp": iso_format(self.now - timedelta(seconds=2)),
"tags": {
"foo": "bar"
},
},
project_id=self.proj1.id,
)
self.proj1env1 = Environment.objects.get(name=env1)
self.proj1env2 = Environment.objects.get(name=env2)
def test_get_group_tag_keys_and_top_values(self):
result = list(
self.ts.get_group_tag_keys_and_top_values(self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id]))
tags = [r.key for r in result]
assert set(tags) == {
"foo", "baz", "environment", "sentry:release", "sentry:user",
"level"
}
result.sort(key=lambda r: r.key)
assert result[0].key == "baz"
assert result[0].top_values[0].value == "quux"
assert result[0].count == 2
assert result[4].key == "sentry:release"
assert result[4].count == 2
top_release_values = result[4].top_values
assert len(top_release_values) == 2
assert {v.value for v in top_release_values} == {"100", "200"}
assert all(v.times_seen == 1 for v in top_release_values)
# Now with only a specific set of keys,
result = list(
self.ts.get_group_tag_keys_and_top_values(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
keys=["environment", "sentry:release"],
))
tags = [r.key for r in result]
assert set(tags) == {"environment", "sentry:release"}
result.sort(key=lambda r: r.key)
assert result[0].key == "environment"
assert result[0].top_values[0].value == "test"
assert result[1].key == "sentry:release"
top_release_values = result[1].top_values
assert len(top_release_values) == 2
assert {v.value for v in top_release_values} == {"100", "200"}
assert all(v.times_seen == 1 for v in top_release_values)
def test_get_top_group_tag_values(self):
resp = self.ts.get_top_group_tag_values(self.proj1.id,
self.proj1group1.id,
self.proj1env1.id, "foo", 1)
assert len(resp) == 1
assert resp[0].times_seen == 2
assert resp[0].key == "foo"
assert resp[0].value == "bar"
assert resp[0].group_id == self.proj1group1.id
def test_get_group_tag_value_count(self):
assert (self.ts.get_group_tag_value_count(self.proj1.id,
self.proj1group1.id,
self.proj1env1.id,
"foo") == 2)
def test_get_tag_keys(self):
expected_keys = {
"baz",
"browser",
"environment",
"foo",
"sentry:release",
"sentry:user",
"level",
}
keys = {
k.key: k
for k in self.ts.get_tag_keys(project_id=self.proj1.id,
environment_id=self.proj1env1.id)
}
assert set(keys) == expected_keys
keys = {
k.key: k
for k in self.ts.get_tag_keys(project_id=self.proj1.id,
environment_id=self.proj1env1.id,
include_values_seen=True)
}
assert set(keys) == expected_keys
def test_get_group_tag_key(self):
with pytest.raises(GroupTagKeyNotFound):
self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="notreal",
)
assert (self.ts.get_group_tag_key(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="foo",
).key == "foo")
keys = {
k.key: k
for k in self.ts.get_group_tag_keys(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_ids=[self.proj1env1.id],
)
}
assert set(keys) == {
"baz", "environment", "foo", "sentry:release", "sentry:user",
"level"
}
def test_get_group_tag_value(self):
with pytest.raises(GroupTagValueNotFound):
self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="foo",
value="notreal",
)
assert (self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="notreal",
) == set())
assert (list(
self.ts.get_group_tag_values(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="foo",
))[0].value == "bar")
assert (self.ts.get_group_tag_value(
project_id=self.proj1.id,
group_id=self.proj1group1.id,
environment_id=self.proj1env1.id,
key="foo",
value="bar",
).value == "bar")
def test_get_tag_key(self):
with pytest.raises(TagKeyNotFound):
self.ts.get_tag_key(project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key="notreal")
def test_get_tag_value(self):
with pytest.raises(TagValueNotFound):
self.ts.get_tag_value(
project_id=self.proj1.id,
environment_id=self.proj1env1.id,
key="foo",
value="notreal",
)
def test_get_tag_value_label(self):
assert self.ts.get_tag_value_label("foo", "notreal") == "notreal"
assert self.ts.get_tag_value_label("sentry:user", None) is None
assert self.ts.get_tag_value_label("sentry:user",
"id:stuff") == "stuff"
assert self.ts.get_tag_value_label("sentry:user",
"email:stuff") == "stuff"
assert self.ts.get_tag_value_label("sentry:user",
"username:stuff") == "stuff"
assert self.ts.get_tag_value_label("sentry:user",
"ip:stuff") == "stuff"
def test_get_groups_user_counts(self):
assert (self.ts.get_groups_user_counts(
project_ids=[self.proj1.id],
group_ids=[self.proj1group1.id, self.proj1group2.id],
environment_ids=[self.proj1env1.id],
) == {
self.proj1group1.id: 2,
self.proj1group2.id: 1
})
# test filtering by date range where there shouldn't be results
assert (self.ts.get_groups_user_counts(
project_ids=[self.proj1.id],
group_ids=[self.proj1group1.id, self.proj1group2.id],
environment_ids=[self.proj1env1.id],
start=self.now - timedelta(days=5),
end=self.now - timedelta(days=4),
) == {})
def test_get_group_ids_for_users(self):
assert self.ts.get_group_ids_for_users(
[self.proj1.id],
[EventUser(project_id=self.proj1.id, ident="user1")]) == {
self.proj1group1.id, self.proj1group2.id
}
assert self.ts.get_group_ids_for_users(
[self.proj1.id],
[EventUser(project_id=self.proj1.id, ident="user2")]) == {
self.proj1group1.id
}
def test_get_group_tag_values_for_users(self):
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident="user1")])
assert len(result) == 2
assert {v.group_id
for v in result} == {self.proj1group1.id, self.proj1group2.id}
assert {v.last_seen
for v in result} == {
self.now - timedelta(seconds=1),
self.now - timedelta(seconds=2),
}
result.sort(key=lambda x: x.last_seen)
assert result[0].last_seen == self.now - timedelta(seconds=2)
assert result[1].last_seen == self.now - timedelta(seconds=1)
for v in result:
assert v.value == "user1"
result = self.ts.get_group_tag_values_for_users(
[EventUser(project_id=self.proj1.id, ident="user2")])
assert len(result) == 1
assert result[0].value == "user2"
assert result[0].last_seen == self.now - timedelta(seconds=2)
def test_get_release_tags(self):
tags = list(
self.ts.get_release_tags(self.proj1.organization_id,
[self.proj1.id], None, ["100"]))
assert len(tags) == 1
one_second_ago = self.now - timedelta(seconds=1)
assert tags[0].last_seen == one_second_ago
assert tags[0].first_seen == one_second_ago
assert tags[0].times_seen == 1
assert tags[0].key == "sentry:release"
def test_get_release_tags_uses_release_project_environment(self):
tags = list(
self.ts.get_release_tags(self.proj1.organization_id,
[self.proj1.id], None, ["100"]))
assert len(tags) == 1
one_second_ago = self.now - timedelta(seconds=1)
assert tags[0].last_seen == one_second_ago
assert tags[0].first_seen == one_second_ago
assert tags[0].times_seen == 1
one_day_ago = self.now - timedelta(days=1)
two_days_ago = self.now - timedelta(days=2)
self.store_event(
data={
"event_id": "5" * 32,
"message": "message3",
"platform": "python",
"environment": None,
"fingerprint": ["group-1"],
"timestamp": iso_format(one_day_ago),
"tags": {
"sentry:release": 100,
},
},
project_id=self.proj1.id,
)
release = Release.objects.create(version="100",
organization=self.organization)
ReleaseProjectEnvironment.objects.create(
release_id=release.id,
project_id=self.proj1.id,
environment_id=self.env3.id,
first_seen=one_day_ago,
)
self.store_event(
data={
"event_id": "6" * 32,
"message": "message3",
"platform": "python",
"environment": None,
"fingerprint": ["group-1"],
"timestamp": iso_format(two_days_ago),
"tags": {
"sentry:release": 100,
},
},
project_id=self.proj1.id,
)
tags = list(
self.ts.get_release_tags(self.proj1.organization_id,
[self.proj1.id], None, ["100"]))
assert tags[0].last_seen == one_second_ago
assert tags[0].first_seen == one_day_ago
assert (
tags[0].times_seen == 2
) # Isn't 3 because start was limited by the ReleaseProjectEnvironment entry
def test_get_group_event_filter(self):
assert self.ts.get_group_event_filter(self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
{"foo": "bar"}, None, None) == {
"event_id__in":
{"1" * 32, "2" * 32}
}
assert (self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
{"foo": "bar"},
(self.now - timedelta(seconds=1)),
None,
) == {
"event_id__in": {"1" * 32}
})
assert (self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
{"foo": "bar"},
None,
(self.now - timedelta(seconds=1)),
) == {
"event_id__in": {"2" * 32}
})
assert (self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id, self.proj1env2.id],
{"foo": "bar"},
None,
None,
) == {
"event_id__in": {"1" * 32, "2" * 32, "4" * 32}
})
assert (self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
{
"foo": "bar",
"sentry:release": "200"
}, # AND
None,
None,
) == {
"event_id__in": {"2" * 32}
})
assert (self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group2.id,
[self.proj1env1.id],
{"browser": "chrome"},
None,
None,
) == {
"event_id__in": {"3" * 32}
})
assert (self.ts.get_group_event_filter(
self.proj1.id,
self.proj1group2.id,
[self.proj1env1.id],
{"browser": "ie"},
None,
None,
) is None)
def test_get_tag_value_paginator(self):
from sentry.tagstore.types import TagValue
assert list(
self.ts.get_tag_value_paginator(
self.proj1.id, self.proj1env1.id,
"sentry:user").get_result(10)) == [
TagValue(
key="sentry:user",
value="id:user1",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1),
),
TagValue(
key="sentry:user",
value="id:user2",
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2),
),
]
assert list(
self.ts.get_tag_value_paginator(
self.proj1.id, self.proj1env1.id, "sentry:user",
query="user1").get_result(10)) == [
TagValue(
key="sentry:user",
value="id:user1",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1),
)
]
def test_get_tag_value_paginator_with_dates(self):
from sentry.tagstore.types import TagValue
day_ago = self.now - timedelta(days=1)
two_days_ago = self.now - timedelta(days=2)
assert list(
self.ts.get_tag_value_paginator(
self.proj1.id,
self.proj1env1.id,
"sentry:user",
start=day_ago,
end=self.now).get_result(10)) == [
TagValue(
key="sentry:user",
value="id:user1",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1),
),
TagValue(
key="sentry:user",
value="id:user2",
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2),
),
]
day_ago = self.now - timedelta(days=1)
assert (list(
self.ts.get_tag_value_paginator(self.proj1.id,
self.proj1env1.id,
"sentry:user",
start=two_days_ago,
end=day_ago).get_result(10)) == [])
def test_numeric_tag_value_paginator(self):
from sentry.tagstore.types import TagValue
assert list(
self.ts.get_tag_value_paginator(
self.proj1.id, self.proj1env1.id,
"stack.lineno").get_result(10)) == [
TagValue(
key="stack.lineno",
value="29",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1),
)
]
assert list(
self.ts.get_tag_value_paginator(
self.proj1.id, self.proj1env1.id, "stack.lineno",
query="30").get_result(10)) == [
TagValue(
key="stack.lineno",
value="29",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=1),
)
]
def test_get_group_tag_value_iter(self):
from sentry.tagstore.types import GroupTagValue
assert list(
self.ts.get_group_tag_value_iter(
self.proj1.id, self.proj1group1.id, [self.proj1env1.id],
"sentry:user")) == [
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user1",
times_seen=1,
first_seen=self.now - timedelta(seconds=1),
last_seen=self.now - timedelta(seconds=1),
),
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user2",
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2),
),
]
def test_get_group_tag_value_paginator(self):
from sentry.tagstore.types import GroupTagValue
assert list(
self.ts.get_group_tag_value_paginator(
self.proj1.id, self.proj1group1.id, [self.proj1env1.id],
"sentry:user").get_result(10)) == [
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user1",
times_seen=1,
first_seen=self.now - timedelta(seconds=1),
last_seen=self.now - timedelta(seconds=1),
),
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user2",
times_seen=1,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2),
),
]
def test_get_group_tag_value_paginator_times_seen(self):
from sentry.tagstore.types import GroupTagValue
self.store_event(
data={
"event_id": "5" * 32,
"message": "message 1",
"platform": "python",
"environment": self.proj1env1.name,
"fingerprint": ["group-1"],
"timestamp": iso_format(self.now - timedelta(seconds=2)),
"tags": {
"foo": "bar",
"baz": "quux",
"sentry:release": 100,
"sentry:user": "******",
},
"user": {
"id": "user2"
},
"exception": exception,
},
project_id=self.proj1.id,
)
assert list(
self.ts.get_group_tag_value_paginator(
self.proj1.id,
self.proj1group1.id,
[self.proj1env1.id],
"sentry:user",
order_by="-times_seen",
).get_result(10)) == [
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user2",
times_seen=2,
first_seen=self.now - timedelta(seconds=2),
last_seen=self.now - timedelta(seconds=2),
),
GroupTagValue(
group_id=self.proj1group1.id,
key="sentry:user",
value="id:user1",
times_seen=1,
first_seen=self.now - timedelta(seconds=1),
last_seen=self.now - timedelta(seconds=1),
),
]
def test_get_group_seen_values_for_environments(self):
assert self.ts.get_group_seen_values_for_environments(
[self.proj1.id], [self.proj1group1.id], [self.proj1env1.id]) == {
self.proj1group1.id: {
"first_seen": self.now - timedelta(seconds=2),
"last_seen": self.now - timedelta(seconds=1),
"times_seen": 2,
}
}
# test where there should be no results because of time filters
assert (self.ts.get_group_seen_values_for_environments(
[self.proj1.id],
[self.proj1group1.id],
[self.proj1env1.id],
start=self.now - timedelta(hours=5),
end=self.now - timedelta(hours=4),
) == {})
def setUp(self):
super(TagStorageTest, self).setUp()
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
self.proj1env1 = self.create_environment(project=self.proj1,
name="test")
self.proj1env2 = self.create_environment(project=self.proj1,
name="test2")
self.proj1group1 = self.create_group(self.proj1)
self.proj1group2 = self.create_group(self.proj1)
hash1 = "1" * 32
hash2 = "2" * 32
GroupHash.objects.create(project=self.proj1,
group=self.proj1group1,
hash=hash1)
GroupHash.objects.create(project=self.proj1,
group=self.proj1group2,
hash=hash2)
self.now = timezone.now().replace(microsecond=0)
data = json.dumps([{
"event_id":
six.text_type(r) * 32,
"primary_hash":
hash1,
"group_id":
self.proj1group1.id,
"project_id":
self.proj1.id,
"message":
"message 1",
"platform":
"python",
"datetime": (self.now - timedelta(seconds=r)
).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
"data": {
"received": calendar.timegm(self.now.timetuple()) - r,
"tags": {
"foo": "bar",
"baz": "quux",
"environment": self.proj1env1.name,
"sentry:release": 100 * r,
"sentry:user": u"id:user{}".format(r),
},
"user": {
"id": u"user{}".format(r),
"email": u"user{}@sentry.io".format(r)
},
},
} for r in [1, 2]] + [{
"event_id":
"3" * 32,
"primary_hash":
hash2,
"group_id":
self.proj1group2.id,
"project_id":
self.proj1.id,
"message":
"message 2",
"platform":
"python",
"datetime": (self.now - timedelta(seconds=2)
).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
"data": {
"received": calendar.timegm(self.now.timetuple()) - 2,
"tags": {
"browser": "chrome",
"environment": self.proj1env1.name,
"sentry:user": "******",
},
"user": {
"id": "user1"
},
},
}] + [{
"event_id":
"4" * 32,
"primary_hash":
hash2,
"group_id":
self.proj1group1.id,
"project_id":
self.proj1.id,
"message":
"message 2",
"platform":
"python",
"datetime": (self.now - timedelta(seconds=2)
).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
"data": {
"received": calendar.timegm(self.now.timetuple()) - 2,
"tags": {
"foo": "bar",
"environment": self.proj1env2.name
},
"user": {
"id": "user1"
},
},
}])
assert (requests.post(settings.SENTRY_SNUBA + "/tests/events/insert",
data=data).status_code == 200)
def setUp(self):
super().setUp()
self.ts = SnubaTagStorage()
self.proj1 = self.create_project()
env1 = "test"
env2 = "test2"
self.env3 = Environment.objects.create(
organization_id=self.proj1.organization_id, name="test3")
self.now = timezone.now().replace(microsecond=0)
self.store_event(
data={
"event_id": "1" * 32,
"message": "message 1",
"platform": "python",
"environment": env1,
"fingerprint": ["group-1"],
"timestamp": iso_format(self.now - timedelta(seconds=1)),
"tags": {
"foo": "bar",
"baz": "quux",
"sentry:release": 100,
"sentry:user": "******",
},
"user": {
"id": "user1"
},
"exception": exception,
},
project_id=self.proj1.id,
)
self.proj1group1 = self.store_event(
data={
"event_id": "2" * 32,
"message": "message 1",
"platform": "python",
"environment": env1,
"fingerprint": ["group-1"],
"timestamp": iso_format(self.now - timedelta(seconds=2)),
"tags": {
"foo": "bar",
"baz": "quux",
"sentry:release": 200,
"sentry:user": "******",
},
"user": {
"id": "user2"
},
"exception": exception,
},
project_id=self.proj1.id,
).group
self.proj1group2 = self.store_event(
data={
"event_id": "3" * 32,
"message": "message 2",
"platform": "python",
"environment": env1,
"fingerprint": ["group-2"],
"timestamp": iso_format(self.now - timedelta(seconds=2)),
"tags": {
"browser": "chrome",
"sentry:user": "******"
},
"user": {
"id": "user1"
},
},
project_id=self.proj1.id,
).group
self.store_event(
data={
"event_id": "4" * 32,
"message": "message2",
"platform": "python",
"environment": env2,
"fingerprint": ["group-1"],
"timestamp": iso_format(self.now - timedelta(seconds=2)),
"tags": {
"foo": "bar"
},
},
project_id=self.proj1.id,
)
self.proj1env1 = Environment.objects.get(name=env1)
self.proj1env2 = Environment.objects.get(name=env2)
