def test_transfer_to_team(self):
from_org = self.create_organization()
from_team = self.create_team(organization=from_org)
to_org = self.create_organization()
to_team = self.create_team(organization=to_org)
project = self.create_project(teams=[from_team])
rule = Rule.objects.create(
project=project,
environment_id=Environment.get_or_create(project, 'production').id,
label='Golden Rule',
data={},
)
project.transfer_to(team=to_team)
project = Project.objects.get(id=project.id)
assert project.teams.count() == 1
assert project.teams.first() == to_team
assert project.organization_id == to_org.id
updated_rule = project.rule_set.get(label='Golden Rule')
assert updated_rule.id == rule.id
assert updated_rule.environment_id != rule.environment_id
assert updated_rule.environment_id == Environment.get_or_create(project, 'production').id
def test_multiple_environments(self):
group = self.create_group()
self.login_as(user=self.user)
environment = Environment.get_or_create(group.project, 'production')
environment2 = Environment.get_or_create(group.project, 'staging')
url = u'/api/0/issues/{}/?enable_snuba=1'.format(group.id)
from sentry.api.endpoints.group_details import tsdb
with mock.patch(
'sentry.api.endpoints.group_details.tsdb.get_range',
side_effect=tsdb.get_range) as get_range:
response = self.client.get(
'%s&environment=production&environment=staging' % (url,),
format='json'
)
assert response.status_code == 200
assert get_range.call_count == 2
for args, kwargs in get_range.call_args_list:
assert kwargs['environment_ids'] == [environment.id, environment2.id]
response = self.client.get('%s&environment=invalid' % (url,), format='json')
assert response.status_code == 404
def test_simple(self):
project = self.create_project()
with pytest.raises(Environment.DoesNotExist):
Environment.get_for_organization_id(
project.organization_id,
'prod',
)
env = Environment.get_or_create(
project=project,
name='prod',
)
assert env.name == 'prod'
assert env.projects.first().id == project.id
env2 = Environment.get_or_create(
project=project,
name='prod',
)
assert env2.id == env.id
with self.assertNumQueries(0):
assert Environment.get_for_organization_id(
project.organization_id,
'prod',
).id == env.id
def test_with_environment(self):
self.login_as(user=self.user)
project = self.create_project()
Environment.get_or_create(
project,
'production',
)
conditions = [
{
'id': 'sentry.rules.conditions.first_seen_event.FirstSeenEventCondition',
'key': 'foo',
'match': 'eq',
'value': 'bar',
}
]
actions = [{'id': 'sentry.rules.actions.notify_event.NotifyEventAction'}]
url = reverse(
'sentry-api-0-project-rules',
kwargs={
'organization_slug': project.organization.slug,
'project_slug': project.slug,
}
)
response = self.client.post(
url,
data={
'name': 'hello world',
'environment': 'production',
'conditions': conditions,
'actions': actions,
'actionMatch': 'any',
'frequency': 30,
},
format='json'
)
assert response.status_code == 200, response.content
assert response.data['id']
assert response.data['environment'] == 'production'
rule = Rule.objects.get(id=response.data['id'])
assert rule.label == 'hello world'
assert rule.environment_id == Environment.get_or_create(
rule.project,
'production',
).id
def test_environment(self):
group = self.create_group()
self.login_as(user=self.user)
environment = Environment.get_or_create(group.project, 'production')
url = u'/api/0/issues/{}/'.format(group.id)
from sentry.api.endpoints.group_details import tsdb
with mock.patch(
'sentry.api.endpoints.group_details.tsdb.get_range',
side_effect=tsdb.get_range) as get_range:
response = self.client.get(url, {'environment': 'production'}, format='json')
assert response.status_code == 200
assert get_range.call_count == 2
for args, kwargs in get_range.call_args_list:
assert kwargs['environment_ids'] == [environment.id]
with mock.patch(
'sentry.api.endpoints.group_details.tsdb.make_series',
side_effect=tsdb.make_series) as make_series:
response = self.client.get(url, {'environment': 'invalid'}, format='json')
assert response.status_code == 200
assert make_series.call_count == 2
def test_environment(self):
group = self.group
environment = Environment.get_or_create(group.project, 'production')
with mock.patch(
'sentry.api.serializers.models.group.snuba_tsdb.get_range',
side_effect=snuba_tsdb.get_range) as get_range:
serialize(
[group],
serializer=StreamGroupSerializerSnuba(
environment_ids=[environment.id],
stats_period='14d',
),
)
assert get_range.call_count == 1
for args, kwargs in get_range.call_args_list:
assert kwargs['environment_ids'] == [environment.id]
with mock.patch(
'sentry.api.serializers.models.group.snuba_tsdb.get_range',
side_effect=snuba_tsdb.get_range) as get_range:
serialize(
[group],
serializer=StreamGroupSerializerSnuba(
environment_ids=None,
stats_period='14d',
)
)
assert get_range.call_count == 1
for args, kwargs in get_range.call_args_list:
assert kwargs['environment_ids'] is None
def test_with_environment(self):
self.login_as(user=self.user)
team = self.create_team()
project1 = self.create_project(teams=[team], name='foo')
self.create_project(teams=[team], name='bar')
rule = project1.rule_set.all()[0]
rule.update(
environment_id=Environment.get_or_create(
rule.project,
'production',
).id,
)
url = reverse(
'sentry-api-0-project-rule-details',
kwargs={
'organization_slug': project1.organization.slug,
'project_slug': project1.slug,
'rule_id': rule.id,
}
)
response = self.client.get(url, format='json')
assert response.status_code == 200, response.content
assert response.data['id'] == six.text_type(rule.id)
assert response.data['environment'] == 'production'
def test_environment(self):
group = self.group
environment = Environment.get_or_create(group.project, 'production')
from sentry.api.serializers.models.group import tsdb
with mock.patch(
'sentry.api.serializers.models.group.tsdb.get_range',
side_effect=tsdb.get_range) as get_range:
serialize(
[group],
serializer=StreamGroupSerializer(
environment_func=lambda: environment,
stats_period='14d',
),
)
assert get_range.call_count == 1
for args, kwargs in get_range.call_args_list:
assert kwargs['environment_id'] == environment.id
def get_invalid_environment():
raise Environment.DoesNotExist()
with mock.patch(
'sentry.api.serializers.models.group.tsdb.make_series',
side_effect=tsdb.make_series) as make_series:
serialize(
[group],
serializer=StreamGroupSerializer(
environment_func=get_invalid_environment,
stats_period='14d',
)
)
assert make_series.call_count == 1
def create_environment(self, **kwargs):
project = kwargs.get('project', self.project)
name = kwargs.get('name', petname.Generate(1, ' ', letters=10))
return Environment.get_or_create(
project=project,
name=name
)
def test_simple(self):
project = self.create_project()
env = Environment.get_or_create(
project=project,
name='prod',
)
assert env.name == 'prod'
assert env.projects.first().id == project.id
env2 = Environment.get_or_create(
project=project,
name='prod',
)
assert env2.id == env.id
def test_environment(self):
self.login_as(user=self.user)
group = self.create_group()
events = {}
for name in ['production', 'development']:
environment = Environment.get_or_create(group.project, name)
tagstore.get_or_create_tag_key(
project_id=group.project_id,
environment_id=environment.id,
key='environment',
)
tagstore.create_tag_value(
project_id=group.project_id,
environment_id=environment.id,
key='environment',
value=name,
)
events[name] = event = self.create_event(
group=group,
tags={'environment': name},
)
tagstore.create_event_tags(
project_id=group.project_id,
group_id=group.id,
environment_id=environment.id,
event_id=event.id,
tags=[
('environment', name),
],
)
url = u'/api/0/issues/{}/events/'.format(group.id)
response = self.client.get(url + '?environment=production', format='json')
assert response.status_code == 200, response.content
assert set(map(lambda x: x['id'], response.data)) == set([
six.text_type(events['production'].id),
])
url = u'/api/0/issues/{}/events/'.format(group.id)
response = self.client.get(url + '?environment=invalid', format='json')
assert response.status_code == 200, response.content
assert response.data == []
url = u'/api/0/issues/{}/events/'.format(group.id)
response = self.client.get(
url + '?environment=production&query=environment:development',
format='json')
assert response.status_code == 200, response.content
assert response.data == []
def get_environment(self):
from sentry.models import Environment
if not hasattr(self, '_environment_cache'):
self._environment_cache = Environment.objects.get(
organization_id=self.project.organization_id,
name=Environment.get_name_or_default(self.get_tag('environment')),
)
return self._environment_cache
def get(self, request, project, environment):
try:
instance = EnvironmentProject.objects.select_related('environment').get(
project=project,
environment__name=Environment.get_name_from_path_segment(environment),
)
except EnvironmentProject.DoesNotExist:
raise ResourceDoesNotExist
return Response(serialize(instance, request.user))
def test_with_environment(self):
self.login_as(user=self.user)
project = self.create_project()
Environment.get_or_create(
project,
'production',
)
rule = Rule.objects.create(project=project, label='foo')
url = reverse(
'sentry-api-0-project-rule-details',
kwargs={
'organization_slug': project.organization.slug,
'project_slug': project.slug,
'rule_id': rule.id,
}
)
response = self.client.put(
url,
data={
'name': 'hello world',
'environment': 'production',
'actionMatch': 'any',
'actions': [],
'conditions': []
},
format='json'
)
assert response.status_code == 200, response.content
assert response.data['id'] == six.text_type(rule.id)
assert response.data['environment'] == 'production'
rule = Rule.objects.get(id=rule.id)
assert rule.label == 'hello world'
assert rule.environment_id == Environment.get_or_create(
rule.project,
'production',
).id
def _setup_tags_for_event(self, event):
tags = dict(event.data['tags'])
try:
environment = self.environments[tags['environment']]
except KeyError:
environment = self.environments[tags['environment']] = Environment.get_or_create(
event.project,
tags['environment'],
)
GroupEnvironment.objects.get_or_create(
environment_id=environment.id,
group_id=event.group_id,
)
for key, value in tags.items():
for environment_id in [AGGREGATE_ENVIRONMENT_ID, environment.id]:
tag_value, created = tagstore.get_or_create_group_tag_value(
project_id=event.project_id,
group_id=event.group_id,
environment_id=environment_id,
key=key,
value=value,
)
if created: # XXX: Hack for tagstore compat
tag_value.update(
times_seen=1,
first_seen=event.datetime,
last_seen=event.datetime,
)
else:
updates = {
'times_seen': tag_value.times_seen + 1,
}
if event.datetime < tag_value.first_seen:
updates['first_seen'] = event.datetime
if event.datetime > tag_value.last_seen:
updates['last_seen'] = event.datetime
if updates:
tag_value.update(**updates)
tagstore.create_event_tags(
project_id=event.project_id,
group_id=event.group_id,
environment_id=environment_id,
event_id=event.id,
tags=tags.items(),
date_added=event.datetime,
)
def test_get_environment(self):
environment = Environment.get_or_create(self.project, 'production')
event = self.create_event(
data={'tags': [
('environment', 'production'),
]}
)
event.get_environment() == environment
with self.assertNumQueries(0):
event.get_environment() == environment
def test_get_environment(self):
environment = Environment.get_or_create(self.project, 'production')
event = self.store_event(
data={
'environment': 'production'
},
project_id=self.project.id
)
assert event.get_environment() == environment
with self.assertNumQueries(0):
event.get_environment() == environment
def create_event(self, *args, **kwargs):
event = super(SnubaSearchTest, self).create_event(*args, **kwargs)
data = event.data.data
tags = dict(data.get('tags', []))
if tags['environment'] not in self.environments:
self.environments[tags['environment']] = Environment.get_or_create(
event.project,
tags['environment'],
)
return event
def _get_event_environment(event, project, cache):
from sentry.models import Environment
environment_name = event.get_tag('environment')
if environment_name not in cache:
try:
environment = Environment.get_for_organization_id(
project.organization_id, environment_name)
except Environment.DoesNotExist:
logger.warn(
'event.environment.does_not_exist',
extra={
'project_id': project.id,
'environment_name': environment_name,
}
)
environment = Environment.get_or_create(project, environment_name)
cache[environment_name] = environment
return cache[environment_name]
def _get_environment_from_request(self, request, organization_id):
if not hasattr(request, '_cached_environment'):
environment_param = request.GET.get('environment')
if environment_param is None:
environment = None
else:
environment = Environment.get_for_organization_id(
name=environment_param,
organization_id=organization_id,
)
request._cached_environment = environment
return request._cached_environment
def validate_environment(self, attrs, source):
name = attrs.get(source)
if name is None:
return attrs
try:
attrs['environment'] = Environment.get_for_organization_id(
self.context['project'].organization_id,
name,
).id
except Environment.DoesNotExist:
raise serializers.ValidationError(u'This environment has not been created.')
return attrs
def create_event(self, *args, **kwargs):
"""\
Takes the results from the existing `create_event` method and
inserts into the local test Snuba cluster so that tests can be
run against the same event data.
Note that we create a GroupHash as necessary because `create_event`
doesn't run them through the 'real' event pipeline. In a perfect
world all test events would go through the full regular pipeline.
"""
from sentry.event_manager import get_hashes_from_fingerprint, md5_from_hash
event = super(SnubaTestCase, self).create_event(*args, **kwargs)
data = event.data.data
tags = dict(data.get('tags', []))
if not data.get('received'):
data['received'] = calendar.timegm(event.datetime.timetuple())
if 'environment' in tags:
environment = Environment.get_or_create(
event.project,
tags['environment'],
)
GroupEnvironment.objects.get_or_create(
environment_id=environment.id,
group_id=event.group_id,
)
hashes = get_hashes_from_fingerprint(
event,
data.get('fingerprint', ['{{ default }}']),
)
primary_hash = md5_from_hash(hashes[0])
grouphash, _ = GroupHash.objects.get_or_create(
project=event.project,
group=event.group,
hash=primary_hash,
)
self.snuba_insert(self.__wrap_event(event, data, grouphash.hash))
return event
def test_transfer_to_team_releases(self):
from_org = self.create_organization()
from_team = self.create_team(organization=from_org)
to_org = self.create_organization()
to_team = self.create_team(organization=to_org)
project = self.create_project(teams=[from_team])
environment = Environment.get_or_create(project, 'production')
release = Release.get_or_create(project=project, version='1.0')
ReleaseProjectEnvironment.objects.create(
project=project,
release=release,
environment=environment,
)
assert ReleaseProjectEnvironment.objects.filter(
project=project,
release=release,
environment=environment,
).exists()
assert ReleaseProject.objects.filter(
project=project,
release=release,
).exists()
project.transfer_to(team=to_team)
project = Project.objects.get(id=project.id)
assert project.teams.count() == 1
assert project.teams.first() == to_team
assert project.organization_id == to_org.id
assert not ReleaseProjectEnvironment.objects.filter(
project=project,
release=release,
environment=environment,
).exists()
assert not ReleaseProject.objects.filter(
project=project,
release=release,
).exists()
def test_tsdb(self):
project = self.project
manager = EventManager(self.make_event(
fingerprint=['totally unique super duper fingerprint'],
environment='totally unique super duper environment',
))
event = manager.save(project)
def query(model, key, **kwargs):
return tsdb.get_sums(model, [key], event.datetime, event.datetime, **kwargs)[key]
assert query(tsdb.models.project, project.id) == 1
assert query(tsdb.models.group, event.group.id) == 1
environment_id = Environment.get_for_organization_id(
event.project.organization_id,
'totally unique super duper environment',
).id
assert query(tsdb.models.project, project.id, environment_id=environment_id) == 1
assert query(tsdb.models.group, event.group.id, environment_id=environment_id) == 1
def create_event(self, *args, **kwargs):
"""\
Takes the results from the existing `create_event` method and
inserts into the local test Snuba cluster so that tests can be
run against the same event data.
Note that we create a GroupHash as necessary because `create_event`
doesn't run them through the 'real' event pipeline. In a perfect
world all test events would go through the full regular pipeline.
"""
# XXX: Use `store_event` instead of this!
event = Factories.create_event(*args, **kwargs)
data = event.data.data
tags = dict(data.get('tags', []))
if not data.get('received'):
data['received'] = calendar.timegm(event.datetime.timetuple())
if 'environment' in tags:
environment = Environment.get_or_create(
event.project,
tags['environment'],
)
GroupEnvironment.objects.get_or_create(
environment_id=environment.id,
group_id=event.group_id,
)
primary_hash = event.get_primary_hash()
grouphash, _ = GroupHash.objects.get_or_create(
project=event.project,
group=event.group,
hash=primary_hash,
)
self.snuba_insert(self.__wrap_event(event, data, grouphash.hash))
return event
def put(self, request, project, environment):
try:
instance = EnvironmentProject.objects.select_related('environment').get(
project=project,
environment__name=Environment.get_name_from_path_segment(environment),
)
except EnvironmentProject.DoesNotExist:
raise ResourceDoesNotExist
serializer = ProjectEnvironmentSerializer(data=request.DATA, partial=True)
if not serializer.is_valid():
return Response(serializer.errors, status=400)
data = serializer.object
fields = {}
if 'isHidden' in data:
fields['is_hidden'] = data['isHidden']
if fields:
instance.update(**fields)
return Response(serialize(instance, request.user))
def test_simple(self):
org = self.create_organization()
commit = Commit.objects.create(organization_id=org.id, repository_id=5)
commit2 = Commit.objects.create(organization_id=org.id, repository_id=6)
# merge to
project = self.create_project(organization=org, name='foo')
environment = Environment.get_or_create(project=project, name='env1')
release = Release.objects.create(version='abcdabc', organization=org)
release.add_project(project)
release_commit = ReleaseCommit.objects.create(
organization_id=org.id, release=release, commit=commit, order=1
)
release_environment = ReleaseEnvironment.objects.create(
organization_id=org.id, project_id=project.id, release_id=release.id, environment_id=environment.id
)
release_project_environment = ReleaseProjectEnvironment.objects.create(
release_id=release.id, project_id=project.id, environment_id=environment.id
)
group_release = GroupRelease.objects.create(
project_id=project.id, release_id=release.id, group_id=1
)
group = self.create_group(project=project, first_release=release)
group_resolution = GroupResolution.objects.create(group=group, release=release)
# merge from #1
project2 = self.create_project(organization=org, name='bar')
environment2 = Environment.get_or_create(project=project2, name='env2')
release2 = Release.objects.create(version='bbbbbbb', organization=org)
release2.add_project(project2)
release_commit2 = ReleaseCommit.objects.create(
organization_id=org.id, release=release2, commit=commit, order=2
)
release_environment2 = ReleaseEnvironment.objects.create(
organization_id=org.id,
project_id=project2.id,
release_id=release2.id,
environment_id=environment2.id,
)
release_project_environment2 = ReleaseProjectEnvironment.objects.create(
release_id=release2.id, project_id=project2.id, environment_id=environment2.id
)
group_release2 = GroupRelease.objects.create(
project_id=project2.id, release_id=release2.id, group_id=2
)
group2 = self.create_group(project=project2, first_release=release2)
group_resolution2 = GroupResolution.objects.create(group=group2, release=release2)
# merge from #2
project3 = self.create_project(organization=org, name='baz')
environment3 = Environment.get_or_create(project=project3, name='env3')
release3 = Release.objects.create(version='cccccc', organization=org)
release3.add_project(project3)
release_commit3 = ReleaseCommit.objects.create(
organization_id=org.id, release=release2, commit=commit2, order=3
)
release_environment3 = ReleaseEnvironment.objects.create(
organization_id=org.id,
project_id=project3.id,
release_id=release3.id,
environment_id=environment3.id,
)
release_project_environment3 = ReleaseProjectEnvironment.objects.create(
release_id=release3.id, project_id=project3.id, environment_id=environment3.id
)
group_release3 = GroupRelease.objects.create(
project_id=project3.id, release_id=release3.id, group_id=3
)
group3 = self.create_group(project=project3, first_release=release3)
group_resolution3 = GroupResolution.objects.create(group=group3, release=release3)
Release.merge(release, [release2, release3])
# ReleaseCommit.release
assert ReleaseCommit.objects.get(id=release_commit.id).release == release
# should not exist because they referenced the same commit
assert not ReleaseCommit.objects.filter(id=release_commit2.id).exists()
assert ReleaseCommit.objects.get(id=release_commit3.id).release == release
# ReleaseEnvironment.release_id
assert ReleaseEnvironment.objects.get(id=release_environment.id).release_id == release.id
assert ReleaseEnvironment.objects.get(id=release_environment2.id).release_id == release.id
assert ReleaseEnvironment.objects.get(id=release_environment3.id).release_id == release.id
# ReleaseProject.release
assert release.projects.count() == 3
assert ReleaseProject.objects.filter(release=release, project=project).exists()
assert ReleaseProject.objects.filter(release=release, project=project2).exists()
assert ReleaseProject.objects.filter(release=release, project=project3).exists()
# ReleaseProjectEnvironment.release
assert ReleaseProjectEnvironment.objects.get(
id=release_project_environment.id).release_id == release.id
assert ReleaseProjectEnvironment.objects.get(
id=release_project_environment2.id).release_id == release.id
assert ReleaseProjectEnvironment.objects.get(
id=release_project_environment3.id).release_id == release.id
# GroupRelease.release_id
assert GroupRelease.objects.get(id=group_release.id).release_id == release.id
assert GroupRelease.objects.get(id=group_release2.id).release_id == release.id
assert GroupRelease.objects.get(id=group_release3.id).release_id == release.id
# GroupResolution.release
assert GroupResolution.objects.get(id=group_resolution.id).release == release
assert GroupResolution.objects.get(id=group_resolution2.id).release == release
assert GroupResolution.objects.get(id=group_resolution3.id).release == release
# Group.first_release
assert Group.objects.get(id=group.id).first_release == release
assert Group.objects.get(id=group2.id).first_release == release
assert Group.objects.get(id=group3.id).first_release == release
# Releases are gone
assert Release.objects.filter(id=release.id).exists()
assert not Release.objects.filter(id=release2.id).exists()
assert not Release.objects.filter(id=release3.id).exists()
def _get_or_create_environment_many(jobs, projects):
for job in jobs:
job["environment"] = Environment.get_or_create(
project=projects[job["project_id"]], name=job["environment"]
)
def test_simple(self):
org = self.create_organization()
commit = Commit.objects.create(organization_id=org.id, repository_id=5)
commit2 = Commit.objects.create(organization_id=org.id,
repository_id=6)
# merge to
project = self.create_project(organization=org, name="foo")
environment = Environment.get_or_create(project=project, name="env1")
release = Release.objects.create(version="abcdabc", organization=org)
release.add_project(project)
release_commit = ReleaseCommit.objects.create(organization_id=org.id,
release=release,
commit=commit,
order=1)
release_environment = ReleaseEnvironment.objects.create(
organization_id=org.id,
project_id=project.id,
release_id=release.id,
environment_id=environment.id,
)
release_project_environment = ReleaseProjectEnvironment.objects.create(
release_id=release.id,
project_id=project.id,
environment_id=environment.id)
group_release = GroupRelease.objects.create(project_id=project.id,
release_id=release.id,
group_id=1)
group = self.create_group(project=project, first_release=release)
group_resolution = GroupResolution.objects.create(group=group,
release=release)
# merge from #1
project2 = self.create_project(organization=org, name="bar")
environment2 = Environment.get_or_create(project=project2, name="env2")
release2 = Release.objects.create(version="bbbbbbb", organization=org)
release2.add_project(project2)
release_commit2 = ReleaseCommit.objects.create(organization_id=org.id,
release=release2,
commit=commit,
order=2)
release_environment2 = ReleaseEnvironment.objects.create(
organization_id=org.id,
project_id=project2.id,
release_id=release2.id,
environment_id=environment2.id,
)
release_project_environment2 = ReleaseProjectEnvironment.objects.create(
release_id=release2.id,
project_id=project2.id,
environment_id=environment2.id)
group_release2 = GroupRelease.objects.create(project_id=project2.id,
release_id=release2.id,
group_id=2)
group2 = self.create_group(project=project2, first_release=release2)
group_resolution2 = GroupResolution.objects.create(group=group2,
release=release2)
# merge from #2
project3 = self.create_project(organization=org, name="baz")
environment3 = Environment.get_or_create(project=project3, name="env3")
release3 = Release.objects.create(version="cccccc", organization=org)
release3.add_project(project3)
release_commit3 = ReleaseCommit.objects.create(organization_id=org.id,
release=release2,
commit=commit2,
order=3)
release_environment3 = ReleaseEnvironment.objects.create(
organization_id=org.id,
project_id=project3.id,
release_id=release3.id,
environment_id=environment3.id,
)
release_project_environment3 = ReleaseProjectEnvironment.objects.create(
release_id=release3.id,
project_id=project3.id,
environment_id=environment3.id)
group_release3 = GroupRelease.objects.create(project_id=project3.id,
release_id=release3.id,
group_id=3)
group3 = self.create_group(project=project3, first_release=release3)
group_resolution3 = GroupResolution.objects.create(group=group3,
release=release3)
Release.merge(release, [release2, release3])
# ReleaseCommit.release
assert ReleaseCommit.objects.get(
id=release_commit.id).release == release
# should not exist because they referenced the same commit
assert not ReleaseCommit.objects.filter(id=release_commit2.id).exists()
assert ReleaseCommit.objects.get(
id=release_commit3.id).release == release
# ReleaseEnvironment.release_id
assert ReleaseEnvironment.objects.get(
id=release_environment.id).release_id == release.id
assert ReleaseEnvironment.objects.get(
id=release_environment2.id).release_id == release.id
assert ReleaseEnvironment.objects.get(
id=release_environment3.id).release_id == release.id
# ReleaseProject.release
assert release.projects.count() == 3
assert ReleaseProject.objects.filter(release=release,
project=project).exists()
assert ReleaseProject.objects.filter(release=release,
project=project2).exists()
assert ReleaseProject.objects.filter(release=release,
project=project3).exists()
# ReleaseProjectEnvironment.release
assert (ReleaseProjectEnvironment.objects.get(
id=release_project_environment.id).release_id == release.id)
assert (ReleaseProjectEnvironment.objects.get(
id=release_project_environment2.id).release_id == release.id)
assert (ReleaseProjectEnvironment.objects.get(
id=release_project_environment3.id).release_id == release.id)
# GroupRelease.release_id
assert GroupRelease.objects.get(
id=group_release.id).release_id == release.id
assert GroupRelease.objects.get(
id=group_release2.id).release_id == release.id
assert GroupRelease.objects.get(
id=group_release3.id).release_id == release.id
# GroupResolution.release
assert GroupResolution.objects.get(
id=group_resolution.id).release == release
assert GroupResolution.objects.get(
id=group_resolution2.id).release == release
assert GroupResolution.objects.get(
id=group_resolution3.id).release == release
# Group.first_release
assert Group.objects.get(id=group.id).first_release == release
assert Group.objects.get(id=group2.id).first_release == release
assert Group.objects.get(id=group3.id).first_release == release
# Releases are gone
assert Release.objects.filter(id=release.id).exists()
assert not Release.objects.filter(id=release2.id).exists()
assert not Release.objects.filter(id=release3.id).exists()
def save(self, project, raw=False):
from sentry.tasks.post_process import index_event_tags
project = Project.objects.get_from_cache(id=project)
data = self.data.copy()
# First we pull out our top-level (non-data attr) kwargs
event_id = data.pop('event_id')
level = data.pop('level')
culprit = data.pop('culprit', None)
logger_name = data.pop('logger', None)
server_name = data.pop('server_name', None)
site = data.pop('site', None)
checksum = data.pop('checksum', None)
fingerprint = data.pop('fingerprint', None)
platform = data.pop('platform', None)
release = data.pop('release', None)
environment = data.pop('environment', None)
# unused
time_spent = data.pop('time_spent', None)
message = data.pop('message', '')
if not culprit:
culprit = generate_culprit(data, platform=platform)
date = datetime.fromtimestamp(data.pop('timestamp'))
date = date.replace(tzinfo=timezone.utc)
kwargs = {
'platform': platform,
}
event = Event(project_id=project.id,
event_id=event_id,
data=data,
time_spent=time_spent,
datetime=date,
**kwargs)
tags = data.get('tags') or []
tags.append(('level', LOG_LEVELS[level]))
if logger_name:
tags.append(('logger', logger_name))
if server_name:
tags.append(('server_name', server_name))
if site:
tags.append(('site', site))
if release:
# TODO(dcramer): we should ensure we create Release objects
tags.append(('sentry:release', release))
if environment:
tags.append(('environment', environment))
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags,
event,
_with_transaction=False)
if added_tags:
tags.extend(added_tags)
event_user = self._get_event_user(project, data)
if event_user:
tags.append(('sentry:user', event_user.tag_value))
# XXX(dcramer): we're relying on mutation of the data object to ensure
# this propagates into Event
data['tags'] = tags
data['fingerprint'] = fingerprint or ['{{ default }}']
for path, iface in event.interfaces.iteritems():
data['tags'].extend(iface.iter_tags())
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.get_path(), None)
# prioritize fingerprint over checksum as its likely the client defaulted
# a checksum whereas the fingerprint was explicit
if fingerprint:
hashes = map(md5_from_hash,
get_hashes_from_fingerprint(event, fingerprint))
elif checksum:
hashes = [checksum]
else:
hashes = map(md5_from_hash, get_hashes_for_event(event))
# TODO(dcramer): temp workaround for complexity
data['message'] = message
event_type = eventtypes.get(data.get('type', 'default'))(data)
event_metadata = event_type.get_metadata()
# TODO(dcramer): temp workaround for complexity
del data['message']
data['type'] = event_type.key
data['metadata'] = event_metadata
# index components into ``Event.message``
# See GH-3248
if event_type.key != 'default':
if 'sentry.interfaces.Message' in data and \
data['sentry.interfaces.Message']['message'] != message:
message = u'{} {}'.format(
message,
data['sentry.interfaces.Message']['message'],
)
if not message:
message = ''
elif not isinstance(message, basestring):
message = force_text(message)
for value in event_metadata.itervalues():
value_u = force_text(value, errors='replace')
if value_u not in message:
message = u'{} {}'.format(message, value_u)
message = trim(message.strip(), settings.SENTRY_MAX_MESSAGE_LENGTH)
event.message = message
kwargs['message'] = message
group_kwargs = kwargs.copy()
group_kwargs.update({
'culprit': culprit,
'logger': logger_name,
'level': level,
'last_seen': date,
'first_seen': date,
'data': {
'last_received':
event.data.get('received')
or float(event.datetime.strftime('%s')),
'type':
event_type.key,
# we cache the events metadata on the group to ensure its
# accessible in the stream
'metadata':
event_metadata,
},
})
if release:
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
group_kwargs['first_release'] = release
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **group_kwargs)
event.group = group
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
try:
with transaction.atomic(using=router.db_for_write(EventMapping)):
EventMapping.objects.create(project=project,
group=group,
event_id=event_id)
except IntegrityError:
self.logger.info('Duplicate EventMapping found for event_id=%s',
event_id,
exc_info=True)
return event
environment = Environment.get_or_create(
project=project,
name=environment,
)
if release:
ReleaseEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date,
)
tsdb.incr_multi([
(tsdb.models.group, group.id),
(tsdb.models.project, project.id),
],
timestamp=event.datetime)
frequencies = [
# (tsdb.models.frequent_projects_by_organization, {
# project.organization_id: {
# project.id: 1,
# },
# }),
# (tsdb.models.frequent_issues_by_project, {
# project.id: {
# group.id: 1,
# },
# })
(tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1,
},
})
]
if release:
frequencies.append((tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1,
},
}))
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(group=group)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
self.logger.info('Duplicate Event found for event_id=%s',
event_id,
exc_info=True)
return event
index_event_tags.delay(
project_id=project.id,
group_id=group.id,
event_id=event.id,
tags=tags,
)
if event_user:
tsdb.record_multi((
(tsdb.models.users_affected_by_group, group.id,
(event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id,
(event_user.tag_value, )),
),
timestamp=event.datetime)
if is_new and release:
buffer.incr(Release, {'new_groups': 1}, {
'id': release.id,
})
safe_execute(Group.objects.add_tags,
group,
tags,
_with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send(project=project,
group=group,
sender=Project)
post_process_group.delay(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
)
else:
self.logger.info(
'Raw event passed; skipping post process for event_id=%s',
event_id)
# TODO: move this to the queue
if is_regression and not raw:
regression_signal.send_robust(sender=Group, instance=group)
return event
def save(self, project, raw=False):
from sentry.tasks.post_process import index_event_tags
data = self.data
project = Project.objects.get_from_cache(id=project)
# Check to make sure we're not about to do a bunch of work that's
# already been done if we've processed an event with this ID. (This
# isn't a perfect solution -- this doesn't handle ``EventMapping`` and
# there's a race condition between here and when the event is actually
# saved, but it's an improvement. See GH-7677.)
try:
event = Event.objects.get(
project_id=project.id,
event_id=data['event_id'],
)
except Event.DoesNotExist:
pass
else:
self.logger.info(
'duplicate.found',
exc_info=True,
extra={
'event_uuid': data['event_id'],
'project_id': project.id,
'model': Event.__name__,
}
)
return event
# First we pull out our top-level (non-data attr) kwargs
event_id = data.pop('event_id')
level = data.pop('level')
transaction_name = data.pop('transaction', None)
culprit = data.pop('culprit', None)
logger_name = data.pop('logger', None)
server_name = data.pop('server_name', None)
site = data.pop('site', None)
checksum = data.pop('checksum', None)
fingerprint = data.pop('fingerprint', None)
platform = data.pop('platform', None)
release = data.pop('release', None)
dist = data.pop('dist', None)
environment = data.pop('environment', None)
# unused
time_spent = data.pop('time_spent', None)
message = data.pop('message', '')
if not culprit:
if transaction_name:
culprit = transaction_name
else:
culprit = generate_culprit(data, platform=platform)
culprit = force_text(culprit)
if transaction_name:
transaction_name = force_text(transaction_name)
recorded_timestamp = data.pop('timestamp')
date = datetime.fromtimestamp(recorded_timestamp)
date = date.replace(tzinfo=timezone.utc)
kwargs = {
'platform': platform,
}
event = Event(
project_id=project.id,
event_id=event_id,
data=data,
time_spent=time_spent,
datetime=date,
**kwargs
)
event._project_cache = project
data = event.data.data
# convert this to a dict to ensure we're only storing one value per key
# as most parts of Sentry dont currently play well with multiple values
tags = dict(data.get('tags') or [])
tags['level'] = LOG_LEVELS[level]
if logger_name:
tags['logger'] = logger_name
if server_name:
tags['server_name'] = server_name
if site:
tags['site'] = site
if environment:
tags['environment'] = environment
if transaction_name:
tags['transaction'] = transaction_name
if release:
# dont allow a conflicting 'release' tag
if 'release' in tags:
del tags['release']
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
tags['sentry:release'] = release.version
if dist and release:
dist = release.add_dist(dist, date)
tags['sentry:dist'] = dist.name
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
if 'user' in tags:
del tags['user']
tags['sentry:user'] = event_user.tag_value
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
normalize_in_app(data)
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags, event, _with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
tags.setdefault(key, value)
for path, iface in six.iteritems(event.interfaces):
for k, v in iface.iter_tags():
tags[k] = v
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.get_path(), None)
# tags are stored as a tuple
tags = tags.items()
data['tags'] = tags
data['fingerprint'] = fingerprint or ['{{ default }}']
# prioritize fingerprint over checksum as its likely the client defaulted
# a checksum whereas the fingerprint was explicit
if fingerprint:
hashes = [md5_from_hash(h) for h in get_hashes_from_fingerprint(event, fingerprint)]
elif checksum:
if HASH_RE.match(checksum):
hashes = [checksum]
else:
hashes = [md5_from_hash([checksum]), checksum]
data['checksum'] = checksum
else:
hashes = [md5_from_hash(h) for h in get_hashes_for_event(event)]
# TODO(dcramer): temp workaround for complexity
data['message'] = message
event_type = eventtypes.get(data.get('type', 'default'))(data)
event_metadata = event_type.get_metadata()
# TODO(dcramer): temp workaround for complexity
del data['message']
data['type'] = event_type.key
data['metadata'] = event_metadata
# index components into ``Event.message``
# See GH-3248
if event_type.key != 'default':
if 'sentry.interfaces.Message' in data and \
data['sentry.interfaces.Message']['message'] != message:
message = u'{} {}'.format(
message,
data['sentry.interfaces.Message']['message'],
)
if not message:
message = ''
elif not isinstance(message, six.string_types):
message = force_text(message)
for value in six.itervalues(event_metadata):
value_u = force_text(value, errors='replace')
if value_u not in message:
message = u'{} {}'.format(message, value_u)
if culprit and culprit not in message:
culprit_u = force_text(culprit, errors='replace')
message = u'{} {}'.format(message, culprit_u)
message = trim(message.strip(), settings.SENTRY_MAX_MESSAGE_LENGTH)
event.message = message
kwargs['message'] = message
received_timestamp = event.data.get('received') or float(event.datetime.strftime('%s'))
group_kwargs = kwargs.copy()
group_kwargs.update(
{
'culprit': culprit,
'logger': logger_name,
'level': level,
'last_seen': date,
'first_seen': date,
'active_at': date,
'data': {
'last_received': received_timestamp,
'type':
event_type.key,
# we cache the events metadata on the group to ensure its
# accessible in the stream
'metadata':
event_metadata,
},
}
)
if release:
group_kwargs['first_release'] = release
try:
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **group_kwargs
)
except HashDiscarded:
event_discarded.send_robust(
project=project,
sender=EventManager,
)
metrics.incr(
'events.discarded',
skip_internal=True,
tags={
'organization_id': project.organization_id,
'platform': platform,
},
)
raise
else:
event_saved.send_robust(
project=project,
event_size=event.size,
sender=EventManager,
)
event.group = group
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
# When an event was sampled, the canonical source of truth
# is the EventMapping table since we aren't going to be writing out an actual
# Event row. Otherwise, if the Event isn't being sampled, we can safely
# rely on the Event table itself as the source of truth and ignore
# EventMapping since it's redundant information.
if is_sample:
try:
with transaction.atomic(using=router.db_for_write(EventMapping)):
EventMapping.objects.create(project=project, group=group, event_id=event_id)
except IntegrityError:
self.logger.info(
'duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': EventMapping.__name__,
}
)
return event
environment = Environment.get_or_create(
project=project,
name=environment,
)
group_environment, is_new_group_environment = GroupEnvironment.get_or_create(
group_id=group.id,
environment_id=environment.id,
defaults={
'first_release_id': release.id if release else None,
},
)
if release:
ReleaseEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
ReleaseProjectEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date,
)
counters = [
(tsdb.models.group, group.id),
(tsdb.models.project, project.id),
]
if release:
counters.append((tsdb.models.release, release.id))
tsdb.incr_multi(counters, timestamp=event.datetime, environment_id=environment.id)
frequencies = [
# (tsdb.models.frequent_projects_by_organization, {
# project.organization_id: {
# project.id: 1,
# },
# }),
# (tsdb.models.frequent_issues_by_project, {
# project.id: {
# group.id: 1,
# },
# })
(tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1,
},
})
]
if release:
frequencies.append(
(tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1,
},
})
)
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(
group=group,
environment=environment,
)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
self.logger.info(
'duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': Event.__name__,
}
)
return event
index_event_tags.delay(
organization_id=project.organization_id,
project_id=project.id,
group_id=group.id,
environment_id=environment.id,
event_id=event.id,
tags=tags,
date_added=event.datetime,
)
if event_user:
tsdb.record_multi(
(
(tsdb.models.users_affected_by_group, group.id, (event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id, (event_user.tag_value, )),
),
timestamp=event.datetime,
environment_id=environment.id,
)
if release:
if is_new:
buffer.incr(
ReleaseProject, {'new_groups': 1}, {
'release_id': release.id,
'project_id': project.id,
}
)
if is_new_group_environment:
buffer.incr(
ReleaseProjectEnvironment, {'new_issues_count': 1}, {
'project_id': project.id,
'release_id': release.id,
'environment_id': environment.id,
}
)
safe_execute(Group.objects.add_tags, group, environment, tags, _with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send_robust(project=project, group=group, sender=Project)
eventstream.insert(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
is_new_group_environment=is_new_group_environment,
primary_hash=hashes[0],
# We are choosing to skip consuming the event back
# in the eventstream if it's flagged as raw.
# This means that we want to publish the event
# through the event stream, but we don't care
# about post processing and handling the commit.
skip_consume=raw,
)
metrics.timing(
'events.latency',
received_timestamp - recorded_timestamp,
tags={
'project_id': project.id,
},
)
return event
def post(self, request, organization, version):
"""
Create a Deploy
```````````````
Create a deploy for a given release.
:pparam string organization_slug: the organization short name
:pparam string version: the version identifier of the release.
:param string environment: the environment you're deploying to
:param string name: the optional name of the deploy
:param url url: the optional url that points to the deploy
:param datetime dateStarted: an optional date that indicates when
the deploy started
:param datetime dateFinished: an optional date that indicates when
the deploy ended. If not provided, the
current time is used.
"""
try:
release = Release.objects.get(
version=version,
organization=organization,
)
except Release.DoesNotExist:
raise ResourceDoesNotExist
if not self.has_release_permission(request, organization, release):
raise PermissionDenied
serializer = DeploySerializer(data=request.DATA)
if serializer.is_valid():
result = serializer.object
try:
env = Environment.objects.get(
organization_id=organization.id,
name=result['environment'],
)
except Environment.DoesNotExist:
# TODO(jess): clean up when changing unique constraint
lock_key = Environment.get_lock_key(organization.id, result['environment'])
lock = locks.get(lock_key, duration=5)
with TimedRetryPolicy(10)(lock.acquire):
try:
env = Environment.objects.get(
organization_id=organization.id,
name=result['environment'],
)
except Environment.DoesNotExist:
env = Environment.objects.create(
organization_id=organization.id,
name=result['environment'],
)
try:
with transaction.atomic():
deploy, created = Deploy.objects.create(
organization_id=organization.id,
release=release,
environment_id=env.id,
date_finished=result.get('dateFinished', timezone.now()),
date_started=result.get('dateStarted'),
name=result.get('name'),
url=result.get('url'),
), True
except IntegrityError:
deploy, created = Deploy.objects.get(
organization_id=organization.id,
release=release,
environment_id=env.id,
), False
deploy.update(
date_finished=result.get('dateFinished', timezone.now()),
date_started=result.get('dateStarted'),
)
activity = None
for project in release.projects.all():
activity = Activity.objects.create(
type=Activity.DEPLOY,
project=project,
ident=release.version,
data={
'version': release.version,
'deploy_id': deploy.id,
'environment': env.name
},
datetime=deploy.date_finished,
)
# Somewhat hacky, only send notification for one
# Deploy Activity record because it will cover all projects
if activity is not None:
activity.send_notification()
# This is the closest status code that makes sense, and we want
# a unique 2xx response code so people can understand when
# behavior differs.
# 208 Already Reported (WebDAV; RFC 5842)
status = 201 if created else 208
return Response(serialize(deploy, request.user), status=status)
return Response(serializer.errors, status=400)
def validate_environment(self, attrs, source):
value = attrs[source]
if not Environment.is_valid_name(value):
raise serializers.ValidationError('Invalid value for environment')
return attrs
def save(self,
project_id,
raw=False,
assume_normalized=False,
cache_key=None):
"""
We re-insert events with duplicate IDs into Snuba, which is responsible
for deduplicating events. Since deduplication in Snuba is on the primary
key (based on event ID, project ID and day), events with same IDs are only
deduplicated if their timestamps fall on the same day. The latest event
always wins and overwrites the value of events received earlier in that day.
Since we increment counters and frequencies here before events get inserted
to eventstream these numbers may be larger than the total number of
events if we receive duplicate event IDs that fall on the same day
(that do not hit cache first).
"""
# Normalize if needed
if not self._normalized:
if not assume_normalized:
self.normalize()
self._normalized = True
data = self._data
with metrics.timer("event_manager.save.project.get_from_cache"):
project = Project.objects.get_from_cache(id=project_id)
with metrics.timer("event_manager.save.organization.get_from_cache"):
project._organization_cache = Organization.objects.get_from_cache(
id=project.organization_id)
# Pull out the culprit
culprit = self.get_culprit()
# Pull the toplevel data we're interested in
level = data.get("level")
# TODO(mitsuhiko): this code path should be gone by July 2018.
# This is going to be fine because no code actually still depends
# on integers here. When we need an integer it will be converted
# into one later. Old workers used to send integers here.
if level is not None and isinstance(level, six.integer_types):
level = LOG_LEVELS[level]
transaction_name = data.get("transaction")
logger_name = data.get("logger")
release = data.get("release")
dist = data.get("dist")
environment = data.get("environment")
recorded_timestamp = data.get("timestamp")
# We need to swap out the data with the one internal to the newly
# created event object
event = self._get_event_instance(project_id=project_id)
self._data = data = event.data.data
event._project_cache = project
date = event.datetime
platform = event.platform
event_id = event.event_id
if transaction_name:
transaction_name = force_text(transaction_name)
# Right now the event type is the signal to skip the group. This
# is going to change a lot.
if event.get_event_type() == "transaction":
issueless_event = True
else:
issueless_event = False
# Some of the data that are toplevel attributes are duplicated
# into tags (logger, level, environment, transaction). These are
# different from legacy attributes which are normalized into tags
# ahead of time (site, server_name).
setdefault_path(data, "tags", value=[])
set_tag(data, "level", level)
if logger_name:
set_tag(data, "logger", logger_name)
if environment:
set_tag(data, "environment", environment)
if transaction_name:
set_tag(data, "transaction", transaction_name)
if release:
# dont allow a conflicting 'release' tag
pop_tag(data, "release")
release = Release.get_or_create(project=project,
version=release,
date_added=date)
set_tag(data, "sentry:release", release.version)
if dist and release:
dist = release.add_dist(dist, date)
# dont allow a conflicting 'dist' tag
pop_tag(data, "dist")
set_tag(data, "sentry:dist", dist.name)
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
pop_tag(data, "user")
set_tag(data, "sentry:user", event_user.tag_value)
with metrics.timer("event_manager.load_grouping_config"):
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
grouping_config = load_grouping_config(
get_grouping_config_dict_for_event_data(data, project))
with metrics.timer("event_manager.normalize_stacktraces_for_grouping"):
normalize_stacktraces_for_grouping(data, grouping_config)
with metrics.timer("event_manager.plugins"):
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags,
event,
_with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
if get_tag(data, key) is None:
set_tag(data, key, value)
with metrics.timer("event_manager.set_tags"):
for path, iface in six.iteritems(event.interfaces):
for k, v in iface.iter_tags():
set_tag(data, k, v)
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.path, None)
with metrics.timer("event_manager.apply_server_fingerprinting"):
# The active grouping config was put into the event in the
# normalize step before. We now also make sure that the
# fingerprint was set to `'{{ default }}' just in case someone
# removed it from the payload. The call to get_hashes will then
# look at `grouping_config` to pick the right parameters.
data["fingerprint"] = data.get("fingerprint") or ["{{ default }}"]
apply_server_fingerprinting(
data, get_fingerprinting_config_for_project(project))
with metrics.timer("event_manager.event.get_hashes"):
# Here we try to use the grouping config that was requested in the
# event. If that config has since been deleted (because it was an
# experimental grouping config) we fall back to the default.
try:
hashes = event.get_hashes()
except GroupingConfigNotFound:
data["grouping_config"] = get_grouping_config_dict_for_project(
project)
hashes = event.get_hashes()
data["hashes"] = hashes
with metrics.timer("event_manager.materialize_metadata"):
# we want to freeze not just the metadata and type in but also the
# derived attributes. The reason for this is that we push this
# data into kafka for snuba processing and our postprocessing
# picks up the data right from the snuba topic. For most usage
# however the data is dynamically overridden by Event.title and
# Event.location (See Event.as_dict)
materialized_metadata = self.materialize_metadata()
data.update(materialized_metadata)
data["culprit"] = culprit
received_timestamp = event.data.get("received") or float(
event.datetime.strftime("%s"))
if not issueless_event:
# The group gets the same metadata as the event when it's flushed but
# additionally the `last_received` key is set. This key is used by
# _save_aggregate.
group_metadata = dict(materialized_metadata)
group_metadata["last_received"] = received_timestamp
kwargs = {
"platform": platform,
"message": event.search_message,
"culprit": culprit,
"logger": logger_name,
"level": LOG_LEVELS_MAP.get(level),
"last_seen": date,
"first_seen": date,
"active_at": date,
"data": group_metadata,
}
if release:
kwargs["first_release"] = release
try:
group, is_new, is_regression = self._save_aggregate(
event=event, hashes=hashes, release=release, **kwargs)
except HashDiscarded:
event_discarded.send_robust(project=project,
sender=EventManager)
metrics.incr(
"events.discarded",
skip_internal=True,
tags={
"organization_id": project.organization_id,
"platform": platform
},
)
raise
event.group = group
else:
group = None
is_new = False
is_regression = False
with metrics.timer("event_manager.event_saved_signal"):
event_saved.send_robust(project=project,
event_size=event.size,
sender=EventManager)
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
environment = Environment.get_or_create(project=project,
name=environment)
if group:
group_environment, is_new_group_environment = GroupEnvironment.get_or_create(
group_id=group.id,
environment_id=environment.id,
defaults={"first_release": release if release else None},
)
else:
is_new_group_environment = False
if release:
ReleaseEnvironment.get_or_create(project=project,
release=release,
environment=environment,
datetime=date)
ReleaseProjectEnvironment.get_or_create(project=project,
release=release,
environment=environment,
datetime=date)
if group:
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date)
counters = [(tsdb.models.project, project.id)]
if group:
counters.append((tsdb.models.group, group.id))
if release:
counters.append((tsdb.models.release, release.id))
with metrics.timer("event_manager.tsdb_incr_group_and_release_counters"
) as metrics_tags:
metrics_tags["has_group"] = "true" if group else "false"
tsdb.incr_multi(counters,
timestamp=event.datetime,
environment_id=environment.id)
frequencies = []
if group:
frequencies.append((tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1
}
}))
if release:
frequencies.append((tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1
}
}))
if frequencies:
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
if group:
UserReport.objects.filter(project=project,
event_id=event_id).update(
group=group, environment=environment)
# Enusre the _metrics key exists. This is usually created during
# and prefilled with ingestion sizes.
event_metrics = event.data.get("_metrics") or {}
event.data["_metrics"] = event_metrics
# Capture the actual size that goes into node store.
event_metrics["bytes.stored.event"] = len(
json.dumps(dict(event.data.items())))
if not issueless_event:
# Load attachments first, but persist them at the very last after
# posting to eventstream to make sure all counters and eventstream are
# incremented for sure.
attachments = self.get_attachments(cache_key, event)
for attachment in attachments:
key = "bytes.stored.%s" % (attachment.type, )
event_metrics[key] = (event_metrics.get(key) or 0) + len(
attachment.data)
# Write the event to Nodestore
event.data.save()
if event_user:
counters = [(tsdb.models.users_affected_by_project, project.id,
(event_user.tag_value, ))]
if group:
counters.append((tsdb.models.users_affected_by_group, group.id,
(event_user.tag_value, )))
with metrics.timer("event_manager.tsdb_record_users_affected"
) as metrics_tags:
metrics_tags["has_group"] = "true" if group else "false"
tsdb.record_multi(counters,
timestamp=event.datetime,
environment_id=environment.id)
if release:
if is_new:
buffer.incr(
ReleaseProject,
{"new_groups": 1},
{
"release_id": release.id,
"project_id": project.id
},
)
if is_new_group_environment:
buffer.incr(
ReleaseProjectEnvironment,
{"new_issues_count": 1},
{
"project_id": project.id,
"release_id": release.id,
"environment_id": environment.id,
},
)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send_robust(project=project,
event=event,
sender=Project)
with metrics.timer("event_manager.eventstream.insert"):
eventstream.insert(
group=group,
event=event,
is_new=is_new,
is_regression=is_regression,
is_new_group_environment=is_new_group_environment,
primary_hash=hashes[0],
# We are choosing to skip consuming the event back
# in the eventstream if it's flagged as raw.
# This means that we want to publish the event
# through the event stream, but we don't care
# about post processing and handling the commit.
skip_consume=raw,
)
if not issueless_event:
# Do this last to ensure signals get emitted even if connection to the
# file store breaks temporarily.
self.save_attachments(attachments, event)
metric_tags = {"from_relay": "_relay_processed" in self._data}
metrics.timing("events.latency",
received_timestamp - recorded_timestamp,
tags=metric_tags)
metrics.timing("events.size.data.post_save",
event.size,
tags=metric_tags)
metrics.incr(
"events.post_save.normalize.errors",
amount=len(self._data.get("errors") or ()),
tags=metric_tags,
)
return event
def save(self, project_id, raw=False, assume_normalized=False):
# Normalize if needed
if not self._normalized:
if not assume_normalized:
self.normalize()
self._normalized = True
data = self._data
project = Project.objects.get_from_cache(id=project_id)
project._organization_cache = Organization.objects.get_from_cache(
id=project.organization_id)
# Check to make sure we're not about to do a bunch of work that's
# already been done if we've processed an event with this ID. (This
# isn't a perfect solution -- this doesn't handle ``EventMapping`` and
# there's a race condition between here and when the event is actually
# saved, but it's an improvement. See GH-7677.)
try:
event = Event.objects.get(
project_id=project.id,
event_id=data['event_id'],
)
except Event.DoesNotExist:
pass
else:
# Make sure we cache on the project before returning
event._project_cache = project
logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': data['event_id'],
'project_id': project.id,
'model': Event.__name__,
})
return event
# Pull out the culprit
culprit = self.get_culprit()
# Pull the toplevel data we're interested in
level = data.get('level')
# TODO(mitsuhiko): this code path should be gone by July 2018.
# This is going to be fine because no code actually still depends
# on integers here. When we need an integer it will be converted
# into one later. Old workers used to send integers here.
if level is not None and isinstance(level, six.integer_types):
level = LOG_LEVELS[level]
transaction_name = data.get('transaction')
logger_name = data.get('logger')
release = data.get('release')
dist = data.get('dist')
environment = data.get('environment')
recorded_timestamp = data.get('timestamp')
# We need to swap out the data with the one internal to the newly
# created event object
event = self._get_event_instance(project_id=project_id)
self._data = data = event.data.data
event._project_cache = project
date = event.datetime
platform = event.platform
event_id = event.event_id
if transaction_name:
transaction_name = force_text(transaction_name)
# Some of the data that are toplevel attributes are duplicated
# into tags (logger, level, environment, transaction). These are
# different from legacy attributes which are normalized into tags
# ahead of time (site, server_name).
setdefault_path(data, 'tags', value=[])
set_tag(data, 'level', level)
if logger_name:
set_tag(data, 'logger', logger_name)
if environment:
set_tag(data, 'environment', environment)
if transaction_name:
set_tag(data, 'transaction', transaction_name)
if release:
# dont allow a conflicting 'release' tag
pop_tag(data, 'release')
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
set_tag(data, 'sentry:release', release.version)
if dist and release:
dist = release.add_dist(dist, date)
# dont allow a conflicting 'dist' tag
pop_tag(data, 'dist')
set_tag(data, 'sentry:dist', dist.name)
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
pop_tag(data, 'user')
set_tag(data, 'sentry:user', event_user.tag_value)
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
grouping_config = load_grouping_config(
get_grouping_config_dict_for_event_data(data, project))
normalize_stacktraces_for_grouping(data, grouping_config)
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags,
event,
_with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
if get_tag(data, key) is None:
set_tag(data, key, value)
for path, iface in six.iteritems(event.interfaces):
for k, v in iface.iter_tags():
set_tag(data, k, v)
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.path, None)
# The active grouping config was put into the event in the
# normalize step before. We now also make sure that the
# fingerprint was set to `'{{ default }}' just in case someone
# removed it from the payload. The call to get_hashes will then
# look at `grouping_config` to pick the right paramters.
data['fingerprint'] = data.get('fingerprint') or ['{{ default }}']
apply_server_fingerprinting(
data, get_fingerprinting_config_for_project(project))
hashes = event.get_hashes()
data['hashes'] = hashes
# we want to freeze not just the metadata and type in but also the
# derived attributes. The reason for this is that we push this
# data into kafka for snuba processing and our postprocessing
# picks up the data right from the snuba topic. For most usage
# however the data is dynamically overriden by Event.title and
# Event.location (See Event.as_dict)
materialized_metadata = self.materialize_metadata()
event_metadata = materialized_metadata['metadata']
data.update(materialized_metadata)
data['culprit'] = culprit
# index components into ``Event.message``
# See GH-3248
event.message = self.get_search_message(event_metadata, culprit)
received_timestamp = event.data.get('received') or float(
event.datetime.strftime('%s'))
# The group gets the same metadata as the event when it's flushed but
# additionally the `last_received` key is set. This key is used by
# _save_aggregate.
group_metadata = dict(materialized_metadata)
group_metadata['last_received'] = received_timestamp
kwargs = {
'platform': platform,
'message': event.message,
'culprit': culprit,
'logger': logger_name,
'level': LOG_LEVELS_MAP.get(level),
'last_seen': date,
'first_seen': date,
'active_at': date,
'data': group_metadata,
}
if release:
kwargs['first_release'] = release
try:
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **kwargs)
except HashDiscarded:
event_discarded.send_robust(
project=project,
sender=EventManager,
)
metrics.incr(
'events.discarded',
skip_internal=True,
tags={
'organization_id': project.organization_id,
'platform': platform,
},
)
raise
else:
event_saved.send_robust(
project=project,
event_size=event.size,
sender=EventManager,
)
event.group = group
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
# When an event was sampled, the canonical source of truth
# is the EventMapping table since we aren't going to be writing out an actual
# Event row. Otherwise, if the Event isn't being sampled, we can safely
# rely on the Event table itself as the source of truth and ignore
# EventMapping since it's redundant information.
if is_sample:
try:
with transaction.atomic(
using=router.db_for_write(EventMapping)):
EventMapping.objects.create(project=project,
group=group,
event_id=event_id)
except IntegrityError:
logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': EventMapping.__name__,
})
return event
environment = Environment.get_or_create(
project=project,
name=environment,
)
group_environment, is_new_group_environment = GroupEnvironment.get_or_create(
group_id=group.id,
environment_id=environment.id,
defaults={
'first_release': release if release else None,
},
)
if release:
ReleaseEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
ReleaseProjectEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date,
)
counters = [
(tsdb.models.group, group.id),
(tsdb.models.project, project.id),
]
if release:
counters.append((tsdb.models.release, release.id))
tsdb.incr_multi(counters,
timestamp=event.datetime,
environment_id=environment.id)
frequencies = [
# (tsdb.models.frequent_projects_by_organization, {
# project.organization_id: {
# project.id: 1,
# },
# }),
# (tsdb.models.frequent_issues_by_project, {
# project.id: {
# group.id: 1,
# },
# })
(tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1,
},
})
]
if release:
frequencies.append((tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1,
},
}))
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(
group=group,
environment=environment,
)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': Event.__name__,
})
return event
tagstore.delay_index_event_tags(
organization_id=project.organization_id,
project_id=project.id,
group_id=group.id,
environment_id=environment.id,
event_id=event.id,
tags=event.tags,
date_added=event.datetime,
)
if event_user:
tsdb.record_multi(
(
(tsdb.models.users_affected_by_group, group.id,
(event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id,
(event_user.tag_value, )),
),
timestamp=event.datetime,
environment_id=environment.id,
)
if release:
if is_new:
buffer.incr(ReleaseProject, {'new_groups': 1}, {
'release_id': release.id,
'project_id': project.id,
})
if is_new_group_environment:
buffer.incr(ReleaseProjectEnvironment, {'new_issues_count': 1},
{
'project_id': project.id,
'release_id': release.id,
'environment_id': environment.id,
})
safe_execute(Group.objects.add_tags,
group,
environment,
event.get_tags(),
_with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send_robust(project=project,
group=group,
sender=Project)
eventstream.insert(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
is_new_group_environment=is_new_group_environment,
primary_hash=hashes[0],
# We are choosing to skip consuming the event back
# in the eventstream if it's flagged as raw.
# This means that we want to publish the event
# through the event stream, but we don't care
# about post processing and handling the commit.
skip_consume=raw,
)
metrics.timing(
'events.latency',
received_timestamp - recorded_timestamp,
tags={
'project_id': project.id,
},
)
metrics.timing('events.size.data.post_save',
event.size,
tags={'project_id': project.id})
return event
def transfer_to(self, team=None, organization=None):
# NOTE: this will only work properly if the new team is in a different
# org than the existing one, which is currently the only use case in
# production
# TODO(jess): refactor this to make it an org transfer only
from sentry.models import (
Environment,
EnvironmentProject,
ProjectTeam,
ReleaseProject,
ReleaseProjectEnvironment,
Rule,
)
if organization is None:
organization = team.organization
old_org_id = self.organization_id
org_changed = old_org_id != organization.id
self.organization = organization
try:
with transaction.atomic():
self.update(organization=organization)
except IntegrityError:
slugify_instance(self,
self.name,
organization=organization,
max_length=50)
self.update(slug=self.slug, organization=organization)
# Both environments and releases are bound at an organization level.
# Due to this, when you transfer a project into another org, we have to
# handle this behavior somehow. We really only have two options here:
# * Copy over all releases/environments into the new org and handle de-duping
# * Delete the bindings and let them reform with new data.
# We're generally choosing to just delete the bindings since new data
# flowing in will recreate links correctly. The tradeoff is that
# historical data is lost, but this is a compromise we're willing to
# take and a side effect of allowing this feature. There are exceptions
# to this however, such as rules, which should maintain their
# configuration when moved across organizations.
if org_changed:
for model in ReleaseProject, ReleaseProjectEnvironment, EnvironmentProject:
model.objects.filter(project_id=self.id).delete()
# this is getting really gross, but make sure there aren't lingering associations
# with old orgs or teams
ProjectTeam.objects.filter(
project=self, team__organization_id=old_org_id).delete()
rules_by_environment_id = defaultdict(set)
for rule_id, environment_id in Rule.objects.filter(
project_id=self.id, environment_id__isnull=False).values_list(
"id", "environment_id"):
rules_by_environment_id[environment_id].add(rule_id)
environment_names = dict(
Environment.objects.filter(
id__in=rules_by_environment_id).values_list("id", "name"))
for environment_id, rule_ids in rules_by_environment_id.items():
Rule.objects.filter(id__in=rule_ids).update(
environment_id=Environment.get_or_create(
self, environment_names[environment_id]).id)
# ensure this actually exists in case from team was null
if team is not None:
self.add_team(team)
def save(self, project, raw=False):
from sentry.tasks.post_process import index_event_tags
data = self.data
project = Project.objects.get_from_cache(id=project)
# Check to make sure we're not about to do a bunch of work that's
# already been done if we've processed an event with this ID. (This
# isn't a perfect solution -- this doesn't handle ``EventMapping`` and
# there's a race condition between here and when the event is actually
# saved, but it's an improvement. See GH-7677.)
try:
event = Event.objects.get(
project_id=project.id,
event_id=data['event_id'],
)
except Event.DoesNotExist:
pass
else:
self.logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': data['event_id'],
'project_id': project.id,
'model': Event.__name__,
})
return event
# First we pull out our top-level (non-data attr) kwargs
event_id = data.pop('event_id')
level = data.pop('level')
culprit = data.pop('transaction', None)
if not culprit:
culprit = data.pop('culprit', None)
logger_name = data.pop('logger', None)
server_name = data.pop('server_name', None)
site = data.pop('site', None)
checksum = data.pop('checksum', None)
fingerprint = data.pop('fingerprint', None)
platform = data.pop('platform', None)
release = data.pop('release', None)
dist = data.pop('dist', None)
environment = data.pop('environment', None)
# unused
time_spent = data.pop('time_spent', None)
message = data.pop('message', '')
if not culprit:
# if we generate an implicit culprit, lets not call it a
# transaction
transaction_name = None
culprit = generate_culprit(data, platform=platform)
else:
transaction_name = culprit
culprit = force_text(culprit)
recorded_timestamp = data.pop('timestamp')
date = datetime.fromtimestamp(recorded_timestamp)
date = date.replace(tzinfo=timezone.utc)
kwargs = {
'platform': platform,
}
event = Event(project_id=project.id,
event_id=event_id,
data=data,
time_spent=time_spent,
datetime=date,
**kwargs)
event._project_cache = project
data = event.data.data
# convert this to a dict to ensure we're only storing one value per key
# as most parts of Sentry dont currently play well with multiple values
tags = dict(data.get('tags') or [])
tags['level'] = LOG_LEVELS[level]
if logger_name:
tags['logger'] = logger_name
if server_name:
tags['server_name'] = server_name
if site:
tags['site'] = site
if environment:
tags['environment'] = environment
if transaction_name:
tags['transaction'] = transaction_name
if release:
# dont allow a conflicting 'release' tag
if 'release' in tags:
del tags['release']
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
tags['sentry:release'] = release.version
if dist and release:
dist = release.add_dist(dist, date)
tags['sentry:dist'] = dist.name
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
if 'user' in tags:
del tags['user']
tags['sentry:user'] = event_user.tag_value
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
normalize_in_app(data)
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags,
event,
_with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
tags.setdefault(key, value)
for path, iface in six.iteritems(event.interfaces):
for k, v in iface.iter_tags():
tags[k] = v
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.get_path(), None)
# tags are stored as a tuple
tags = tags.items()
data['tags'] = tags
data['fingerprint'] = fingerprint or ['{{ default }}']
# prioritize fingerprint over checksum as its likely the client defaulted
# a checksum whereas the fingerprint was explicit
if fingerprint:
hashes = [
md5_from_hash(h)
for h in get_hashes_from_fingerprint(event, fingerprint)
]
elif checksum:
if HASH_RE.match(checksum):
hashes = [checksum]
else:
hashes = [md5_from_hash([checksum]), checksum]
data['checksum'] = checksum
else:
hashes = [md5_from_hash(h) for h in get_hashes_for_event(event)]
# TODO(dcramer): temp workaround for complexity
data['message'] = message
event_type = eventtypes.get(data.get('type', 'default'))(data)
event_metadata = event_type.get_metadata()
# TODO(dcramer): temp workaround for complexity
del data['message']
data['type'] = event_type.key
data['metadata'] = event_metadata
# index components into ``Event.message``
# See GH-3248
if event_type.key != 'default':
if 'sentry.interfaces.Message' in data and \
data['sentry.interfaces.Message']['message'] != message:
message = u'{} {}'.format(
message,
data['sentry.interfaces.Message']['message'],
)
if not message:
message = ''
elif not isinstance(message, six.string_types):
message = force_text(message)
for value in six.itervalues(event_metadata):
value_u = force_text(value, errors='replace')
if value_u not in message:
message = u'{} {}'.format(message, value_u)
if culprit and culprit not in message:
culprit_u = force_text(culprit, errors='replace')
message = u'{} {}'.format(message, culprit_u)
message = trim(message.strip(), settings.SENTRY_MAX_MESSAGE_LENGTH)
event.message = message
kwargs['message'] = message
received_timestamp = event.data.get('received') or float(
event.datetime.strftime('%s'))
group_kwargs = kwargs.copy()
group_kwargs.update({
'culprit': culprit,
'logger': logger_name,
'level': level,
'last_seen': date,
'first_seen': date,
'active_at': date,
'data': {
'last_received': received_timestamp,
'type': event_type.key,
# we cache the events metadata on the group to ensure its
# accessible in the stream
'metadata': event_metadata,
},
})
if release:
group_kwargs['first_release'] = release
try:
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **group_kwargs)
except HashDiscarded:
event_discarded.send_robust(
project=project,
sender=EventManager,
)
metrics.incr(
'events.discarded',
skip_internal=True,
tags={
'organization_id': project.organization_id,
'platform': platform,
},
)
raise
else:
event_saved.send_robust(
project=project,
event_size=event.size,
sender=EventManager,
)
event.group = group
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
# When an event was sampled, the canonical source of truth
# is the EventMapping table since we aren't going to be writing out an actual
# Event row. Otherwise, if the Event isn't being sampled, we can safely
# rely on the Event table itself as the source of truth and ignore
# EventMapping since it's redundant information.
if is_sample:
try:
with transaction.atomic(
using=router.db_for_write(EventMapping)):
EventMapping.objects.create(project=project,
group=group,
event_id=event_id)
except IntegrityError:
self.logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': EventMapping.__name__,
})
return event
environment = Environment.get_or_create(
project=project,
name=environment,
)
group_environment, is_new_group_environment = GroupEnvironment.get_or_create(
group_id=group.id,
environment_id=environment.id,
defaults={
'first_release_id': release.id if release else None,
},
)
if release:
ReleaseEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
ReleaseProjectEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date,
)
counters = [
(tsdb.models.group, group.id),
(tsdb.models.project, project.id),
]
if release:
counters.append((tsdb.models.release, release.id))
tsdb.incr_multi(counters,
timestamp=event.datetime,
environment_id=environment.id)
frequencies = [
# (tsdb.models.frequent_projects_by_organization, {
# project.organization_id: {
# project.id: 1,
# },
# }),
# (tsdb.models.frequent_issues_by_project, {
# project.id: {
# group.id: 1,
# },
# })
(tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1,
},
})
]
if release:
frequencies.append((tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1,
},
}))
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(
group=group,
environment=environment,
)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
self.logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': Event.__name__,
})
return event
index_event_tags.delay(
organization_id=project.organization_id,
project_id=project.id,
group_id=group.id,
environment_id=environment.id,
event_id=event.id,
tags=tags,
date_added=event.datetime,
)
if event_user:
tsdb.record_multi(
(
(tsdb.models.users_affected_by_group, group.id,
(event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id,
(event_user.tag_value, )),
),
timestamp=event.datetime,
environment_id=environment.id,
)
if release:
if is_new:
buffer.incr(ReleaseProject, {'new_groups': 1}, {
'release_id': release.id,
'project_id': project.id,
})
if is_new_group_environment:
buffer.incr(ReleaseProjectEnvironment, {'new_issues_count': 1},
{
'project_id': project.id,
'release_id': release.id,
'environment_id': environment.id,
})
safe_execute(Group.objects.add_tags,
group,
environment,
tags,
_with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send(project=project,
group=group,
sender=Project)
eventstream.publish(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
is_new_group_environment=is_new_group_environment,
primary_hash=hashes[0],
# We are choosing to skip consuming the event back
# in the eventstream if it's flagged as raw.
# This means that we want to publish the event
# through the event stream, but we don't care
# about post processing and handling the commit.
skip_consume=raw,
)
metrics.timing(
'events.latency',
received_timestamp - recorded_timestamp,
tags={
'project_id': project.id,
},
)
return event
def setUp(self):
self.project = self.create_project()
self.release = Release.get_or_create(self.project, '1.0')
self.environment1 = Environment.get_or_create(self.project, 'prod')
self.environment2 = Environment.get_or_create(self.project, 'staging')
self.timestamp = float(int(time() - 300))
def test_event_user(self):
manager = EventManager(
make_event(event_id='a',
environment='totally unique environment',
**{'user': {
'id': '1',
}}))
manager.normalize()
with self.tasks():
event = manager.save(self.project.id)
environment_id = Environment.get_for_organization_id(
event.project.organization_id,
'totally unique environment',
).id
assert tsdb.get_distinct_counts_totals(
tsdb.models.users_affected_by_group,
(event.group.id, ),
event.datetime,
event.datetime,
) == {
event.group.id: 1,
}
assert tsdb.get_distinct_counts_totals(
tsdb.models.users_affected_by_project,
(event.project.id, ),
event.datetime,
event.datetime,
) == {
event.project.id: 1,
}
assert tsdb.get_distinct_counts_totals(
tsdb.models.users_affected_by_group,
(event.group.id, ),
event.datetime,
event.datetime,
environment_id=environment_id,
) == {
event.group.id: 1,
}
assert tsdb.get_distinct_counts_totals(
tsdb.models.users_affected_by_project,
(event.project.id, ),
event.datetime,
event.datetime,
environment_id=environment_id,
) == {
event.project.id: 1,
}
euser = EventUser.objects.get(
project_id=self.project.id,
ident='1',
)
assert event.get_tag('sentry:user') == euser.tag_value
# ensure event user is mapped to tags in second attempt
manager = EventManager(
make_event(event_id='b', **{'user': {
'id': '1',
'name': 'jane',
}}))
manager.normalize()
with self.tasks():
event = manager.save(self.project.id)
euser = EventUser.objects.get(id=euser.id)
assert event.get_tag('sentry:user') == euser.tag_value
assert euser.name == 'jane'
assert euser.ident == '1'
def get_environment_name(event):
return Environment.get_name_or_default(event.get_tag("environment"))
def save(self, project, raw=False):
from sentry.tasks.post_process import index_event_tags
project = Project.objects.get_from_cache(id=project)
data = self.data.copy()
# First we pull out our top-level (non-data attr) kwargs
event_id = data.pop('event_id')
level = data.pop('level')
culprit = data.pop('culprit', None)
logger_name = data.pop('logger', None)
server_name = data.pop('server_name', None)
site = data.pop('site', None)
checksum = data.pop('checksum', None)
fingerprint = data.pop('fingerprint', None)
platform = data.pop('platform', None)
release = data.pop('release', None)
dist = data.pop('dist', None)
environment = data.pop('environment', None)
# unused
time_spent = data.pop('time_spent', None)
message = data.pop('message', '')
if not culprit:
# if we generate an implicit culprit, lets not call it a
# transaction
transaction_name = None
culprit = generate_culprit(data, platform=platform)
else:
transaction_name = culprit
date = datetime.fromtimestamp(data.pop('timestamp'))
date = date.replace(tzinfo=timezone.utc)
kwargs = {
'platform': platform,
}
event = Event(project_id=project.id,
event_id=event_id,
data=data,
time_spent=time_spent,
datetime=date,
**kwargs)
# convert this to a dict to ensure we're only storing one value per key
# as most parts of Sentry dont currently play well with multiple values
tags = dict(data.get('tags') or [])
tags['level'] = LOG_LEVELS[level]
if logger_name:
tags['logger'] = logger_name
if server_name:
tags['server_name'] = server_name
if site:
tags['site'] = site
if environment:
tags['environment'] = environment
if transaction_name:
tags['transaction'] = transaction_name
if release:
# dont allow a conflicting 'release' tag
if 'release' in tags:
del tags['release']
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
tags['sentry:release'] = release.version
if dist and release:
dist = release.add_dist(dist, date)
tags['sentry:dist'] = dist.name
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
if 'user' in tags:
del tags['user']
tags['sentry:user'] = event_user.tag_value
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
normalize_in_app(data)
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags,
event,
_with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
tags.setdefault(key, value)
# tags are stored as a tuple
tags = tags.items()
# XXX(dcramer): we're relying on mutation of the data object to ensure
# this propagates into Event
data['tags'] = tags
data['fingerprint'] = fingerprint or ['{{ default }}']
for path, iface in six.iteritems(event.interfaces):
data['tags'].extend(iface.iter_tags())
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.get_path(), None)
# prioritize fingerprint over checksum as its likely the client defaulted
# a checksum whereas the fingerprint was explicit
if fingerprint:
hashes = [
md5_from_hash(h)
for h in get_hashes_from_fingerprint(event, fingerprint)
]
elif checksum:
hashes = [checksum]
data['checksum'] = checksum
else:
hashes = [md5_from_hash(h) for h in get_hashes_for_event(event)]
# TODO(dcramer): temp workaround for complexity
data['message'] = message
event_type = eventtypes.get(data.get('type', 'default'))(data)
event_metadata = event_type.get_metadata()
# TODO(dcramer): temp workaround for complexity
del data['message']
data['type'] = event_type.key
data['metadata'] = event_metadata
# index components into ``Event.message``
# See GH-3248
if event_type.key != 'default':
if 'sentry.interfaces.Message' in data and \
data['sentry.interfaces.Message']['message'] != message:
message = u'{} {}'.format(
message,
data['sentry.interfaces.Message']['message'],
)
if not message:
message = ''
elif not isinstance(message, six.string_types):
message = force_text(message)
for value in six.itervalues(event_metadata):
value_u = force_text(value, errors='replace')
if value_u not in message:
message = u'{} {}'.format(message, value_u)
if culprit and culprit not in message:
culprit_u = force_text(culprit, errors='replace')
message = u'{} {}'.format(message, culprit_u)
message = trim(message.strip(), settings.SENTRY_MAX_MESSAGE_LENGTH)
event.message = message
kwargs['message'] = message
group_kwargs = kwargs.copy()
group_kwargs.update({
'culprit': culprit,
'logger': logger_name,
'level': level,
'last_seen': date,
'first_seen': date,
'active_at': date,
'data': {
'last_received':
event.data.get('received')
or float(event.datetime.strftime('%s')),
'type':
event_type.key,
# we cache the events metadata on the group to ensure its
# accessible in the stream
'metadata':
event_metadata,
},
})
if release:
group_kwargs['first_release'] = release
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **group_kwargs)
event.group = group
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
try:
with transaction.atomic(using=router.db_for_write(EventMapping)):
EventMapping.objects.create(project=project,
group=group,
event_id=event_id)
except IntegrityError:
self.logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': EventMapping.__name__,
})
return event
environment = Environment.get_or_create(
project=project,
name=environment,
)
if release:
ReleaseEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date,
)
counters = [
(tsdb.models.group, group.id),
(tsdb.models.project, project.id),
]
if release:
counters.append((tsdb.models.release, release.id))
tsdb.incr_multi(counters, timestamp=event.datetime)
frequencies = [
# (tsdb.models.frequent_projects_by_organization, {
# project.organization_id: {
# project.id: 1,
# },
# }),
# (tsdb.models.frequent_issues_by_project, {
# project.id: {
# group.id: 1,
# },
# })
(tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1,
},
})
]
if release:
frequencies.append((tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1,
},
}))
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(group=group)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
self.logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': Event.__name__,
})
return event
index_event_tags.delay(
organization_id=project.organization_id,
project_id=project.id,
group_id=group.id,
event_id=event.id,
tags=tags,
)
if event_user:
tsdb.record_multi((
(tsdb.models.users_affected_by_group, group.id,
(event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id,
(event_user.tag_value, )),
),
timestamp=event.datetime)
if is_new and release:
buffer.incr(ReleaseProject, {'new_groups': 1}, {
'release_id': release.id,
'project_id': project.id,
})
safe_execute(Group.objects.add_tags,
group,
tags,
_with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send(project=project,
group=group,
sender=Project)
post_process_group.delay(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
)
else:
self.logger.info('post_process.skip.raw_event',
extra={'event_id': event.id})
# TODO: move this to the queue
if is_regression and not raw:
regression_signal.send_robust(sender=Group, instance=group)
return event
def test_slack_channel_id_saved(self):
self.login_as(user=self.user)
project = self.create_project()
rule = Rule.objects.create(
project=project,
environment_id=Environment.get_or_create(project, "production").id,
label="foo",
)
integration = Integration.objects.create(
provider="slack",
name="Awesome Team",
external_id="TXXXXXXX1",
metadata={
"access_token": "xoxp-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx"
},
)
integration.add_organization(project.organization, self.user)
url = reverse(
"sentry-api-0-project-rule-details",
kwargs={
"organization_slug": project.organization.slug,
"project_slug": project.slug,
"rule_id": rule.id,
},
)
responses.add(
method=responses.GET,
url="https://slack.com/api/conversations.info",
status=200,
content_type="application/json",
body=json.dumps({
"ok": "true",
"channel": {
"name": "team-team-team",
"id": "CSVK0921"
}
}),
)
response = self.client.put(
url,
data={
"name":
"hello world",
"environment":
None,
"actionMatch":
"any",
"actions": [{
"id":
"sentry.integrations.slack.notify_action.SlackNotifyServiceAction",
"name":
"Send a notification to the funinthesun Slack workspace to #team-team-team and show tags [] in notification",
"workspace": integration.id,
"channel": "#team-team-team",
"input_channel_id": "CSVK0921",
}],
"conditions": [{
"id":
"sentry.rules.conditions.first_seen_event.FirstSeenEventCondition"
}],
},
format="json",
)
assert response.status_code == 200, response.content
assert response.data["id"] == str(rule.id)
assert response.data["actions"][0]["channel_id"] == "CSVK0921"
def validate_environment(self, value):
if not Environment.is_valid_name(value):
raise serializers.ValidationError("Invalid value for environment")
return value
def save(self, project_id, raw=False, assume_normalized=False):
# Normalize if needed
if not self._normalized:
if not assume_normalized:
self.normalize()
self._normalized = True
data = self._data
project = Project.objects.get_from_cache(id=project_id)
project._organization_cache = Organization.objects.get_from_cache(
id=project.organization_id
)
# Check to make sure we're not about to do a bunch of work that's
# already been done if we've processed an event with this ID. (This
# isn't a perfect solution -- this doesn't handle ``EventMapping`` and
# there's a race condition between here and when the event is actually
# saved, but it's an improvement. See GH-7677.)
try:
event = Event.objects.get(project_id=project.id, event_id=data["event_id"])
except Event.DoesNotExist:
pass
else:
# Make sure we cache on the project before returning
event._project_cache = project
logger.info(
"duplicate.found",
exc_info=True,
extra={
"event_uuid": data["event_id"],
"project_id": project.id,
"model": Event.__name__,
},
)
return event
# Pull out the culprit
culprit = self.get_culprit()
# Pull the toplevel data we're interested in
level = data.get("level")
# TODO(mitsuhiko): this code path should be gone by July 2018.
# This is going to be fine because no code actually still depends
# on integers here. When we need an integer it will be converted
# into one later. Old workers used to send integers here.
if level is not None and isinstance(level, six.integer_types):
level = LOG_LEVELS[level]
transaction_name = data.get("transaction")
logger_name = data.get("logger")
release = data.get("release")
dist = data.get("dist")
environment = data.get("environment")
recorded_timestamp = data.get("timestamp")
# We need to swap out the data with the one internal to the newly
# created event object
event = self._get_event_instance(project_id=project_id)
self._data = data = event.data.data
event._project_cache = project
date = event.datetime
platform = event.platform
event_id = event.event_id
if transaction_name:
transaction_name = force_text(transaction_name)
# Right now the event type is the signal to skip the group. This
# is going to change a lot.
if event.get_event_type() == "transaction":
issueless_event = True
else:
issueless_event = False
# Some of the data that are toplevel attributes are duplicated
# into tags (logger, level, environment, transaction). These are
# different from legacy attributes which are normalized into tags
# ahead of time (site, server_name).
setdefault_path(data, "tags", value=[])
set_tag(data, "level", level)
if logger_name:
set_tag(data, "logger", logger_name)
if environment:
set_tag(data, "environment", environment)
if transaction_name:
set_tag(data, "transaction", transaction_name)
if release:
# dont allow a conflicting 'release' tag
pop_tag(data, "release")
release = Release.get_or_create(project=project, version=release, date_added=date)
set_tag(data, "sentry:release", release.version)
if dist and release:
dist = release.add_dist(dist, date)
# dont allow a conflicting 'dist' tag
pop_tag(data, "dist")
set_tag(data, "sentry:dist", dist.name)
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
pop_tag(data, "user")
set_tag(data, "sentry:user", event_user.tag_value)
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
grouping_config = load_grouping_config(
get_grouping_config_dict_for_event_data(data, project)
)
normalize_stacktraces_for_grouping(data, grouping_config)
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags, event, _with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
if get_tag(data, key) is None:
set_tag(data, key, value)
for path, iface in six.iteritems(event.interfaces):
for k, v in iface.iter_tags():
set_tag(data, k, v)
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.path, None)
# The active grouping config was put into the event in the
# normalize step before. We now also make sure that the
# fingerprint was set to `'{{ default }}' just in case someone
# removed it from the payload. The call to get_hashes will then
# look at `grouping_config` to pick the right paramters.
data["fingerprint"] = data.get("fingerprint") or ["{{ default }}"]
apply_server_fingerprinting(data, get_fingerprinting_config_for_project(project))
# Here we try to use the grouping config that was requested in the
# event. If that config has since been deleted (because it was an
# experimental grouping config) we fall back to the default.
try:
hashes = event.get_hashes()
except GroupingConfigNotFound:
data["grouping_config"] = get_grouping_config_dict_for_project(project)
hashes = event.get_hashes()
data["hashes"] = hashes
# we want to freeze not just the metadata and type in but also the
# derived attributes. The reason for this is that we push this
# data into kafka for snuba processing and our postprocessing
# picks up the data right from the snuba topic. For most usage
# however the data is dynamically overriden by Event.title and
# Event.location (See Event.as_dict)
materialized_metadata = self.materialize_metadata()
event_metadata = materialized_metadata["metadata"]
data.update(materialized_metadata)
data["culprit"] = culprit
# index components into ``Event.message``
# See GH-3248
event.message = self.get_search_message(event_metadata, culprit)
received_timestamp = event.data.get("received") or float(event.datetime.strftime("%s"))
if not issueless_event:
# The group gets the same metadata as the event when it's flushed but
# additionally the `last_received` key is set. This key is used by
# _save_aggregate.
group_metadata = dict(materialized_metadata)
group_metadata["last_received"] = received_timestamp
kwargs = {
"platform": platform,
"message": event.message,
"culprit": culprit,
"logger": logger_name,
"level": LOG_LEVELS_MAP.get(level),
"last_seen": date,
"first_seen": date,
"active_at": date,
"data": group_metadata,
}
if release:
kwargs["first_release"] = release
try:
group, is_new, is_regression = self._save_aggregate(
event=event, hashes=hashes, release=release, **kwargs
)
except HashDiscarded:
event_discarded.send_robust(project=project, sender=EventManager)
metrics.incr(
"events.discarded",
skip_internal=True,
tags={"organization_id": project.organization_id, "platform": platform},
)
raise
else:
event_saved.send_robust(project=project, event_size=event.size, sender=EventManager)
event.group = group
else:
group = None
is_new = False
is_regression = False
event_saved.send_robust(project=project, event_size=event.size, sender=EventManager)
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
environment = Environment.get_or_create(project=project, name=environment)
if group:
group_environment, is_new_group_environment = GroupEnvironment.get_or_create(
group_id=group.id,
environment_id=environment.id,
defaults={"first_release": release if release else None},
)
else:
is_new_group_environment = False
if release:
ReleaseEnvironment.get_or_create(
project=project, release=release, environment=environment, datetime=date
)
ReleaseProjectEnvironment.get_or_create(
project=project, release=release, environment=environment, datetime=date
)
if group:
grouprelease = GroupRelease.get_or_create(
group=group, release=release, environment=environment, datetime=date
)
counters = [(tsdb.models.project, project.id)]
if group:
counters.append((tsdb.models.group, group.id))
if release:
counters.append((tsdb.models.release, release.id))
tsdb.incr_multi(counters, timestamp=event.datetime, environment_id=environment.id)
frequencies = [
# (tsdb.models.frequent_projects_by_organization, {
# project.organization_id: {
# project.id: 1,
# },
# }),
# (tsdb.models.frequent_issues_by_project, {
# project.id: {
# group.id: 1,
# },
# })
]
if group:
frequencies.append(
(tsdb.models.frequent_environments_by_group, {group.id: {environment.id: 1}})
)
if release:
frequencies.append(
(tsdb.models.frequent_releases_by_group, {group.id: {grouprelease.id: 1}})
)
if frequencies:
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
if group:
UserReport.objects.filter(project=project, event_id=event_id).update(
group=group, environment=environment
)
# save the event
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
logger.info(
"duplicate.found",
exc_info=True,
extra={
"event_uuid": event_id,
"project_id": project.id,
"group_id": group.id if group else None,
"model": Event.__name__,
},
)
return event
tagstore.delay_index_event_tags(
organization_id=project.organization_id,
project_id=project.id,
group_id=group.id if group else None,
environment_id=environment.id,
event_id=event.id,
tags=event.tags,
date_added=event.datetime,
)
if event_user:
counters = [
(tsdb.models.users_affected_by_project, project.id, (event_user.tag_value,))
]
if group:
counters.append(
(tsdb.models.users_affected_by_group, group.id, (event_user.tag_value,))
)
tsdb.record_multi(counters, timestamp=event.datetime, environment_id=environment.id)
if release:
if is_new:
buffer.incr(
ReleaseProject,
{"new_groups": 1},
{"release_id": release.id, "project_id": project.id},
)
if is_new_group_environment:
buffer.incr(
ReleaseProjectEnvironment,
{"new_issues_count": 1},
{
"project_id": project.id,
"release_id": release.id,
"environment_id": environment.id,
},
)
if group:
safe_execute(
Group.objects.add_tags,
group,
environment,
event.get_tags(),
_with_transaction=False,
)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send_robust(project=project, event=event, sender=Project)
eventstream.insert(
group=group,
event=event,
is_new=is_new,
is_regression=is_regression,
is_new_group_environment=is_new_group_environment,
primary_hash=hashes[0],
# We are choosing to skip consuming the event back
# in the eventstream if it's flagged as raw.
# This means that we want to publish the event
# through the event stream, but we don't care
# about post processing and handling the commit.
skip_consume=raw,
)
metrics.timing("events.latency", received_timestamp - recorded_timestamp)
metrics.timing("events.size.data.post_save", event.size)
return event
def test_event_user(self):
manager = EventManager(
make_event(event_id="a",
environment="totally unique environment",
**{"user": {
"id": "1"
}}))
manager.normalize()
with self.tasks():
event = manager.save(self.project.id)
environment_id = Environment.get_for_organization_id(
event.project.organization_id, "totally unique environment").id
assert tsdb.get_distinct_counts_totals(
tsdb.models.users_affected_by_group, (event.group.id, ),
event.datetime, event.datetime) == {
event.group.id: 1
}
assert tsdb.get_distinct_counts_totals(
tsdb.models.users_affected_by_project,
(event.project.id, ),
event.datetime,
event.datetime,
) == {
event.project.id: 1
}
assert tsdb.get_distinct_counts_totals(
tsdb.models.users_affected_by_group,
(event.group.id, ),
event.datetime,
event.datetime,
environment_id=environment_id,
) == {
event.group.id: 1
}
assert tsdb.get_distinct_counts_totals(
tsdb.models.users_affected_by_project,
(event.project.id, ),
event.datetime,
event.datetime,
environment_id=environment_id,
) == {
event.project.id: 1
}
euser = EventUser.objects.get(project_id=self.project.id, ident="1")
assert event.get_tag("sentry:user") == euser.tag_value
# ensure event user is mapped to tags in second attempt
manager = EventManager(
make_event(event_id="b", **{"user": {
"id": "1",
"name": "jane"
}}))
manager.normalize()
with self.tasks():
event = manager.save(self.project.id)
euser = EventUser.objects.get(id=euser.id)
assert event.get_tag("sentry:user") == euser.tag_value
assert euser.name == "jane"
assert euser.ident == "1"
def get_invalid_environment():
raise Environment.DoesNotExist()
def test_valid_name(val, expected):
assert Environment.is_valid_name(val) == expected
def save(self, project, raw=False):
from sentry.tasks.post_process import index_event_tags
project = Project.objects.get_from_cache(id=project)
data = self.data.copy()
# First we pull out our top-level (non-data attr) kwargs
event_id = data.pop('event_id')
level = data.pop('level')
culprit = data.pop('transaction', None)
if not culprit:
culprit = data.pop('culprit', None)
logger_name = data.pop('logger', None)
server_name = data.pop('server_name', None)
site = data.pop('site', None)
checksum = data.pop('checksum', None)
fingerprint = data.pop('fingerprint', None)
platform = data.pop('platform', None)
release = data.pop('release', None)
dist = data.pop('dist', None)
environment = data.pop('environment', None)
# unused
time_spent = data.pop('time_spent', None)
message = data.pop('message', '')
if not culprit:
# if we generate an implicit culprit, lets not call it a
# transaction
transaction_name = None
culprit = generate_culprit(data, platform=platform)
else:
transaction_name = culprit
date = datetime.fromtimestamp(data.pop('timestamp'))
date = date.replace(tzinfo=timezone.utc)
kwargs = {
'platform': platform,
}
event = Event(
project_id=project.id,
event_id=event_id,
data=data,
time_spent=time_spent,
datetime=date,
**kwargs
)
event._project_cache = project
# convert this to a dict to ensure we're only storing one value per key
# as most parts of Sentry dont currently play well with multiple values
tags = dict(data.get('tags') or [])
tags['level'] = LOG_LEVELS[level]
if logger_name:
tags['logger'] = logger_name
if server_name:
tags['server_name'] = server_name
if site:
tags['site'] = site
if environment:
tags['environment'] = environment
if transaction_name:
tags['transaction'] = transaction_name
if release:
# dont allow a conflicting 'release' tag
if 'release' in tags:
del tags['release']
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
tags['sentry:release'] = release.version
if dist and release:
dist = release.add_dist(dist, date)
tags['sentry:dist'] = dist.name
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
if 'user' in tags:
del tags['user']
tags['sentry:user'] = event_user.tag_value
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
normalize_in_app(data)
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags, event, _with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
tags.setdefault(key, value)
# tags are stored as a tuple
tags = tags.items()
# XXX(dcramer): we're relying on mutation of the data object to ensure
# this propagates into Event
data['tags'] = tags
data['fingerprint'] = fingerprint or ['{{ default }}']
for path, iface in six.iteritems(event.interfaces):
data['tags'].extend(iface.iter_tags())
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.get_path(), None)
# prioritize fingerprint over checksum as its likely the client defaulted
# a checksum whereas the fingerprint was explicit
if fingerprint:
hashes = [md5_from_hash(h) for h in get_hashes_from_fingerprint(event, fingerprint)]
elif checksum:
hashes = [checksum]
data['checksum'] = checksum
else:
hashes = [md5_from_hash(h) for h in get_hashes_for_event(event)]
# TODO(dcramer): temp workaround for complexity
data['message'] = message
event_type = eventtypes.get(data.get('type', 'default'))(data)
event_metadata = event_type.get_metadata()
# TODO(dcramer): temp workaround for complexity
del data['message']
data['type'] = event_type.key
data['metadata'] = event_metadata
# index components into ``Event.message``
# See GH-3248
if event_type.key != 'default':
if 'sentry.interfaces.Message' in data and \
data['sentry.interfaces.Message']['message'] != message:
message = u'{} {}'.format(
message,
data['sentry.interfaces.Message']['message'],
)
if not message:
message = ''
elif not isinstance(message, six.string_types):
message = force_text(message)
for value in six.itervalues(event_metadata):
value_u = force_text(value, errors='replace')
if value_u not in message:
message = u'{} {}'.format(message, value_u)
if culprit and culprit not in message:
culprit_u = force_text(culprit, errors='replace')
message = u'{} {}'.format(message, culprit_u)
message = trim(message.strip(), settings.SENTRY_MAX_MESSAGE_LENGTH)
event.message = message
kwargs['message'] = message
group_kwargs = kwargs.copy()
group_kwargs.update(
{
'culprit': culprit,
'logger': logger_name,
'level': level,
'last_seen': date,
'first_seen': date,
'active_at': date,
'data': {
'last_received':
event.data.get('received') or float(event.datetime.strftime('%s')),
'type':
event_type.key,
# we cache the events metadata on the group to ensure its
# accessible in the stream
'metadata':
event_metadata,
},
}
)
if release:
group_kwargs['first_release'] = release
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **group_kwargs
)
event.group = group
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
try:
with transaction.atomic(using=router.db_for_write(EventMapping)):
EventMapping.objects.create(project=project, group=group, event_id=event_id)
except IntegrityError:
self.logger.info(
'duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': EventMapping.__name__,
}
)
return event
environment = Environment.get_or_create(
project=project,
name=environment,
)
if release:
ReleaseEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date,
)
counters = [
(tsdb.models.group, group.id),
(tsdb.models.project, project.id),
]
if release:
counters.append((tsdb.models.release, release.id))
tsdb.incr_multi(counters, timestamp=event.datetime)
frequencies = [
# (tsdb.models.frequent_projects_by_organization, {
# project.organization_id: {
# project.id: 1,
# },
# }),
# (tsdb.models.frequent_issues_by_project, {
# project.id: {
# group.id: 1,
# },
# })
(tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1,
},
})
]
if release:
frequencies.append(
(tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1,
},
})
)
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(group=group)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
self.logger.info(
'duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': Event.__name__,
}
)
return event
index_event_tags.delay(
organization_id=project.organization_id,
project_id=project.id,
group_id=group.id,
event_id=event.id,
tags=tags,
)
if event_user:
tsdb.record_multi(
(
(tsdb.models.users_affected_by_group, group.id, (event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id, (event_user.tag_value, )),
),
timestamp=event.datetime
)
if is_new and release:
buffer.incr(
ReleaseProject, {'new_groups': 1}, {
'release_id': release.id,
'project_id': project.id,
}
)
safe_execute(Group.objects.add_tags, group, tags, _with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send(project=project, group=group, sender=Project)
post_process_group.delay(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
)
else:
self.logger.info('post_process.skip.raw_event', extra={'event_id': event.id})
# TODO: move this to the queue
if is_regression and not raw:
regression_signal.send_robust(sender=Group, instance=group)
return event
def create_environment(self, **kwargs):
project = kwargs.get('project', self.project)
name = kwargs.get('name', petname.Generate(1, ' ', letters=10))
return Environment.get_or_create(project=project, name=name)
