def get(self, request, project):
"""
Retrieve a Project
``````````````````
Return details on an individual project.
:pparam string organization_slug: the slug of the organization the
project belongs to.
:pparam string project_slug: the slug of the project to delete.
:auth: required
"""
active_plugins = [
{
'name': plugin.get_title(),
'id': plugin.slug,
}
for plugin in plugins.configurable_for_project(project, version=None)
if safe_execute(plugin.is_enabled, project)
and safe_execute(plugin.has_project_conf)
]
data = serialize(project, request.user)
data['options'] = {
'sentry:origins': '\n'.join(project.get_option('sentry:origins', ['*']) or []),
'sentry:resolve_age': int(project.get_option('sentry:resolve_age', 0)),
'sentry:scrub_data': bool(project.get_option('sentry:scrub_data', True)),
'sentry:sensitive_fields': project.get_option('sentry:sensitive_fields', []),
}
data['activePlugins'] = active_plugins
data['team'] = serialize(project.team, request.user)
data['organization'] = serialize(project.organization, request.user)
return Response(data)
def notification_settings(request):
forms = []
for plugin in plugins.all():
for form in safe_execute(plugin.get_notification_forms) or ():
form = safe_execute(form, plugin, request.user, request.POST or None)
if not form:
continue
helper = FormHelper()
helper.form_tag = False
forms.append((form, helper))
# Ensure our form comes first
helper = FormHelper()
helper.form_tag = False
forms = [(NotificationSettingsForm(request.user, request.POST or None), helper)] + forms
if request.POST:
if all(f.is_valid() for f, h in forms):
for form, helper in forms:
form.save()
messages.add_message(request, messages.SUCCESS, "Your settings were saved.")
return HttpResponseRedirect(request.path)
context = csrf(request)
context.update({"forms": forms, "page": "notifications"})
return render_to_response("sentry/account/notifications.html", context, request)
def process_event(cache_key, start_time=None, **kwargs):
from sentry.plugins import plugins
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'process'})
error_logger.error('process.failed.empty', extra={'cache_key': cache_key})
return
project = data['project']
Raven.tags_context({
'project': project,
})
# TODO(dcramer): ideally we would know if data changed by default
has_changed = False
for plugin in plugins.all(version=2):
processors = safe_execute(plugin.get_event_preprocessors, data=data, _with_transaction=False)
for processor in (processors or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data['project'] == project, 'Project cannot be mutated by preprocessor'
if has_changed:
default_cache.set(cache_key, data, 3600)
save_event.delay(cache_key=cache_key, data=None, start_time=start_time)
def notification_settings(request):
forms = []
for plugin in plugins.all():
for form in safe_execute(plugin.get_notification_forms) or ():
form = safe_execute(form, plugin, request.user, request.POST or None)
if not form:
continue
forms.append(form)
# Ensure our form comes first
forms = [
NotificationSettingsForm(request.user, request.POST or None),
] + forms
if request.POST:
if all(f.is_valid() for f in forms):
for form in forms:
form.save()
messages.add_message(request, messages.SUCCESS, 'Your settings were saved.')
return HttpResponseRedirect(request.path)
context = csrf(request)
context.update({
'forms': forms,
'page': 'notifications',
})
return render_to_response('sentry/account/notifications.html', context, request)
def send_async(self, to=None, bcc=None):
from sentry.tasks.email import send_email
fmt = options.get('system.logging-format')
messages = self.get_built_messages(to, bcc=bcc)
log_mail_queued = partial(
logger.info,
event='mail.queued',
message_type=self.type,
)
for message in messages:
safe_execute(
send_email.delay,
message=message,
_with_transaction=False,
)
message_id = message.extra_headers['Message-Id']
if fmt == LoggingFormat.HUMAN:
log_mail_queued(
message_id=message_id,
message_to=self.format_to(message.to),
)
elif fmt == LoggingFormat.MACHINE:
for recipient in message.to:
log_mail_queued(
message_id=message_id,
message_to=recipient,
)
def post_process_group(event, is_new, is_regression, is_sample, **kwargs):
"""
Fires post processing hooks for a group.
"""
from sentry.models import Project
from sentry.rules.processor import RuleProcessor
project = Project.objects.get_from_cache(id=event.group.project_id)
_capture_stats(event, is_new)
if settings.SENTRY_ENABLE_EXPLORE_CODE:
record_affected_code.delay(event=event)
record_affected_user.delay(event=event)
record_additional_tags(event=event)
rp = RuleProcessor(event, is_new, is_regression, is_sample)
# TODO(dcramer): ideally this would fanout, but serializing giant
# objects back and forth isn't super efficient
for callback, futures in rp.apply():
safe_execute(callback, event, futures)
for plugin in plugins.for_project(project):
plugin_post_process_group(
plugin_slug=plugin.slug,
event=event,
is_new=is_new,
is_regresion=is_regression,
is_sample=is_sample,
)
def preprocess_event(cache_key=None, data=None, start_time=None, **kwargs):
from sentry.plugins import plugins
if cache_key:
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'})
logger.error('Data not available in preprocess_event (cache_key=%s)', cache_key)
return
project = data['project']
# TODO(dcramer): ideally we would know if data changed by default
has_changed = False
for plugin in plugins.all(version=2):
for processor in (safe_execute(plugin.get_event_preprocessors) or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data['project'] == project, 'Project cannot be mutated by preprocessor'
if has_changed and cache_key:
default_cache.set(cache_key, data, 3600)
if cache_key:
data = None
save_event.delay(cache_key=cache_key, data=data, start_time=start_time)
def post_process_group(event, is_new, is_regression, is_sample, **kwargs):
"""
Fires post processing hooks for a group.
"""
from sentry.models import Project
from sentry.rules.processor import RuleProcessor
project = Project.objects.get_from_cache(id=event.group.project_id)
if settings.SENTRY_ENABLE_EXPLORE_CODE:
record_affected_code.delay(event=event)
if settings.SENTRY_ENABLE_EXPLORE_USERS:
record_affected_user.delay(event=event)
for plugin in plugins.for_project(project):
plugin_post_process_group.apply_async(
kwargs={
'plugin_slug': plugin.slug,
'event': event,
'is_new': is_new,
'is_regresion': is_regression,
'is_sample': is_sample,
},
expires=300,
)
rp = RuleProcessor(event, is_new, is_regression, is_sample)
# TODO(dcramer): ideally this would fanout, but serializing giant
# objects back and forth isn't super efficient
for callback, futures in rp.apply():
safe_execute(callback, event, futures)
def send_async(self, to=None, cc=None, bcc=None):
from sentry.tasks.email import send_email
fmt = options.get('system.logging-format')
messages = self.get_built_messages(to, cc=cc, bcc=bcc)
extra = {'message_type': self.type}
loggable = [v for k, v in six.iteritems(self.context) if hasattr(v, 'id')]
for context in loggable:
extra['%s_id' % type(context).__name__.lower()] = context.id
log_mail_queued = partial(logger.info, 'mail.queued', extra=extra)
for message in messages:
safe_execute(
send_email.delay,
message=message,
_with_transaction=False,
)
extra['message_id'] = message.extra_headers['Message-Id']
metrics.incr('email.queued', instance=self.type)
if fmt == LoggingFormat.HUMAN:
extra['message_to'] = self.format_to(message.to),
log_mail_queued()
elif fmt == LoggingFormat.MACHINE:
for recipient in message.to:
extra['message_to'] = recipient
log_mail_queued()
def post_process_group(event, is_new, is_regression, is_sample, **kwargs):
"""
Fires post processing hooks for a group.
"""
from sentry.models import Project
from sentry.rules.processor import RuleProcessor
project_id = event.group.project_id
Raven.tags_context({
'project': project_id,
})
project = Project.objects.get_from_cache(id=project_id)
_capture_stats(event, is_new)
rp = RuleProcessor(event, is_new, is_regression, is_sample)
# TODO(dcramer): ideally this would fanout, but serializing giant
# objects back and forth isn't super efficient
for callback, futures in rp.apply():
safe_execute(callback, event, futures)
for plugin in plugins.for_project(project):
plugin_post_process_group(
plugin_slug=plugin.slug,
event=event,
is_new=is_new,
is_regresion=is_regression,
is_sample=is_sample,
)
def notification_settings(request):
forms = []
for plugin in plugins.all():
for form in safe_execute(plugin.get_notification_forms) or ():
form = safe_execute(form, plugin, request.user, request.POST or None)
if not form:
continue
helper = FormHelper()
helper.form_tag = False
forms.append((form, helper))
# Ensure our form comes first
helper = FormHelper()
helper.form_tag = False
forms = [
(NotificationSettingsForm(request.user, request.POST or None), helper),
] + forms
if request.POST:
if all(f.is_valid() for f, h in forms):
for form, helper in forms:
form.save()
response = HttpResponseRedirect(reverse('sentry-account-settings-notifications') + '?success=1')
return response
context = csrf(request)
context.update({
'forms': forms,
'page': 'notifications',
})
return render_to_response('sentry/account/notifications.html', context, request)
def notification_settings(request):
settings_form = NotificationSettingsForm(request.user, request.POST or None)
# TODO(dcramer): this is an extremely bad pattern and we need a more optimal
# solution for rendering this (that ideally plays well with the org data)
project_list = []
organization_list = Organization.objects.get_for_user(
user=request.user,
)
for organization in organization_list:
team_list = Team.objects.get_for_user(
user=request.user,
organization=organization,
)
for team in team_list:
project_list.extend(
Project.objects.get_for_user(
user=request.user,
team=team,
)
)
project_forms = [
(project, ProjectEmailOptionsForm(
project, request.user,
request.POST or None,
prefix='project-%s' % (project.id,)
))
for project in sorted(project_list, key=lambda x: (
x.team.name if x.team else None, x.name))
]
ext_forms = []
for plugin in plugins.all():
for form in safe_execute(plugin.get_notification_forms) or ():
form = safe_execute(form, plugin, request.user, request.POST or None, prefix=plugin.slug)
if not form:
continue
ext_forms.append(form)
if request.POST:
all_forms = list(itertools.chain(
[settings_form], ext_forms, (f for _, f in project_forms)
))
if all(f.is_valid() for f in all_forms):
for form in all_forms:
form.save()
messages.add_message(request, messages.SUCCESS, 'Your settings were saved.')
return HttpResponseRedirect(request.path)
context = csrf(request)
context.update({
'settings_form': settings_form,
'project_forms': project_forms,
'ext_forms': ext_forms,
'page': 'notifications',
'AUTH_PROVIDERS': get_auth_providers(),
})
return render_to_response('sentry/account/notifications.html', context, request)
def get_attrs(self, item_list, user):
from sentry.plugins import plugins
GroupMeta.objects.populate_cache(item_list)
attach_foreignkey(item_list, Group.project, ['team'])
if user.is_authenticated() and item_list:
bookmarks = set(GroupBookmark.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', flat=True))
seen_groups = dict(GroupSeen.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', 'last_seen'))
else:
bookmarks = set()
seen_groups = {}
assignees = dict(
(a.group_id, a.user)
for a in GroupAssignee.objects.filter(
group__in=item_list,
).select_related('user')
)
user_counts = dict(
GroupTagKey.objects.filter(
group__in=item_list,
key='sentry:user',
).values_list('group', 'values_seen')
)
snoozes = dict(
GroupSnooze.objects.filter(
group__in=item_list,
).values_list('group', 'until')
)
result = {}
for item in item_list:
active_date = item.active_at or item.last_seen
annotations = []
for plugin in plugins.for_project(project=item.project, version=1):
safe_execute(plugin.tags, None, item, annotations)
for plugin in plugins.for_project(project=item.project, version=2):
annotations.extend(safe_execute(plugin.get_annotations, group=item) or ())
result[item] = {
'assigned_to': serialize(assignees.get(item.id)),
'is_bookmarked': item.id in bookmarks,
'has_seen': seen_groups.get(item.id, active_date) > active_date,
'annotations': annotations,
'user_count': user_counts.get(item.id, 0),
'snooze': snoozes.get(item.id),
}
return result
def after(self, event, state):
group = event.group
for plugin in self.get_plugins():
if not safe_execute(plugin.should_notify, group=group, event=event):
continue
safe_execute(plugin.notify_users, group=group, event=event)
def plugin_post_process_group(plugin_slug, event, **kwargs):
"""
Fires post processing hooks for a group.
"""
Raven.tags_context({
'project': event.project_id,
})
plugin = plugins.get(plugin_slug)
safe_execute(plugin.post_process, event=event, group=event.group, **kwargs)
def after(self, event, state):
group = event.group
notification = Notification(event=event, rule=self.rule)
for plugin in self.get_plugins():
if not safe_execute(plugin.should_notify, group=group, event=event):
continue
safe_execute(plugin.notify, notification)
def process_request(self, request):
# HACK: bootstrap some env crud if we haven't yet
if not settings.SENTRY_URL_PREFIX:
settings.SENTRY_URL_PREFIX = request.build_absolute_uri(reverse('sentry')).strip('/')
# bind request to env
env.request = request
safe_execute(self.load_user_conf, request)
def notify_users(self, group, event, fail_silently=False):
project = group.project
if not self.is_configured(project):
return
url = self.get_webhook_url(project)
payload = self.get_group_data(group, event)
safe_execute(self.send_webhook, url, payload)
def post_process_group(group, **kwargs):
"""
Fires post processing hooks for a group.
"""
from sentry.plugins import plugins
from sentry.utils.safe import safe_execute
for plugin in plugins.all():
if safe_execute(plugin.is_enabled, group.project):
safe_execute(plugin.post_process, group=group, **kwargs)
def process_request(self, request):
if settings.MAINTENANCE:
return
if '/api/0/' in request.path:
return
safe_execute(self.load_user_conf, request)
super(SentryLocaleMiddleware, self).process_request(request)
def post_process(self, group, event, is_new, is_sample, **kwargs):
if not is_new:
return
if not self.is_configured(group.project):
return
data = simplejson.dumps(self.get_group_data(group, event))
for url in self.get_webhook_urls(group.project):
safe_execute(self.send_webhook, url, data)
def test_with_simple_function(self):
def simple(a):
return a
assert safe_execute(simple, 1) == 1
def simple(a):
raise Exception()
assert safe_execute(simple, 1) is None
def get_attrs(self, item_list, user):
from sentry.plugins import plugins
GroupMeta.objects.populate_cache(item_list)
attach_foreignkey(item_list, Group.project, ["team"])
if user.is_authenticated() and item_list:
bookmarks = set(
GroupBookmark.objects.filter(user=user, group__in=item_list).values_list("group_id", flat=True)
)
seen_groups = dict(
GroupSeen.objects.filter(user=user, group__in=item_list).values_list("group_id", "last_seen")
)
else:
bookmarks = set()
seen_groups = {}
tag_counts = defaultdict(dict)
tag_results = GroupTagKey.objects.filter(group__in=item_list).values_list("key", "group", "values_seen")
for key, group_id, values_seen in tag_results:
tag_counts[key][group_id] = values_seen
assignees = dict(
(a.group_id, a.user) for a in GroupAssignee.objects.filter(group__in=item_list).select_related("user")
)
result = {}
for item in item_list:
active_date = item.active_at or item.last_seen
tags = {}
for key in tag_counts.iterkeys():
label = TAG_LABELS.get(key, key.replace("_", " ")).lower()
try:
value = tag_counts[key].get(item.id, 0)
except KeyError:
value = 0
tags[key] = {"label": label, "count": value}
annotations = []
for plugin in plugins.for_project(project=item.project, version=1):
safe_execute(plugin.tags, None, item, annotations)
for plugin in plugins.for_project(project=item.project, version=2):
annotations.extend(safe_execute(plugin.get_annotations, item) or ())
result[item] = {
"assigned_to": serialize(assignees.get(item.id)),
"is_bookmarked": item.id in bookmarks,
"has_seen": seen_groups.get(item.id, active_date) > active_date,
"tags": tags,
"annotations": annotations,
}
return result
def handle(self, request):
settings_form = self.notification_settings_form(
request.user, request.POST or None)
reports_form = NotificationReportSettingsForm(
request.user, request.POST or None,
prefix='reports')
project_list = list(Project.objects.filter(
team__organizationmemberteam__organizationmember__user=request.user,
team__organizationmemberteam__is_active=True,
status=ProjectStatus.VISIBLE,
).distinct())
project_forms = [
(project, ProjectEmailOptionsForm(
project, request.user,
request.POST or None,
prefix='project-%s' % (project.id,)
))
for project in sorted(project_list, key=lambda x: (
x.organization.name, x.name))
]
ext_forms = []
for plugin in plugins.all():
for form in safe_execute(plugin.get_notification_forms, _with_transaction=False) or ():
form = safe_execute(form, plugin, request.user, request.POST or None, prefix=plugin.slug,
_with_transaction=False)
if not form:
continue
ext_forms.append(form)
if request.POST:
all_forms = list(itertools.chain(
[settings_form, reports_form],
ext_forms,
(f for _, f in project_forms)
))
if all(f.is_valid() for f in all_forms):
for form in all_forms:
form.save()
messages.add_message(request, messages.SUCCESS, 'Your settings were saved.')
return HttpResponseRedirect(request.path)
context = csrf(request)
context.update({
'settings_form': settings_form,
'project_forms': project_forms,
'reports_form': reports_form,
'ext_forms': ext_forms,
'page': 'notifications',
'AUTH_PROVIDERS': get_auth_providers(),
})
return render_to_response('sentry/account/notifications.html', context, request)
def get_annotations(group, request=None):
project = group.project
annotation_list = []
for plugin in plugins.for_project(project, version=2):
for value in (safe_execute(plugin.get_annotations, group=group) or ()):
annotation = safe_execute(Annotation, **value)
if annotation:
annotation_list.append(annotation)
return annotation_list
def get_widgets(group, request):
project = group.project
for plugin in plugins.all():
if not safe_execute(plugin.is_enabled, project):
continue
resp = safe_execute(plugin.widget, request, group)
if resp:
yield resp.render(request)
def test_with_instance_method(self):
class Foo(object):
def simple(self, a):
return a
assert safe_execute(Foo().simple, 1) == 1
class Foo(object):
def simple(self, a):
raise Exception()
assert safe_execute(Foo().simple, 1) is None
def process_request(self, request):
# No locale for static media
# This avoids touching user session, which means we avoid
# setting `Vary: Cookie` as a response header which will
# break HTTP caching entirely.
self.__skip_caching = request.path_info.startswith(settings.ANONYMOUS_STATIC_PREFIXES)
if self.__skip_caching:
return
safe_execute(self.load_user_conf, request, _with_transaction=False)
super(SentryLocaleMiddleware, self).process_request(request)
def post_process(self, group, event, is_new, is_sample, **kwargs):
if not self.is_configured(group.project):
return
data = simplejson.dumps(self.get_group_data(group, event))
for url in self.get_tcp_server_urls(group.project):
try:
split_url = url.split(':')
safe_execute(self.send_tcp_data, split_url[0], int(split_url[1]), data)
except Exception as e:
print e
def post_process(self, group, event, is_new, is_sample, **kwargs):
# should we send this?
only_new = self.get_option('new_only', event.project)
if only_new and not is_new:
# no, we shouldn't
return
if not self.is_configured(group.project):
# bail!
return
# get the configuration variables
endpoint = self.get_option('message_hook_url', event.project)
format = self.get_option('format', event.project)
if endpoint and format:
# we need to build the URL
# this is might throw a KeyError(?!)
try:
failed_url = event.request.build_absolute_uri()
except Exception, e:
failed_url = ''
if failed_url == '':
# oh, the irony.
failed_url = '(URL unavailable)'
# helper for building this up:
event_url = '/%s/%s/group/%d/events/%d/' % (group.project.team.slug, group.project.slug, group.id, event.id)
# we need:
# id, checksum, project_slug, project_name,
# logger, level, culprit, message, failed_url
data = {
'id': str(group.id),
'checksum': group.checksum,
'project_slug': group.project.slug,
'project_name': group.project.name,
'logger': group.logger,
'level': group.get_level_display(),
'culprit': group.culprit,
'message': event.message,
'failed_url': failed_url,
'error_line': event.error().encode('utf-8').split('\n')[0],
'event_url': event_url
}
# now we have the data and the format string
# glue them together
formatted_data = format % data
# so send it:
safe_execute(self.send_to_notifico, endpoint, formatted_data)
def _do_process_event(
cache_key,
start_time,
event_id,
process_task,
data=None,
data_has_changed=None,
from_symbolicate=False,
):
from sentry.plugins.base import plugins
if data is None:
data = event_processing_store.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "process"
},
skip_internal=False)
error_logger.error("process.failed.empty",
extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_project(project_id)
event_id = data["event_id"]
with sentry_sdk.start_span(
op="tasks.store.process_event.get_project_from_cache"):
project = Project.objects.get_from_cache(id=project_id)
with metrics.timer(
"tasks.store.process_event.organization.get_from_cache"):
project._organization_cache = Organization.objects.get_from_cache(
id=project.organization_id)
has_changed = bool(data_has_changed)
with sentry_sdk.start_span(
op="tasks.store.process_event.get_reprocessing_revision"):
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
# Stacktrace based event processors.
with sentry_sdk.start_span(op="task.store.process_event.stacktraces"):
with metrics.timer("tasks.store.process_event.stacktraces",
tags={"from_symbolicate": from_symbolicate}):
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
# Second round of datascrubbing after stacktrace and language-specific
# processing. First round happened as part of ingest.
#
# *Right now* the only sensitive data that is added in stacktrace
# processing are usernames in filepaths, so we run directly after
# stacktrace processors.
#
# We do not yet want to deal with context data produced by plugins like
# sessionstack or fullstory (which are in `get_event_preprocessors`), as
# this data is very unlikely to be sensitive data. This is why scrubbing
# happens somewhere in the middle of the pipeline.
#
# On the other hand, Javascript event error translation is happening after
# this block because it uses `get_event_preprocessors` instead of
# `get_event_enhancers`.
#
# We are fairly confident, however, that this should run *before*
# re-normalization as it is hard to find sensitive data in partially
# trimmed strings.
if has_changed and options.get("processing.can-use-scrubbers"):
with sentry_sdk.start_span(op="task.store.datascrubbers.scrub"):
with metrics.timer("tasks.store.datascrubbers.scrub",
tags={"from_symbolicate": from_symbolicate}):
new_data = safe_execute(scrub_data,
project=project,
event=data.data)
# XXX(markus): When datascrubbing is finally "totally stable", we might want
# to drop the event if it crashes to avoid saving PII
if new_data is not None:
data.data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
with sentry_sdk.start_span(
op="task.store.process_event.preprocessors") as span:
span.set_data("plugin", plugin.slug)
span.set_data("from_symbolicate", from_symbolicate)
with metrics.timer(
"tasks.store.process_event.preprocessors",
tags={
"plugin": plugin.slug,
"from_symbolicate": from_symbolicate
},
):
processors = safe_execute(plugin.get_event_preprocessors,
data=data,
_with_transaction=False)
for processor in processors or ():
try:
result = processor(data)
except Exception:
error_logger.exception(
"tasks.store.preprocessors.error")
data.setdefault("_metrics",
{})["flag.processing.error"] = True
has_changed = True
else:
if result:
data = result
has_changed = True
assert data[
"project"] == project_id, "Project cannot be mutated by plugins"
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
# Run some of normalization again such that we don't:
# - persist e.g. incredibly large stacktraces from minidumps
# - store event timestamps that are older than our retention window
# (also happening with minidumps)
normalizer = StoreNormalizer(remove_other=False,
is_renormalize=True,
**DEFAULT_STORE_NORMALIZER_ARGS)
data = normalizer.normalize_event(dict(data))
issues = data.get("processing_issues")
try:
if issues and create_failed_event(
cache_key,
data,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev,
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke ourselves again. This happens when the reprocessing
# revision changed while we were processing.
_do_preprocess_event(cache_key, data, start_time, event_id,
process_task, project)
return
cache_key = event_processing_store.store(data)
from_reprocessing = process_task is process_event_from_reprocessing
submit_save_event(project, from_reprocessing, cache_key, event_id,
start_time, data)
def post_process_group(event, is_new, is_regression, is_sample, **kwargs):
"""
Fires post processing hooks for a group.
"""
# NOTE: we must pass through the full Event object, and not an
# event_id since the Event object may not actually have been stored
# in the database due to sampling.
from sentry.models import Project
from sentry.models.group import get_group_with_redirect
from sentry.rules.processor import RuleProcessor
from sentry.tasks.servicehooks import process_service_hook
# Re-bind Group since we're pickling the whole Event object
# which may contain a stale Group.
event.group, _ = get_group_with_redirect(event.group_id)
event.group_id = event.group.id
project_id = event.group.project_id
Raven.tags_context({
'project': project_id,
})
# Re-bind Project since we're pickling the whole Event object
# which may contain a stale Project.
event.project = Project.objects.get_from_cache(id=project_id)
_capture_stats(event, is_new)
# we process snoozes before rules as it might create a regression
process_snoozes(event.group)
rp = RuleProcessor(event, is_new, is_regression)
has_alert = False
# TODO(dcramer): ideally this would fanout, but serializing giant
# objects back and forth isn't super efficient
for callback, futures in rp.apply():
has_alert = True
safe_execute(callback, event, futures)
if features.has(
'projects:servicehooks',
project=event.project,
):
allowed_events = set(['event.created'])
if has_alert:
allowed_events.add('event.alert')
if allowed_events:
for servicehook_id, events in _get_service_hooks(
project_id=event.project_id):
if any(e in allowed_events for e in events):
process_service_hook.delay(
hook_id=servicehook_id,
event=event,
)
for plugin in plugins.for_project(event.project):
plugin_post_process_group(
plugin_slug=plugin.slug,
event=event,
is_new=is_new,
is_regresion=is_regression,
is_sample=is_sample,
)
event_processed.send_robust(
sender=post_process_group,
project=event.project,
group=event.group,
event=event,
)
def get_attrs(self, item_list, user):
from sentry.plugins import plugins
GroupMeta.objects.populate_cache(item_list)
attach_foreignkey(item_list, Group.project)
if user.is_authenticated() and item_list:
bookmarks = set(
GroupBookmark.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', flat=True)
)
seen_groups = dict(
GroupSeen.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', 'last_seen')
)
subscriptions = self._get_subscriptions(item_list, user)
else:
bookmarks = set()
seen_groups = {}
subscriptions = defaultdict(lambda: (False, None))
assignees = {
a.group_id: a.assigned_actor() for a in
GroupAssignee.objects.filter(
group__in=item_list,
)
}
resolved_assignees = Actor.resolve_dict(assignees)
ignore_items = {g.group_id: g for g in GroupSnooze.objects.filter(
group__in=item_list,
)}
resolved_item_list = [i for i in item_list if i.status == GroupStatus.RESOLVED]
if resolved_item_list:
release_resolutions = {
i[0]: i[1:]
for i in GroupResolution.objects.filter(
group__in=resolved_item_list,
).values_list(
'group',
'type',
'release__version',
'actor_id',
)
}
# due to our laziness, and django's inability to do a reasonable join here
# we end up with two queries
commit_results = list(Commit.objects.extra(
select={
'group_id': 'sentry_grouplink.group_id',
},
tables=['sentry_grouplink'],
where=[
'sentry_grouplink.linked_id = sentry_commit.id',
'sentry_grouplink.group_id IN ({})'.format(
', '.join(six.text_type(i.id) for i in resolved_item_list)),
'sentry_grouplink.linked_type = %s',
'sentry_grouplink.relationship = %s',
],
params=[
int(GroupLink.LinkedType.commit),
int(GroupLink.Relationship.resolves),
]
))
commit_resolutions = {
i.group_id: d for i, d in itertools.izip(commit_results, serialize(commit_results, user))
}
else:
release_resolutions = {}
commit_resolutions = {}
actor_ids = set(r[-1] for r in six.itervalues(release_resolutions))
actor_ids.update(r.actor_id for r in six.itervalues(ignore_items))
if actor_ids:
users = list(User.objects.filter(
id__in=actor_ids,
is_active=True,
))
actors = {u.id: d for u, d in itertools.izip(users, serialize(users, user))}
else:
actors = {}
share_ids = dict(GroupShare.objects.filter(
group__in=item_list,
).values_list('group_id', 'uuid'))
result = {}
seen_stats = self._get_seen_stats(item_list, user)
for item in item_list:
active_date = item.active_at or item.first_seen
annotations = []
for plugin in plugins.for_project(project=item.project, version=1):
safe_execute(plugin.tags, None, item, annotations, _with_transaction=False)
for plugin in plugins.for_project(project=item.project, version=2):
annotations.extend(
safe_execute(plugin.get_annotations, group=item, _with_transaction=False) or ()
)
from sentry.integrations import IntegrationFeatures
for integration in Integration.objects.filter(
organizations=item.project.organization_id):
if not (integration.has_feature(IntegrationFeatures.ISSUE_BASIC) or integration.has_feature(
IntegrationFeatures.ISSUE_SYNC)):
continue
install = integration.get_installation(item.project.organization_id)
annotations.extend(
safe_execute(install.get_annotations, group=item, _with_transaction=False) or ()
)
resolution_actor = None
resolution_type = None
resolution = release_resolutions.get(item.id)
if resolution:
resolution_type = 'release'
resolution_actor = actors.get(resolution[-1])
if not resolution:
resolution = commit_resolutions.get(item.id)
if resolution:
resolution_type = 'commit'
ignore_item = ignore_items.get(item.id)
if ignore_item:
ignore_actor = actors.get(ignore_item.actor_id)
else:
ignore_actor = None
result[item] = {
'assigned_to': resolved_assignees.get(item.id),
'is_bookmarked': item.id in bookmarks,
'subscription': subscriptions[item.id],
'has_seen': seen_groups.get(item.id, active_date) > active_date,
'annotations': annotations,
'ignore_until': ignore_item,
'ignore_actor': ignore_actor,
'resolution': resolution,
'resolution_type': resolution_type,
'resolution_actor': resolution_actor,
'share_id': share_ids.get(item.id),
}
result[item].update(seen_stats.get(item, {}))
return result
def get_attrs(self, item_list, user):
from sentry.plugins.base import plugins
from sentry.integrations import IntegrationFeatures
from sentry.models import PlatformExternalIssue
GroupMeta.objects.populate_cache(item_list)
attach_foreignkey(item_list, Group.project)
if user.is_authenticated() and item_list:
bookmarks = set(
GroupBookmark.objects.filter(user=user, group__in=item_list).values_list(
"group_id", flat=True
)
)
seen_groups = dict(
GroupSeen.objects.filter(user=user, group__in=item_list).values_list(
"group_id", "last_seen"
)
)
subscriptions = self._get_subscriptions(item_list, user)
else:
bookmarks = set()
seen_groups = {}
subscriptions = defaultdict(lambda: (False, None))
assignees = {
a.group_id: a.assigned_actor()
for a in GroupAssignee.objects.filter(group__in=item_list)
}
resolved_assignees = Actor.resolve_dict(assignees)
ignore_items = {g.group_id: g for g in GroupSnooze.objects.filter(group__in=item_list)}
resolved_item_list = [i for i in item_list if i.status == GroupStatus.RESOLVED]
if resolved_item_list:
release_resolutions = {
i[0]: i[1:]
for i in GroupResolution.objects.filter(group__in=resolved_item_list).values_list(
"group", "type", "release__version", "actor_id"
)
}
# due to our laziness, and django's inability to do a reasonable join here
# we end up with two queries
commit_results = list(
Commit.objects.extra(
select={"group_id": "sentry_grouplink.group_id"},
tables=["sentry_grouplink"],
where=[
"sentry_grouplink.linked_id = sentry_commit.id",
"sentry_grouplink.group_id IN ({})".format(
", ".join(six.text_type(i.id) for i in resolved_item_list)
),
"sentry_grouplink.linked_type = %s",
"sentry_grouplink.relationship = %s",
],
params=[int(GroupLink.LinkedType.commit), int(GroupLink.Relationship.resolves)],
)
)
commit_resolutions = {
i.group_id: d for i, d in zip(commit_results, serialize(commit_results, user))
}
else:
release_resolutions = {}
commit_resolutions = {}
actor_ids = set(r[-1] for r in six.itervalues(release_resolutions))
actor_ids.update(r.actor_id for r in six.itervalues(ignore_items))
if actor_ids:
users = list(User.objects.filter(id__in=actor_ids, is_active=True))
actors = {u.id: d for u, d in zip(users, serialize(users, user))}
else:
actors = {}
share_ids = dict(
GroupShare.objects.filter(group__in=item_list).values_list("group_id", "uuid")
)
result = {}
seen_stats = self._get_seen_stats(item_list, user)
annotations_by_group_id = defaultdict(list)
organization_id_list = list(set(item.project.organization_id for item in item_list))
# if no groups, then we can't proceed but this seems to be a valid use case
if not item_list:
return {}
if len(organization_id_list) > 1:
# this should never happen but if it does we should know about it
logger.warn(
u"Found multiple organizations for groups: %s, with orgs: %s"
% ([item.id for item in item_list], organization_id_list)
)
# should only have 1 org at this point
organization_id = organization_id_list[0]
organization = Organization.objects.get_from_cache(id=organization_id)
has_unhandled_flag = features.has(
"organizations:unhandled-issue-flag", organization, actor=user
)
# find all the integration installs that have issue tracking
for integration in Integration.objects.filter(organizations=organization_id):
if not (
integration.has_feature(IntegrationFeatures.ISSUE_BASIC)
or integration.has_feature(IntegrationFeatures.ISSUE_SYNC)
):
continue
install = integration.get_installation(organization_id)
local_annotations_by_group_id = (
safe_execute(
install.get_annotations_for_group_list,
group_list=item_list,
_with_transaction=False,
)
or {}
)
merge_list_dictionaries(annotations_by_group_id, local_annotations_by_group_id)
# find the external issues for sentry apps and add them in
local_annotations_by_group_id = (
safe_execute(
PlatformExternalIssue.get_annotations_for_group_list,
group_list=item_list,
_with_transaction=False,
)
or {}
)
merge_list_dictionaries(annotations_by_group_id, local_annotations_by_group_id)
snuba_stats = {}
if has_unhandled_flag:
snuba_stats = self._get_group_snuba_stats(item_list, seen_stats)
for item in item_list:
active_date = item.active_at or item.first_seen
annotations = []
annotations.extend(annotations_by_group_id[item.id])
# add the annotations for plugins
# note that the model GroupMeta where all the information is stored is already cached at the top of this function
# so these for loops doesn't make a bunch of queries
for plugin in plugins.for_project(project=item.project, version=1):
safe_execute(plugin.tags, None, item, annotations, _with_transaction=False)
for plugin in plugins.for_project(project=item.project, version=2):
annotations.extend(
safe_execute(plugin.get_annotations, group=item, _with_transaction=False) or ()
)
resolution_actor = None
resolution_type = None
resolution = release_resolutions.get(item.id)
if resolution:
resolution_type = "release"
resolution_actor = actors.get(resolution[-1])
if not resolution:
resolution = commit_resolutions.get(item.id)
if resolution:
resolution_type = "commit"
ignore_item = ignore_items.get(item.id)
if ignore_item:
ignore_actor = actors.get(ignore_item.actor_id)
else:
ignore_actor = None
result[item] = {
"assigned_to": resolved_assignees.get(item.id),
"is_bookmarked": item.id in bookmarks,
"subscription": subscriptions[item.id],
"has_seen": seen_groups.get(item.id, active_date) > active_date,
"annotations": annotations,
"ignore_until": ignore_item,
"ignore_actor": ignore_actor,
"resolution": resolution,
"resolution_type": resolution_type,
"resolution_actor": resolution_actor,
"share_id": share_ids.get(item.id),
}
if has_unhandled_flag:
result[item]["is_unhandled"] = bool(snuba_stats.get(item.id, {}).get("unhandled"))
result[item].update(seen_stats.get(item, {}))
return result
def process(self, request, project, auth, data, **kwargs):
event_received.send_robust(ip=request.META['REMOTE_ADDR'], sender=type(self))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(app.quotas.is_rate_limited, project=project)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
if rate_limit is not None and rate_limit.is_limited:
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.project_total_rejected, project.id),
(app.tsdb.models.organization_total_received, project.organization_id),
(app.tsdb.models.organization_total_rejected, project.organization_id),
])
raise APIRateLimited(rate_limit.retry_after)
else:
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.organization_total_received, project.organization_id),
])
result = plugins.first('has_perm', request.user, 'create_event', project)
if result is False:
raise APIForbidden('Creation of this event was blocked')
content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '')
if content_encoding == 'gzip':
data = decompress_gzip(data)
elif content_encoding == 'deflate':
data = decompress_deflate(data)
elif not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
# mutates data
validate_data(project, data, auth.client)
except InvalidData as e:
raise APIError(u'Invalid data: %s (%s)' % (six.text_type(e), type(e)))
# mutates data
manager = EventManager(data, version=auth.version)
data = manager.normalize()
# insert IP address if not available
if auth.is_public:
ensure_has_ip(data, request.META['REMOTE_ADDR'])
event_id = data['event_id']
# TODO(dcramer): ideally we'd only validate this if the event_id was
# supplied by the user
cache_key = 'ev:%s:%s' % (project.id, event_id,)
if cache.get(cache_key) is not None:
logger.warning('Discarded recent duplicate event from project %s/%s (id=%s)', project.organization.slug, project.slug, event_id)
raise InvalidRequest('An event with the same ID already exists.')
if project.get_option('sentry:scrub_data', True):
# We filter data immediately before it ever gets into the queue
inst = SensitiveDataFilter()
inst.apply(data)
# mutates data (strips a lot of context if not queued)
insert_data_to_database(data)
cache.set(cache_key, '', 60 * 5)
logger.debug('New event from project %s/%s (id=%s)', project.organization.slug, project.slug, event_id)
return event_id
def save(self, project, raw=False):
project = Project.objects.get_from_cache(id=project)
data = self.data.copy()
# First we pull out our top-level (non-data attr) kwargs
event_id = data.pop('event_id')
message = data.pop('message')
level = data.pop('level')
culprit = data.pop('culprit', None)
time_spent = data.pop('time_spent', None)
logger_name = data.pop('logger', None)
server_name = data.pop('server_name', None)
site = data.pop('site', None)
checksum = data.pop('checksum', None)
fingerprint = data.pop('fingerprint', None)
platform = data.pop('platform', None)
release = data.pop('release', None)
environment = data.pop('environment', None)
if not culprit:
culprit = generate_culprit(data)
date = datetime.fromtimestamp(data.pop('timestamp'))
date = date.replace(tzinfo=timezone.utc)
kwargs = {
'message': message,
'platform': platform,
}
event = Event(project=project,
event_id=event_id,
data=data,
time_spent=time_spent,
datetime=date,
**kwargs)
tags = data.get('tags') or []
tags.append(('level', LOG_LEVELS[level]))
if logger_name:
tags.append(('logger', logger_name))
if server_name:
tags.append(('server_name', server_name))
if site:
tags.append(('site', site))
if release:
# TODO(dcramer): we should ensure we create Release objects
tags.append(('sentry:release', release))
if environment:
tags.append(('environment', environment))
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags,
event,
_with_transaction=False)
if added_tags:
tags.extend(added_tags)
event_user = self._get_event_user(project, data)
if event_user:
tags.append(('sentry:user', event_user.tag_value))
# XXX(dcramer): we're relying on mutation of the data object to ensure
# this propagates into Event
data['tags'] = tags
data['fingerprint'] = fingerprint or ['{{ default }}']
# prioritize fingerprint over checksum as its likely the client defaulted
# a checksum whereas the fingerprint was explicit
if fingerprint:
hashes = map(md5_from_hash,
get_hashes_from_fingerprint(event, fingerprint))
elif checksum:
hashes = [checksum]
else:
hashes = map(md5_from_hash, get_hashes_for_event(event))
group_kwargs = kwargs.copy()
group_kwargs.update({
'culprit': culprit,
'logger': logger_name,
'level': level,
'last_seen': date,
'first_seen': date,
'time_spent_total': time_spent or 0,
'time_spent_count': time_spent and 1 or 0,
})
if release:
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
group_kwargs['first_release'] = release
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **group_kwargs)
event.group = group
event.group_id = group.id
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
try:
with transaction.atomic():
EventMapping.objects.create(project=project,
group=group,
event_id=event_id)
except IntegrityError:
self.logger.info('Duplicate EventMapping found for event_id=%s',
event_id)
return event
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(group=group)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic():
event.save()
except IntegrityError:
self.logger.info('Duplicate Event found for event_id=%s',
event_id)
return event
if event_user:
tsdb.record_multi((
(tsdb.models.users_affected_by_group, group.id,
(event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id,
(event_user.tag_value, )),
),
timestamp=event.datetime)
if is_new and release:
buffer.incr(Release, {'new_groups': 1}, {
'id': release.id,
})
safe_execute(Group.objects.add_tags,
group,
tags,
_with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
post_process_group.delay(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
)
else:
self.logger.info(
'Raw event passed; skipping post process for event_id=%s',
event_id)
# TODO: move this to the queue
if is_regression and not raw:
regression_signal.send_robust(sender=Group, instance=group)
return event
def _decode(value):
if value:
return safe_execute(json.loads, value)
def post_process_group(event,
is_new,
is_regression,
is_new_group_environment,
cache_key=None,
group_id=None,
**kwargs):
"""
Fires post processing hooks for a group.
"""
from sentry.eventstore.models import Event
from sentry.eventstore.processing import event_processing_store
from sentry.utils import snuba
from sentry.reprocessing2 import is_reprocessed_event
with snuba.options_override({"consistent": True}):
# The event parameter will be removed after transitioning to
# event_processing_store is complete.
if cache_key and event is None:
data = event_processing_store.get(cache_key)
if not data:
logger.info(
"post_process.skipped",
extra={
"cache_key": cache_key,
"reason": "missing_cache"
},
)
return
event = Event(project_id=data["project"],
event_id=data["event_id"],
group_id=group_id,
data=data)
elif check_event_already_post_processed(event):
if cache_key:
event_processing_store.delete_by_key(cache_key)
logger.info(
"post_process.skipped",
extra={
"project_id": event.project_id,
"event_id": event.event_id,
"reason": "duplicate",
},
)
return
if is_reprocessed_event(event.data):
logger.info(
"post_process.skipped",
extra={
"project_id": event.project_id,
"event_id": event.event_id,
"reason": "reprocessed",
},
)
return
set_current_project(event.project_id)
# NOTE: we must pass through the full Event object, and not an
# event_id since the Event object may not actually have been stored
# in the database due to sampling.
from sentry.models import Project, Organization, EventDict
from sentry.models.group import get_group_with_redirect
from sentry.rules.processor import RuleProcessor
from sentry.tasks.servicehooks import process_service_hook
# Re-bind node data to avoid renormalization. We only want to
# renormalize when loading old data from the database.
event.data = EventDict(event.data, skip_renormalization=True)
if event.group_id:
# Re-bind Group since we're reading the Event object
# from cache, which may contain a stale group and project
event.group, _ = get_group_with_redirect(event.group_id)
event.group_id = event.group.id
# Re-bind Project and Org since we're reading the Event object
# from cache which may contain stale parent models.
event.project = Project.objects.get_from_cache(id=event.project_id)
event.project._organization_cache = Organization.objects.get_from_cache(
id=event.project.organization_id)
bind_organization_context(event.project.organization)
_capture_stats(event, is_new)
if event.group_id:
# we process snoozes before rules as it might create a regression
# but not if it's new because you can't immediately snooze a new group
has_reappeared = False if is_new else process_snoozes(event.group)
handle_owner_assignment(event.project, event.group, event)
rp = RuleProcessor(event, is_new, is_regression,
is_new_group_environment, has_reappeared)
has_alert = False
# TODO(dcramer): ideally this would fanout, but serializing giant
# objects back and forth isn't super efficient
for callback, futures in rp.apply():
has_alert = True
with sentry_sdk.start_transaction(op="post_process_group",
name="rule_processor_apply",
sampled=True):
safe_execute(callback, event, futures)
if features.has("projects:servicehooks", project=event.project):
allowed_events = set(["event.created"])
if has_alert:
allowed_events.add("event.alert")
if allowed_events:
for servicehook_id, events in _get_service_hooks(
project_id=event.project_id):
if any(e in allowed_events for e in events):
process_service_hook.delay(
servicehook_id=servicehook_id, event=event)
from sentry.tasks.sentry_apps import process_resource_change_bound
if event.get_event_type(
) == "error" and _should_send_error_created_hooks(event.project):
process_resource_change_bound.delay(action="created",
sender="Error",
instance_id=event.event_id,
instance=event)
if is_new:
process_resource_change_bound.delay(action="created",
sender="Group",
instance_id=event.group_id)
from sentry.plugins.base import plugins
for plugin in plugins.for_project(event.project):
plugin_post_process_group(plugin_slug=plugin.slug,
event=event,
is_new=is_new,
is_regresion=is_regression)
event_processed.send_robust(
sender=post_process_group,
project=event.project,
event=event,
primary_hash=kwargs.get("primary_hash"),
)
if cache_key:
with metrics.timer("tasks.post_process.delete_event_cache"):
event_processing_store.delete_by_key(cache_key)
def process_event(event_manager, project, key, remote_addr, helper,
attachments, project_config):
event_received.send_robust(ip=remote_addr,
project=project,
sender=process_event)
start_time = time()
data = event_manager.get_data()
should_filter, filter_reason = event_manager.should_filter()
del event_manager
event_id = data["event_id"]
if should_filter:
signals_in_consumer = decide_signals_in_consumer()
if not signals_in_consumer:
# Mark that the event_filtered signal is sent. Do this before emitting
# the outcome to avoid a potential race between OutcomesConsumer and
# `event_filtered.send_robust` below.
mark_signal_sent(project_config.project_id, event_id)
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.FILTERED,
filter_reason,
event_id=event_id,
)
metrics.incr("events.blacklisted",
tags={"reason": filter_reason},
skip_internal=False)
if not signals_in_consumer:
event_filtered.send_robust(ip=remote_addr,
project=project,
sender=process_event)
# relay will no longer be able to provide information about filter
# status so to see the impact we're adding a way to turn on relay
# like behavior here.
if options.get("store.lie-about-filter-status"):
return event_id
raise APIForbidden("Event dropped due to filter: %s" %
(filter_reason, ))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(quotas.is_rate_limited,
project=project,
key=key,
_with_transaction=False)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
# XXX(dcramer): when the rate limiter fails we drop events to ensure
# it cannot cascade
if rate_limit is None or rate_limit.is_limited:
if rate_limit is None:
api_logger.debug("Dropped event due to error with rate limiter")
signals_in_consumer = decide_signals_in_consumer()
if not signals_in_consumer:
# Mark that the event_dropped signal is sent. Do this before emitting
# the outcome to avoid a potential race between OutcomesConsumer and
# `event_dropped.send_robust` below.
mark_signal_sent(project_config.project_id, event_id)
reason = rate_limit.reason_code if rate_limit else None
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.RATE_LIMITED,
reason,
event_id=event_id,
)
metrics.incr("events.dropped",
tags={"reason": reason or "unknown"},
skip_internal=False)
if not signals_in_consumer:
event_dropped.send_robust(ip=remote_addr,
project=project,
reason_code=reason,
sender=process_event)
if rate_limit is not None:
raise APIRateLimited(rate_limit.retry_after)
# TODO(dcramer): ideally we'd only validate this if the event_id was
# supplied by the user
cache_key = "ev:%s:%s" % (project_config.project_id, event_id)
if cache.get(cache_key) is not None:
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.INVALID,
"duplicate",
event_id=event_id,
)
raise APIForbidden("An event with the same ID already exists (%s)" %
(event_id, ))
datascrubbing_settings = project_config.config.get(
"datascrubbingSettings") or {}
data = semaphore.scrub_event(datascrubbing_settings, dict(data))
# mutates data (strips a lot of context if not queued)
helper.insert_data_to_database(data,
start_time=start_time,
attachments=attachments)
cache.set(cache_key, "", 60 * 60) # Cache for 1 hour
api_logger.debug("New event received (%s)", event_id)
event_accepted.send_robust(ip=remote_addr,
data=data,
project=project,
sender=process_event)
return event_id
def get_attrs(self, item_list, user):
from sentry.plugins import plugins
GroupMeta.objects.populate_cache(item_list)
attach_foreignkey(item_list, Group.project)
if user.is_authenticated() and item_list:
bookmarks = set(
GroupBookmark.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', flat=True))
seen_groups = dict(
GroupSeen.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', 'last_seen'))
subscriptions = self._get_subscriptions(item_list, user)
else:
bookmarks = set()
seen_groups = {}
subscriptions = defaultdict(lambda: (False, None))
assignees = {
a.group_id: a.assigned_actor()
for a in GroupAssignee.objects.filter(group__in=item_list, )
}
resolved_assignees = Actor.resolve_dict(assignees)
try:
environment = self.environment_func()
except Environment.DoesNotExist:
user_counts = {}
first_seen = {}
last_seen = {}
times_seen = {}
else:
project_id = item_list[0].project_id
item_ids = [g.id for g in item_list]
user_counts = tagstore.get_groups_user_counts(
project_id,
item_ids,
environment_id=environment and environment.id,
)
first_seen = {}
last_seen = {}
times_seen = {}
if environment is not None:
environment_tagvalues = tagstore.get_group_list_tag_value(
project_id,
item_ids,
environment.id,
'environment',
environment.name,
)
for item_id, value in environment_tagvalues.items():
first_seen[item_id] = value.first_seen
last_seen[item_id] = value.last_seen
times_seen[item_id] = value.times_seen
else:
for item in item_list:
first_seen[item.id] = item.first_seen
last_seen[item.id] = item.last_seen
times_seen[item.id] = item.times_seen
ignore_items = {
g.group_id: g
for g in GroupSnooze.objects.filter(group__in=item_list, )
}
resolutions = {
i[0]: i[1:]
for i in GroupResolution.objects.filter(group__in=item_list,
).values_list(
'group',
'type',
'release__version',
'actor_id',
)
}
actor_ids = set(r[-1] for r in six.itervalues(resolutions))
actor_ids.update(r.actor_id for r in six.itervalues(ignore_items))
if actor_ids:
users = list(
User.objects.filter(
id__in=actor_ids,
is_active=True,
))
actors = {
u.id: d
for u, d in itertools.izip(users, serialize(users, user))
}
else:
actors = {}
share_ids = dict(
GroupShare.objects.filter(group__in=item_list, ).values_list(
'group_id', 'uuid'))
result = {}
for item in item_list:
active_date = item.active_at or item.first_seen
annotations = []
for plugin in plugins.for_project(project=item.project, version=1):
safe_execute(plugin.tags,
None,
item,
annotations,
_with_transaction=False)
for plugin in plugins.for_project(project=item.project, version=2):
annotations.extend(
safe_execute(plugin.get_annotations,
group=item,
_with_transaction=False) or ())
resolution = resolutions.get(item.id)
if resolution:
resolution_actor = actors.get(resolution[-1])
else:
resolution_actor = None
ignore_item = ignore_items.get(item.id)
if ignore_item:
ignore_actor = actors.get(ignore_item.actor_id)
else:
ignore_actor = None
result[item] = {
'assigned_to': resolved_assignees.get(item.id),
'is_bookmarked': item.id in bookmarks,
'subscription': subscriptions[item.id],
'has_seen':
seen_groups.get(item.id, active_date) > active_date,
'annotations': annotations,
'user_count': user_counts.get(item.id, 0),
'ignore_until': ignore_item,
'ignore_actor': ignore_actor,
'resolution': resolution,
'resolution_actor': resolution_actor,
'share_id': share_ids.get(item.id),
'times_seen': times_seen.get(item.id, 0),
'first_seen': first_seen.get(item.id), # TODO: missing?
'last_seen': last_seen.get(item.id),
}
return result
def for_project(self, project, version=1):
for plugin in self.all(version=version):
if not safe_execute(
plugin.is_enabled, project, _with_transaction=False):
continue
yield plugin
def apply_rule(self, rule):
match = rule.data.get("action_match") or Rule.DEFAULT_ACTION_MATCH
condition_list = rule.data.get("conditions", ())
frequency = rule.data.get("frequency") or Rule.DEFAULT_FREQUENCY
# XXX(dcramer): if theres no condition should we really skip it,
# or should we just apply it blindly?
if not condition_list:
return
if (rule.environment_id is not None
and self.event.get_environment().id != rule.environment_id):
return
status = self.get_rule_status(rule)
now = timezone.now()
freq_offset = now - timedelta(minutes=frequency)
if status.last_active and status.last_active > freq_offset:
return
state = self.get_state()
condition_iter = (self.condition_matches(c, state, rule)
for c in condition_list)
if match == "all":
passed = all(condition_iter)
elif match == "any":
passed = any(condition_iter)
elif match == "none":
passed = not any(condition_iter)
else:
self.logger.error("Unsupported action_match %r for rule %d", match,
rule.id)
return
if passed:
passed = (GroupRuleStatus.objects.filter(id=status.id).exclude(
last_active__gt=freq_offset).update(last_active=now))
if not passed:
return
if randrange(10) == 0:
analytics.record(
"issue_alert.fired",
issue_id=self.group.id,
project_id=rule.project.id,
organization_id=rule.project.organization.id,
rule_id=rule.id,
)
for action in rule.data.get("actions", ()):
action_cls = rules.get(action["id"])
if action_cls is None:
self.logger.warn("Unregistered action %r", action["id"])
continue
action_inst = action_cls(self.project, data=action, rule=rule)
results = safe_execute(action_inst.after,
event=self.event,
state=state,
_with_transaction=False)
if results is None:
self.logger.warn("Action %s did not return any futures",
action["id"])
continue
for future in results:
key = future.key if future.key is not None else future.callback
rule_future = RuleFuture(rule=rule, kwargs=future.kwargs)
if key not in self.grouped_futures:
self.grouped_futures[key] = (future.callback,
[rule_future])
else:
self.grouped_futures[key][1].append(rule_future)
def save_data(self, project, data, raw=False):
# TODO: this function is way too damn long and needs refactored
# the inner imports also suck so let's try to move it away from
# the objects manager
# TODO: culprit should default to "most recent" frame in stacktraces when
# it's not provided.
from sentry.plugins import plugins
from sentry.models import Event, Project, EventMapping
project = Project.objects.get_from_cache(id=project)
# First we pull out our top-level (non-data attr) kwargs
event_id = data.pop('event_id')
message = data.pop('message')
culprit = data.pop('culprit')
level = data.pop('level')
time_spent = data.pop('time_spent')
logger_name = data.pop('logger')
server_name = data.pop('server_name')
site = data.pop('site')
date = data.pop('timestamp')
checksum = data.pop('checksum')
platform = data.pop('platform')
if 'sentry.interfaces.Exception' in data:
if 'values' not in data['sentry.interfaces.Exception']:
data['sentry.interfaces.Exception'] = {
'values': [data['sentry.interfaces.Exception']]
}
# convert stacktrace + exception into expanded exception
if 'sentry.interfaces.Stacktrace' in data:
data['sentry.interfaces.Exception']['values'][0][
'stacktrace'] = data.pop('sentry.interfaces.Stacktrace')
kwargs = {
'level': level,
'message': message,
'platform': platform,
'culprit': culprit or '',
'logger': logger_name,
}
event = Event(project=project,
event_id=event_id,
data=data,
server_name=server_name,
site=site,
time_spent=time_spent,
datetime=date,
**kwargs)
# Calculate the checksum from the first highest scoring interface
if not checksum:
checksum = get_checksum_from_event(event)
event.checksum = checksum
group_kwargs = kwargs.copy()
group_kwargs.update({
'last_seen': date,
'first_seen': date,
'time_spent_total': time_spent or 0,
'time_spent_count': time_spent and 1 or 0,
})
tags = data['tags']
tags.append(('level', LOG_LEVELS[level]))
if logger:
tags.append(('logger', logger_name))
if server_name:
tags.append(('server_name', server_name))
if site:
tags.append(('site', site))
for plugin in plugins.for_project(project):
added_tags = safe_execute(plugin.get_tags, event)
if added_tags:
tags.extend(added_tags)
try:
group, is_new, is_sample = self._create_group(event=event,
tags=data['tags'],
**group_kwargs)
except Exception as exc:
# TODO: should we mail admins when there are failures?
try:
logger.exception(u'Unable to process log entry: %s', exc)
except Exception, exc:
warnings.warn(u'Unable to process log entry: %s', exc)
return
def get_attrs(self, item_list, user):
from sentry.plugins import plugins
GroupMeta.objects.populate_cache(item_list)
attach_foreignkey(item_list, Group.project)
if user.is_authenticated() and item_list:
bookmarks = set(
GroupBookmark.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', flat=True)
)
seen_groups = dict(
GroupSeen.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', 'last_seen')
)
subscriptions = self._get_subscriptions(item_list, user)
else:
bookmarks = set()
seen_groups = {}
subscriptions = defaultdict(lambda: (False, None))
assignees = dict(
(a.group_id, a.user)
for a in GroupAssignee.objects.filter(
group__in=item_list,
).select_related('user')
)
user_counts = tagstore.get_group_values_seen([g.id for g in item_list], 'sentry:user')
ignore_items = {g.group_id: g for g in GroupSnooze.objects.filter(
group__in=item_list,
)}
resolutions = {
i[0]: i[1:]
for i in GroupResolution.objects.filter(
group__in=item_list,
).values_list(
'group',
'type',
'release__version',
'actor_id',
)
}
actor_ids = set(r[-1] for r in six.itervalues(resolutions))
actor_ids.update(r.actor_id for r in six.itervalues(ignore_items))
if actor_ids:
users = list(User.objects.filter(
id__in=actor_ids,
is_active=True,
))
actors = {u.id: d for u, d in izip(users, serialize(users, user))}
else:
actors = {}
result = {}
for item in item_list:
active_date = item.active_at or item.first_seen
annotations = []
for plugin in plugins.for_project(project=item.project, version=1):
safe_execute(plugin.tags, None, item, annotations, _with_transaction=False)
for plugin in plugins.for_project(project=item.project, version=2):
annotations.extend(
safe_execute(plugin.get_annotations, group=item, _with_transaction=False) or ()
)
resolution = resolutions.get(item.id)
if resolution:
resolution_actor = actors.get(resolution[-1])
else:
resolution_actor = None
ignore_item = ignore_items.get(item.id)
if ignore_item:
ignore_actor = actors.get(ignore_item.actor_id)
else:
ignore_actor = None
result[item] = {
'assigned_to': serialize(assignees.get(item.id)),
'is_bookmarked': item.id in bookmarks,
'subscription': subscriptions[item.id],
'has_seen': seen_groups.get(item.id, active_date) > active_date,
'annotations': annotations,
'user_count': user_counts.get(item.id, 0),
'ignore_until': ignore_item,
'ignore_actor': ignore_actor,
'resolution': resolution,
'resolution_actor': resolution_actor,
}
return result
def _do_process_event(cache_key,
start_time,
event_id,
process_task,
data=None):
from sentry.plugins import plugins
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "process"
},
skip_internal=False)
error_logger.error("process.failed.empty",
extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
with configure_scope() as scope:
scope.set_tag("project", project_id)
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
try:
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
enhancers = safe_execute(plugin.get_event_enhancers, data=data)
for enhancer in enhancers or ():
enhanced = safe_execute(
enhancer, data, _passthrough_errors=(RetrySymbolication, ))
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
except RetrySymbolication as e:
if start_time and (time() - start_time) > 3600:
raise RuntimeError("Event spent one hour in processing")
retry_process_event.apply_async(
args=(),
kwargs={
"process_task_name": process_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(plugin.get_event_preprocessors,
data=data,
_with_transaction=False)
for processor in processors or ():
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data[
"project"] == project_id, "Project cannot be mutated by preprocessor"
project = Project.objects.get_from_cache(id=project_id)
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
# Run some of normalization again such that we don't:
# - persist e.g. incredibly large stacktraces from minidumps
# - store event timestamps that are older than our retention window
# (also happening with minidumps)
normalizer = StoreNormalizer(remove_other=False,
is_renormalize=True,
**DEFAULT_STORE_NORMALIZER_ARGS)
data = normalizer.normalize_event(dict(data))
issues = data.get("processing_issues")
try:
if issues and create_failed_event(
cache_key,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev,
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
from_reprocessing = process_task is process_event_from_reprocessing
submit_process(project, from_reprocessing, cache_key, event_id,
start_time, data)
process_task.delay(cache_key,
start_time=start_time,
event_id=event_id)
return
default_cache.set(cache_key, data, 3600)
submit_save_event(project, cache_key, event_id, start_time, data)
def post_process_group(
is_new, is_regression, is_new_group_environment, cache_key, group_id=None, **kwargs
):
"""
Fires post processing hooks for a group.
"""
from sentry.eventstore.models import Event
from sentry.eventstore.processing import event_processing_store
from sentry.utils import snuba
from sentry.reprocessing2 import is_reprocessed_event
with snuba.options_override({"consistent": True}):
# We use the data being present/missing in the processing store
# to ensure that we don't duplicate work should the forwarding consumers
# need to rewind history.
data = event_processing_store.get(cache_key)
if not data:
logger.info(
"post_process.skipped",
extra={"cache_key": cache_key, "reason": "missing_cache"},
)
return
event = Event(
project_id=data["project"], event_id=data["event_id"], group_id=group_id, data=data
)
set_current_project(event.project_id)
is_reprocessed = is_reprocessed_event(event.data)
# NOTE: we must pass through the full Event object, and not an
# event_id since the Event object may not actually have been stored
# in the database due to sampling.
from sentry.models import (
Commit,
Project,
Organization,
EventDict,
GroupInboxReason,
)
from sentry.models.groupinbox import add_group_to_inbox
from sentry.models.group import get_group_with_redirect
from sentry.rules.processor import RuleProcessor
from sentry.tasks.servicehooks import process_service_hook
from sentry.tasks.groupowner import process_suspect_commits
# Re-bind node data to avoid renormalization. We only want to
# renormalize when loading old data from the database.
event.data = EventDict(event.data, skip_renormalization=True)
# Re-bind Project and Org since we're reading the Event object
# from cache which may contain stale parent models.
event.project = Project.objects.get_from_cache(id=event.project_id)
event.project._organization_cache = Organization.objects.get_from_cache(
id=event.project.organization_id
)
if event.group_id:
# Re-bind Group since we're reading the Event object
# from cache, which may contain a stale group and project
event.group, _ = get_group_with_redirect(event.group_id)
event.group_id = event.group.id
event.group.project = event.project
event.group.project._organization_cache = event.project._organization_cache
bind_organization_context(event.project.organization)
_capture_stats(event, is_new)
if event.group_id and is_reprocessed and is_new:
add_group_to_inbox(event.group, GroupInboxReason.REPROCESSED)
if event.group_id and not is_reprocessed:
# we process snoozes before rules as it might create a regression
# but not if it's new because you can't immediately snooze a new group
has_reappeared = False if is_new else process_snoozes(event.group)
if not has_reappeared: # If true, we added the .UNIGNORED reason already
if is_new:
add_group_to_inbox(event.group, GroupInboxReason.NEW)
elif is_regression:
add_group_to_inbox(event.group, GroupInboxReason.REGRESSION)
handle_owner_assignment(event.project, event.group, event)
rp = RuleProcessor(
event, is_new, is_regression, is_new_group_environment, has_reappeared
)
has_alert = False
# TODO(dcramer): ideally this would fanout, but serializing giant
# objects back and forth isn't super efficient
for callback, futures in rp.apply():
has_alert = True
with sentry_sdk.start_transaction(
op="post_process_group", name="rule_processor_apply", sampled=True
):
safe_execute(callback, event, futures, _with_transaction=False)
try:
lock = locks.get(
"w-o:{}-d-l".format(event.group_id),
duration=10,
)
with lock.acquire():
has_commit_key = "w-o:{}-h-c".format(event.project.organization_id)
org_has_commit = cache.get(has_commit_key)
if org_has_commit is None:
org_has_commit = Commit.objects.filter(
organization_id=event.project.organization_id
).exists()
cache.set(has_commit_key, org_has_commit, 3600)
if org_has_commit:
group_cache_key = "w-o-i:g-{}".format(event.group_id)
if cache.get(group_cache_key):
metrics.incr(
"sentry.tasks.process_suspect_commits.debounce",
tags={"detail": "w-o-i:g debounce"},
)
else:
from sentry.utils.committers import get_frame_paths
cache.set(group_cache_key, True, 604800) # 1 week in seconds
event_frames = get_frame_paths(event.data)
process_suspect_commits.delay(
event_id=event.event_id,
event_platform=event.platform,
event_frames=event_frames,
group_id=event.group_id,
project_id=event.project_id,
)
except UnableToAcquireLock:
pass
except Exception:
logger.exception("Failed to process suspect commits")
if features.has("projects:servicehooks", project=event.project):
allowed_events = set(["event.created"])
if has_alert:
allowed_events.add("event.alert")
if allowed_events:
for servicehook_id, events in _get_service_hooks(project_id=event.project_id):
if any(e in allowed_events for e in events):
process_service_hook.delay(servicehook_id=servicehook_id, event=event)
from sentry.tasks.sentry_apps import process_resource_change_bound
if event.get_event_type() == "error" and _should_send_error_created_hooks(
event.project
):
process_resource_change_bound.delay(
action="created", sender="Error", instance_id=event.event_id, instance=event
)
if is_new:
process_resource_change_bound.delay(
action="created", sender="Group", instance_id=event.group_id
)
from sentry.plugins.base import plugins
for plugin in plugins.for_project(event.project):
plugin_post_process_group(
plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression
)
from sentry import similarity
safe_execute(similarity.record, event.project, [event], _with_transaction=False)
if event.group_id:
# Patch attachments that were ingested on the standalone path.
update_existing_attachments(event)
if not is_reprocessed:
event_processed.send_robust(
sender=post_process_group,
project=event.project,
event=event,
primary_hash=kwargs.get("primary_hash"),
)
with metrics.timer("tasks.post_process.delete_event_cache"):
event_processing_store.delete_by_key(cache_key)
def _do_process_event(
cache_key,
start_time,
event_id,
process_task,
data=None,
data_has_changed=None,
new_process_behavior=None,
):
from sentry.plugins.base import plugins
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "process"
},
skip_internal=False)
error_logger.error("process.failed.empty",
extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_project(project_id)
event_id = data["event_id"]
project = Project.objects.get_from_cache(id=project_id)
has_changed = bool(data_has_changed)
new_process_behavior = bool(new_process_behavior)
metrics.incr("tasks.store.process_event.new_process_behavior",
tags={"value": new_process_behavior})
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
try:
if not new_process_behavior:
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
with metrics.timer("tasks.store.process_event.enhancers",
tags={"plugin": plugin.slug}):
enhancers = safe_execute(plugin.get_event_enhancers,
data=data)
for enhancer in enhancers or ():
enhanced = safe_execute(
enhancer,
data,
_passthrough_errors=(RetrySymbolication, ))
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
with metrics.timer("tasks.store.process_event.stacktraces"):
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
except RetrySymbolication as e:
if start_time and (
time() -
start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning("process.slow",
extra={
"project_id": project_id,
"event_id": event_id
})
if start_time and (
time() -
start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
error_logger.exception(
"process.failed.infinite_retry",
extra={
"project_id": project_id,
"event_id": event_id
},
)
else:
retry_process_event.apply_async(
args=(),
kwargs={
"process_task_name": process_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
# Second round of datascrubbing after stacktrace and language-specific
# processing. First round happened as part of ingest.
#
# *Right now* the only sensitive data that is added in stacktrace
# processing are usernames in filepaths, so we run directly after
# stacktrace processors and `get_event_enhancers`.
#
# We do not yet want to deal with context data produced by plugins like
# sessionstack or fullstory (which are in `get_event_preprocessors`), as
# this data is very unlikely to be sensitive data. This is why scrubbing
# happens somewhere in the middle of the pipeline.
#
# On the other hand, Javascript event error translation is happening after
# this block because it uses `get_event_preprocessors` instead of
# `get_event_enhancers`.
#
# We are fairly confident, however, that this should run *before*
# re-normalization as it is hard to find sensitive data in partially
# trimmed strings.
if (has_changed and options.get("processing.can-use-scrubbers")
and features.has("organizations:datascrubbers-v2",
project.organization,
actor=None)):
with metrics.timer("tasks.store.datascrubbers.scrub"):
project_config = get_project_config(project)
new_data = safe_execute(scrub_data,
project_config=project_config,
event=data.data)
# XXX(markus): When datascrubbing is finally "totally stable", we might want
# to drop the event if it crashes to avoid saving PII
if new_data is not None:
data.data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
with metrics.timer("tasks.store.process_event.preprocessors",
tags={"plugin": plugin.slug}):
processors = safe_execute(plugin.get_event_preprocessors,
data=data,
_with_transaction=False)
for processor in processors or ():
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data[
"project"] == project_id, "Project cannot be mutated by plugins"
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
# Run some of normalization again such that we don't:
# - persist e.g. incredibly large stacktraces from minidumps
# - store event timestamps that are older than our retention window
# (also happening with minidumps)
normalizer = StoreNormalizer(remove_other=False,
is_renormalize=True,
**DEFAULT_STORE_NORMALIZER_ARGS)
data = normalizer.normalize_event(dict(data))
issues = data.get("processing_issues")
try:
if issues and create_failed_event(
cache_key,
data,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev,
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
_do_preprocess_event(cache_key, data, start_time, event_id,
process_task, project)
return
default_cache.set(cache_key, data, 3600)
submit_save_event(project, cache_key, event_id, start_time, data)
def _do_symbolicate_event(cache_key,
start_time,
event_id,
symbolicate_task,
data=None):
from sentry.lang.native.processing import get_symbolication_function
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "symbolicate"
},
skip_internal=False)
error_logger.error("symbolicate.failed.empty",
extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_project(project_id)
event_id = data["event_id"]
project = Project.objects.get_from_cache(id=project_id)
symbolication_function = get_symbolication_function(data)
has_changed = False
from_reprocessing = symbolicate_task is symbolicate_event_from_reprocessing
try:
with metrics.timer("tasks.store.symbolicate_event.symbolication"):
symbolicated_data = safe_execute(
symbolication_function,
data,
_passthrough_errors=(RetrySymbolication, ))
if symbolicated_data:
data = symbolicated_data
has_changed = True
except RetrySymbolication as e:
error_logger.warn("retry symbolication")
if start_time and (
time() -
start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning("symbolicate.slow",
extra={
"project_id": project_id,
"event_id": event_id
})
if start_time and (
time() -
start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
error_logger.exception(
"symbolicate.failed.infinite_retry",
extra={
"project_id": project_id,
"event_id": event_id
},
)
else:
# Requeue the task in the "sleep" queue
retry_symbolicate_event.apply_async(
args=(),
kwargs={
"symbolicate_task_name": symbolicate_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
default_cache.set(cache_key, data, 3600)
submit_process(
project,
from_reprocessing,
cache_key,
event_id,
start_time,
data,
has_changed,
new_process_behavior=True,
)
def get_plugins_with_status(project):
return [
(plugin, safe_execute(plugin.is_enabled, project))
for plugin in plugins.all(version=None)
if plugin.can_enable_for_projects()
]
def get_attrs(self, item_list, user):
from sentry.plugins import plugins
GroupMeta.objects.populate_cache(item_list)
attach_foreignkey(item_list, Group.project)
if user.is_authenticated() and item_list:
bookmarks = set(
GroupBookmark.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', flat=True))
seen_groups = dict(
GroupSeen.objects.filter(
user=user,
group__in=item_list,
).values_list('group_id', 'last_seen'))
subscriptions = self._get_subscriptions(item_list, user)
else:
bookmarks = set()
seen_groups = {}
subscriptions = set()
assignees = dict((a.group_id, a.user)
for a in GroupAssignee.objects.filter(
group__in=item_list, ).select_related('user'))
user_counts = dict(
GroupTagKey.objects.filter(
group__in=item_list,
key='sentry:user',
).values_list('group', 'values_seen'))
snoozes = dict(
GroupSnooze.objects.filter(group__in=item_list, ).values_list(
'group', 'until'))
pending_resolutions = dict(
GroupResolution.objects.filter(
group__in=item_list,
status=GroupResolutionStatus.PENDING,
).values_list('group', 'release'))
result = {}
for item in item_list:
active_date = item.active_at or item.last_seen
annotations = []
for plugin in plugins.for_project(project=item.project, version=1):
safe_execute(plugin.tags,
None,
item,
annotations,
_with_transaction=False)
for plugin in plugins.for_project(project=item.project, version=2):
annotations.extend(
safe_execute(plugin.get_annotations,
group=item,
_with_transaction=False) or ())
result[item] = {
'assigned_to': serialize(assignees.get(item.id)),
'is_bookmarked': item.id in bookmarks,
'is_subscribed': item.id in subscriptions,
'has_seen':
seen_groups.get(item.id, active_date) > active_date,
'annotations': annotations,
'user_count': user_counts.get(item.id, 0),
'snooze': snoozes.get(item.id),
'pending_resolution': pending_resolutions.get(item.id),
}
return result
def handle(self, request):
settings_form = self.notification_settings_form(
request.user, request.POST or None)
reports_form = NotificationReportSettingsForm(
request.user, request.POST or None,
prefix='reports')
org_list = list(Organization.objects.filter(
status=OrganizationStatus.VISIBLE,
member_set__user=request.user,
).distinct())
org_list = [
o for o in org_list if features.has(
'organizations:release-commits',
o,
actor=request.user
)
]
org_forms = [
(org, NotificationDeploySettingsForm(
request.user,
org,
request.POST or None,
prefix='deploys-org-%s' % (org.id,)
))
for org in sorted(org_list, key=lambda o: o.name)
]
project_list = list(Project.objects.filter(
team__organizationmemberteam__organizationmember__user=request.user,
team__organizationmemberteam__is_active=True,
status=ProjectStatus.VISIBLE,
).distinct())
project_forms = [
(project, ProjectEmailOptionsForm(
project, request.user,
request.POST or None,
prefix='project-%s' % (project.id,)
))
for project in sorted(project_list, key=lambda x: (
x.organization.name, x.name))
]
ext_forms = []
for plugin in plugins.all():
for form in safe_execute(plugin.get_notification_forms, _with_transaction=False) or ():
form = safe_execute(form, plugin, request.user, request.POST or None, prefix=plugin.slug,
_with_transaction=False)
if not form:
continue
ext_forms.append(form)
if request.POST:
all_forms = list(itertools.chain(
[settings_form, reports_form],
ext_forms,
(f for _, f in project_forms),
(f for _, f in org_forms)
))
if all(f.is_valid() for f in all_forms):
for form in all_forms:
form.save()
messages.add_message(request, messages.SUCCESS, 'Your settings were saved.')
return HttpResponseRedirect(request.path)
context = csrf(request)
context.update({
'settings_form': settings_form,
'project_forms': project_forms,
'org_forms': org_forms,
'reports_form': reports_form,
'ext_forms': ext_forms,
'page': 'notifications',
'AUTH_PROVIDERS': get_auth_providers(),
})
return render_to_response('sentry/account/notifications.html', context, request)
def save(self, project, raw=False):
# TODO: culprit should default to "most recent" frame in stacktraces when
# it's not provided.
project = Project.objects.get_from_cache(id=project)
data = self.data.copy()
# First we pull out our top-level (non-data attr) kwargs
event_id = data.pop('event_id')
message = data.pop('message')
level = data.pop('level')
culprit = data.pop('culprit', None) or ''
time_spent = data.pop('time_spent', None)
logger_name = data.pop('logger', None)
server_name = data.pop('server_name', None)
site = data.pop('site', None)
checksum = data.pop('checksum', None)
platform = data.pop('platform', None)
release = data.pop('release', None)
date = datetime.fromtimestamp(data.pop('timestamp'))
date = date.replace(tzinfo=timezone.utc)
kwargs = {
'message': message,
'platform': platform,
}
event = Event(project=project,
event_id=event_id,
data=data,
time_spent=time_spent,
datetime=date,
**kwargs)
# Calculate the checksum from the first highest scoring interface
if checksum:
hashes = [checksum]
else:
hashes = get_hashes_for_event(event)
# TODO(dcramer): remove checksum usage
event.checksum = hashes[0]
group_kwargs = kwargs.copy()
group_kwargs.update({
'culprit': culprit,
'logger': logger_name,
'level': level,
'last_seen': date,
'first_seen': date,
'time_spent_total': time_spent or 0,
'time_spent_count': time_spent and 1 or 0,
})
tags = data['tags']
tags.append(('level', LOG_LEVELS[level]))
if logger_name:
tags.append(('logger', logger_name))
if server_name:
tags.append(('server_name', server_name))
if site:
tags.append(('site', site))
if release:
# TODO(dcramer): we should ensure we create Release objects
tags.append(('sentry:release', release))
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags,
event,
_with_transaction=False)
if added_tags:
tags.extend(added_tags)
group, is_new, is_regression, is_sample = safe_execute(
self._save_aggregate,
event=event,
tags=tags,
hashes=hashes,
**group_kwargs)
using = group._state.db
event.group = group
# Rounded down to the nearest interval
safe_execute(Group.objects.add_tags,
group,
tags,
_with_transaction=False)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic():
event.save()
except IntegrityError:
self.logger.info('Duplicate Event found for event_id=%s',
event_id)
return event
try:
with transaction.atomic():
EventMapping.objects.create(project=project,
group=group,
event_id=event_id)
except IntegrityError:
self.logger.info('Duplicate EventMapping found for event_id=%s',
event_id)
return event
if not raw:
post_process_group.delay(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
)
else:
self.logger.info(
'Raw event passed; skipping post process for event_id=%s',
event_id)
index_event.delay(event)
# TODO: move this to the queue
if is_regression and not raw:
regression_signal.send_robust(sender=Group, instance=group)
return event
def apply_rule(self, rule):
"""
If all conditions and filters pass, execute every action.
:param rule: `Rule` object
:return: void
"""
condition_match = rule.data.get(
"action_match") or Rule.DEFAULT_CONDITION_MATCH
filter_match = rule.data.get(
"filter_match") or Rule.DEFAULT_FILTER_MATCH
rule_condition_list = rule.data.get("conditions", ())
frequency = rule.data.get("frequency") or Rule.DEFAULT_FREQUENCY
if (rule.environment_id is not None
and self.event.get_environment().id != rule.environment_id):
return
status = self.get_rule_status(rule)
now = timezone.now()
freq_offset = now - timedelta(minutes=frequency)
if status.last_active and status.last_active > freq_offset:
return
state = self.get_state()
condition_list = []
filter_list = []
for rule_cond in rule_condition_list:
if self.get_rule_type(rule_cond) == "condition/event":
condition_list.append(rule_cond)
else:
filter_list.append(rule_cond)
# if conditions exist evaluate them, otherwise move to the filters section
if condition_list:
condition_iter = (self.condition_matches(c, state, rule)
for c in condition_list)
condition_func = self.get_match_function(condition_match)
if condition_func:
condition_passed = condition_func(condition_iter)
else:
self.logger.error("Unsupported condition_match %r for rule %d",
condition_match, rule.id)
return
if not condition_passed:
return
# if filters exist evaluate them, otherwise pass
if filter_list:
filter_iter = (self.condition_matches(f, state, rule)
for f in filter_list)
filter_func = self.get_match_function(filter_match)
if filter_func:
passed = filter_func(filter_iter)
else:
self.logger.error("Unsupported filter_match %r for rule %d",
filter_match, rule.id)
return
else:
passed = True
if passed:
passed = (GroupRuleStatus.objects.filter(id=status.id).exclude(
last_active__gt=freq_offset).update(last_active=now))
if not passed:
return
if randrange(10) == 0:
analytics.record(
"issue_alert.fired",
issue_id=self.group.id,
project_id=rule.project.id,
organization_id=rule.project.organization.id,
rule_id=rule.id,
)
for action in rule.data.get("actions", ()):
action_cls = rules.get(action["id"])
if action_cls is None:
self.logger.warn("Unregistered action %r", action["id"])
continue
action_inst = action_cls(self.project, data=action, rule=rule)
results = safe_execute(action_inst.after,
event=self.event,
state=state,
_with_transaction=False)
if results is None:
self.logger.warn("Action %s did not return any futures",
action["id"])
continue
for future in results:
key = future.key if future.key is not None else future.callback
rule_future = RuleFuture(rule=rule, kwargs=future.kwargs)
if key not in self.grouped_futures:
self.grouped_futures[key] = (future.callback,
[rule_future])
else:
self.grouped_futures[key][1].append(rule_future)
def plugin_post_process_group(plugin_slug, group, **kwargs):
"""
Fires post processing hooks for a group.
"""
plugin = plugins.get(plugin_slug)
safe_execute(plugin.post_process, group=group, **kwargs)
def save(self, project, raw=False):
from sentry.tasks.post_process import index_event_tags
project = Project.objects.get_from_cache(id=project)
data = self.data.copy()
# First we pull out our top-level (non-data attr) kwargs
event_id = data.pop('event_id')
level = data.pop('level')
culprit = data.pop('culprit', None)
logger_name = data.pop('logger', None)
server_name = data.pop('server_name', None)
site = data.pop('site', None)
checksum = data.pop('checksum', None)
fingerprint = data.pop('fingerprint', None)
platform = data.pop('platform', None)
release = data.pop('release', None)
environment = data.pop('environment', None)
# unused
time_spent = data.pop('time_spent', None)
message = data.pop('message', '')
if not culprit:
culprit = generate_culprit(data, platform=platform)
date = datetime.fromtimestamp(data.pop('timestamp'))
date = date.replace(tzinfo=timezone.utc)
kwargs = {
'platform': platform,
}
event = Event(project_id=project.id,
event_id=event_id,
data=data,
time_spent=time_spent,
datetime=date,
**kwargs)
tags = data.get('tags') or []
tags.append(('level', LOG_LEVELS[level]))
if logger_name:
tags.append(('logger', logger_name))
if server_name:
tags.append(('server_name', server_name))
if site:
tags.append(('site', site))
if release:
# TODO(dcramer): we should ensure we create Release objects
tags.append(('sentry:release', release))
if environment:
tags.append(('environment', environment))
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags,
event,
_with_transaction=False)
if added_tags:
tags.extend(added_tags)
event_user = self._get_event_user(project, data)
if event_user:
tags.append(('sentry:user', event_user.tag_value))
# XXX(dcramer): we're relying on mutation of the data object to ensure
# this propagates into Event
data['tags'] = tags
data['fingerprint'] = fingerprint or ['{{ default }}']
# Get rid of ephemeral interface data
for interface_class, _ in iter_interfaces():
interface = interface_class()
if interface.ephemeral:
data.pop(interface.get_path(), None)
# prioritize fingerprint over checksum as its likely the client defaulted
# a checksum whereas the fingerprint was explicit
if fingerprint:
hashes = map(md5_from_hash,
get_hashes_from_fingerprint(event, fingerprint))
elif checksum:
hashes = [checksum]
else:
hashes = map(md5_from_hash, get_hashes_for_event(event))
# TODO(dcramer): temp workaround for complexity
data['message'] = message
event_type = eventtypes.get(data.get('type', 'default'))(data)
event_metadata = event_type.get_metadata()
# TODO(dcramer): temp workaround for complexity
del data['message']
data['type'] = event_type.key
data['metadata'] = event_metadata
# index components into ``Event.message``
# See GH-3248
if event_type.key != 'default':
if 'sentry.interfaces.Message' in data and \
data['sentry.interfaces.Message']['message'] != message:
message = u'{} {}'.format(
message,
data['sentry.interfaces.Message']['message'],
)
if not message:
message = ''
elif not isinstance(message, basestring):
message = force_text(message)
for value in event_metadata.itervalues():
value_u = force_text(value, errors='replace')
if value_u not in message:
message = u'{} {}'.format(message, value_u)
message = trim(message.strip(), settings.SENTRY_MAX_MESSAGE_LENGTH)
event.message = message
kwargs['message'] = message
group_kwargs = kwargs.copy()
group_kwargs.update({
'culprit': culprit,
'logger': logger_name,
'level': level,
'last_seen': date,
'first_seen': date,
'data': {
'last_received':
event.data.get('received')
or float(event.datetime.strftime('%s')),
'type':
event_type.key,
# we cache the events metadata on the group to ensure its
# accessible in the stream
'metadata':
event_metadata,
},
})
if release:
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
group_kwargs['first_release'] = release
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **group_kwargs)
event.group = group
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
try:
with transaction.atomic(using=router.db_for_write(EventMapping)):
EventMapping.objects.create(project=project,
group=group,
event_id=event_id)
except IntegrityError:
self.logger.info('Duplicate EventMapping found for event_id=%s',
event_id,
exc_info=True)
return event
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(group=group)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
self.logger.info('Duplicate Event found for event_id=%s',
event_id,
exc_info=True)
return event
index_event_tags.delay(
project_id=project.id,
group_id=group.id,
event_id=event.id,
tags=tags,
)
if event_user:
tsdb.record_multi((
(tsdb.models.users_affected_by_group, group.id,
(event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id,
(event_user.tag_value, )),
),
timestamp=event.datetime)
if is_new and release:
buffer.incr(Release, {'new_groups': 1}, {
'id': release.id,
})
safe_execute(Group.objects.add_tags,
group,
tags,
_with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send(project=project,
group=group,
sender=Project)
post_process_group.delay(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
)
else:
self.logger.info(
'Raw event passed; skipping post process for event_id=%s',
event_id)
# TODO: move this to the queue
if is_regression and not raw:
regression_signal.send_robust(sender=Group, instance=group)
return event
def save(self, project_id, raw=False, assume_normalized=False):
# Normalize if needed
if not self._normalized:
if not assume_normalized:
self.normalize()
self._normalized = True
data = self._data
project = Project.objects.get_from_cache(id=project_id)
project._organization_cache = Organization.objects.get_from_cache(
id=project.organization_id)
# Check to make sure we're not about to do a bunch of work that's
# already been done if we've processed an event with this ID. (This
# isn't a perfect solution -- this doesn't handle ``EventMapping`` and
# there's a race condition between here and when the event is actually
# saved, but it's an improvement. See GH-7677.)
try:
event = Event.objects.get(
project_id=project.id,
event_id=data['event_id'],
)
except Event.DoesNotExist:
pass
else:
# Make sure we cache on the project before returning
event._project_cache = project
logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': data['event_id'],
'project_id': project.id,
'model': Event.__name__,
})
return event
# Pull out the culprit
culprit = self.get_culprit()
# Pull the toplevel data we're interested in
level = data.get('level')
# TODO(mitsuhiko): this code path should be gone by July 2018.
# This is going to be fine because no code actually still depends
# on integers here. When we need an integer it will be converted
# into one later. Old workers used to send integers here.
if level is not None and isinstance(level, six.integer_types):
level = LOG_LEVELS[level]
transaction_name = data.get('transaction')
logger_name = data.get('logger')
release = data.get('release')
dist = data.get('dist')
environment = data.get('environment')
recorded_timestamp = data.get('timestamp')
# We need to swap out the data with the one internal to the newly
# created event object
event = self._get_event_instance(project_id=project_id)
self._data = data = event.data.data
event._project_cache = project
date = event.datetime
platform = event.platform
event_id = event.event_id
if transaction_name:
transaction_name = force_text(transaction_name)
# Some of the data that are toplevel attributes are duplicated
# into tags (logger, level, environment, transaction). These are
# different from legacy attributes which are normalized into tags
# ahead of time (site, server_name).
setdefault_path(data, 'tags', value=[])
set_tag(data, 'level', level)
if logger_name:
set_tag(data, 'logger', logger_name)
if environment:
set_tag(data, 'environment', environment)
if transaction_name:
set_tag(data, 'transaction', transaction_name)
if release:
# dont allow a conflicting 'release' tag
pop_tag(data, 'release')
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
set_tag(data, 'sentry:release', release.version)
if dist and release:
dist = release.add_dist(dist, date)
# dont allow a conflicting 'dist' tag
pop_tag(data, 'dist')
set_tag(data, 'sentry:dist', dist.name)
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
pop_tag(data, 'user')
set_tag(data, 'sentry:user', event_user.tag_value)
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
normalize_in_app(data)
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags,
event,
_with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
if get_tag(data, key) is None:
set_tag(data, key, value)
for path, iface in six.iteritems(event.interfaces):
for k, v in iface.iter_tags():
set_tag(data, k, v)
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.path, None)
# The active grouping config was put into the event in the
# normalize step before. We now also make sure that the
# fingerprint was set to `'{{ default }}' just in case someone
# removed it from the payload. The call to get_hashes will then
# look at `grouping_config` to pick the right paramters.
data['fingerprint'] = data.get('fingerprint') or ['{{ default }}']
hashes = event.get_hashes()
data['hashes'] = hashes
# we want to freeze not just the metadata and type in but also the
# derived attributes. The reason for this is that we push this
# data into kafka for snuba processing and our postprocessing
# picks up the data right from the snuba topic. For most usage
# however the data is dynamically overriden by Event.title and
# Event.location (See Event.as_dict)
materialized_metadata = self.materialize_metadata()
event_metadata = materialized_metadata['metadata']
data.update(materialized_metadata)
data['culprit'] = culprit
# index components into ``Event.message``
# See GH-3248
event.message = self.get_search_message(event_metadata, culprit)
received_timestamp = event.data.get('received') or float(
event.datetime.strftime('%s'))
# The group gets the same metadata as the event when it's flushed but
# additionally the `last_received` key is set. This key is used by
# _save_aggregate.
group_metadata = dict(materialized_metadata)
group_metadata['last_received'] = received_timestamp
kwargs = {
'platform': platform,
'message': event.message,
'culprit': culprit,
'logger': logger_name,
'level': LOG_LEVELS_MAP.get(level),
'last_seen': date,
'first_seen': date,
'active_at': date,
'data': group_metadata,
}
if release:
kwargs['first_release'] = release
try:
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **kwargs)
except HashDiscarded:
event_discarded.send_robust(
project=project,
sender=EventManager,
)
metrics.incr(
'events.discarded',
skip_internal=True,
tags={
'organization_id': project.organization_id,
'platform': platform,
},
)
raise
else:
event_saved.send_robust(
project=project,
event_size=event.size,
sender=EventManager,
)
event.group = group
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
# When an event was sampled, the canonical source of truth
# is the EventMapping table since we aren't going to be writing out an actual
# Event row. Otherwise, if the Event isn't being sampled, we can safely
# rely on the Event table itself as the source of truth and ignore
# EventMapping since it's redundant information.
if is_sample:
try:
with transaction.atomic(
using=router.db_for_write(EventMapping)):
EventMapping.objects.create(project=project,
group=group,
event_id=event_id)
except IntegrityError:
logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': EventMapping.__name__,
})
return event
environment = Environment.get_or_create(
project=project,
name=environment,
)
group_environment, is_new_group_environment = GroupEnvironment.get_or_create(
group_id=group.id,
environment_id=environment.id,
defaults={
'first_release': release if release else None,
},
)
if release:
ReleaseEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
ReleaseProjectEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date,
)
counters = [
(tsdb.models.group, group.id),
(tsdb.models.project, project.id),
]
if release:
counters.append((tsdb.models.release, release.id))
tsdb.incr_multi(counters,
timestamp=event.datetime,
environment_id=environment.id)
frequencies = [
# (tsdb.models.frequent_projects_by_organization, {
# project.organization_id: {
# project.id: 1,
# },
# }),
# (tsdb.models.frequent_issues_by_project, {
# project.id: {
# group.id: 1,
# },
# })
(tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1,
},
})
]
if release:
frequencies.append((tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1,
},
}))
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(
group=group,
environment=environment,
)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': Event.__name__,
})
return event
tagstore.delay_index_event_tags(
organization_id=project.organization_id,
project_id=project.id,
group_id=group.id,
environment_id=environment.id,
event_id=event.id,
tags=event.tags,
date_added=event.datetime,
)
if event_user:
tsdb.record_multi(
(
(tsdb.models.users_affected_by_group, group.id,
(event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id,
(event_user.tag_value, )),
),
timestamp=event.datetime,
environment_id=environment.id,
)
if release:
if is_new:
buffer.incr(ReleaseProject, {'new_groups': 1}, {
'release_id': release.id,
'project_id': project.id,
})
if is_new_group_environment:
buffer.incr(ReleaseProjectEnvironment, {'new_issues_count': 1},
{
'project_id': project.id,
'release_id': release.id,
'environment_id': environment.id,
})
safe_execute(Group.objects.add_tags,
group,
environment,
event.get_tags(),
_with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send_robust(project=project,
group=group,
sender=Project)
eventstream.insert(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
is_new_group_environment=is_new_group_environment,
primary_hash=hashes[0],
# We are choosing to skip consuming the event back
# in the eventstream if it's flagged as raw.
# This means that we want to publish the event
# through the event stream, but we don't care
# about post processing and handling the commit.
skip_consume=raw,
)
metrics.timing(
'events.latency',
received_timestamp - recorded_timestamp,
tags={
'project_id': project.id,
},
)
metrics.timing('events.size.data.post_save',
event.size,
tags={'project_id': project.id})
return event
def post_process_group(event, is_new, is_regression, is_sample,
is_new_group_environment, **kwargs):
"""
Fires post processing hooks for a group.
"""
with snuba.options_override({'consistent': True}):
if check_event_already_post_processed(event):
logger.info('post_process.skipped',
extra={
'project_id': event.project_id,
'event_id': event.event_id,
'reason': 'duplicate',
})
return
# NOTE: we must pass through the full Event object, and not an
# event_id since the Event object may not actually have been stored
# in the database due to sampling.
from sentry.models import Project
from sentry.models.group import get_group_with_redirect
from sentry.rules.processor import RuleProcessor
from sentry.tasks.servicehooks import process_service_hook
# Re-bind node data to avoid renormalization. We only want to
# renormalize when loading old data from the database.
event.data = EventDict(event.data, skip_renormalization=True)
# Re-bind Group since we're pickling the whole Event object
# which may contain a stale Group.
event.group, _ = get_group_with_redirect(event.group_id)
event.group_id = event.group.id
project_id = event.group.project_id
with configure_scope() as scope:
scope.set_tag("project", project_id)
# Re-bind Project since we're pickling the whole Event object
# which may contain a stale Project.
event.project = Project.objects.get_from_cache(id=project_id)
_capture_stats(event, is_new)
# we process snoozes before rules as it might create a regression
has_reappeared = process_snoozes(event.group)
rp = RuleProcessor(event, is_new, is_regression,
is_new_group_environment, has_reappeared)
has_alert = False
# TODO(dcramer): ideally this would fanout, but serializing giant
# objects back and forth isn't super efficient
for callback, futures in rp.apply():
has_alert = True
safe_execute(callback, event, futures)
if features.has(
'projects:servicehooks',
project=event.project,
):
allowed_events = set(['event.created'])
if has_alert:
allowed_events.add('event.alert')
if allowed_events:
for servicehook_id, events in _get_service_hooks(
project_id=event.project_id):
if any(e in allowed_events for e in events):
process_service_hook.delay(
servicehook_id=servicehook_id,
event=event,
)
if is_new:
process_resource_change_bound.delay(
action='created',
sender='Group',
instance_id=event.group_id,
)
for plugin in plugins.for_project(event.project):
plugin_post_process_group(
plugin_slug=plugin.slug,
event=event,
is_new=is_new,
is_regresion=is_regression,
is_sample=is_sample,
)
event_processed.send_robust(
sender=post_process_group,
project=event.project,
group=event.group,
event=event,
primary_hash=kwargs.get('primary_hash'),
)
def process_event(event_manager, project, key, remote_addr, helper, attachments, project_config):
event_received.send_robust(ip=remote_addr, project=project, sender=process_event)
start_time = time()
data = event_manager.get_data()
should_filter, filter_reason = event_manager.should_filter()
del event_manager
event_id = data["event_id"]
if should_filter:
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.FILTERED,
filter_reason,
event_id=event_id,
)
metrics.incr("events.blacklisted", tags={"reason": filter_reason}, skip_internal=False)
event_filtered.send_robust(ip=remote_addr, project=project, sender=process_event)
raise APIForbidden("Event dropped due to filter: %s" % (filter_reason,))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(
quotas.is_rate_limited, project=project, key=key, _with_transaction=False
)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
# XXX(dcramer): when the rate limiter fails we drop events to ensure
# it cannot cascade
if rate_limit is None or rate_limit.is_limited:
if rate_limit is None:
api_logger.debug("Dropped event due to error with rate limiter")
reason = rate_limit.reason_code if rate_limit else None
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.RATE_LIMITED,
reason,
event_id=event_id,
)
metrics.incr("events.dropped", tags={"reason": reason or "unknown"}, skip_internal=False)
event_dropped.send_robust(
ip=remote_addr, project=project, reason_code=reason, sender=process_event
)
if rate_limit is not None:
raise APIRateLimited(rate_limit.retry_after)
# TODO(dcramer): ideally we'd only validate this if the event_id was
# supplied by the user
cache_key = "ev:%s:%s" % (project_config.project_id, event_id)
if cache.get(cache_key) is not None:
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.INVALID,
"duplicate",
event_id=event_id,
)
raise APIForbidden("An event with the same ID already exists (%s)" % (event_id,))
config = project_config.config
datascrubbing_settings = config.get("datascrubbingSettings") or {}
scrub_ip_address = datascrubbing_settings.get("scrubIpAddresses")
scrub_data = datascrubbing_settings.get("scrubData")
if scrub_data:
# We filter data immediately before it ever gets into the queue
sensitive_fields = datascrubbing_settings.get("sensitiveFields")
exclude_fields = datascrubbing_settings.get("excludeFields")
scrub_defaults = datascrubbing_settings.get("scrubDefaults")
SensitiveDataFilter(
fields=sensitive_fields, include_defaults=scrub_defaults, exclude_fields=exclude_fields
).apply(data)
if scrub_ip_address:
# We filter data immediately before it ever gets into the queue
helper.ensure_does_not_have_ip(data)
# mutates data (strips a lot of context if not queued)
helper.insert_data_to_database(data, start_time=start_time, attachments=attachments)
cache.set(cache_key, "", 60 * 60) # Cache for 1 hour
api_logger.debug("New event received (%s)", event_id)
event_accepted.send_robust(ip=remote_addr, data=data, project=project, sender=process_event)
return event_id
def for_project(self, project):
for plugin in self.all():
if not safe_execute(plugin.is_enabled, project):
continue
yield plugin
def save(self, project, raw=False):
from sentry.tasks.post_process import index_event_tags
project = Project.objects.get_from_cache(id=project)
# Check to make sure we're not about to do a bunch of work that's
# already been done if we've processed an event with this ID. (This
# isn't a perfect solution -- this doesn't handle ``EventMapping`` and
# there's a race condition between here and when the event is actually
# saved, but it's an improvement. See GH-7677.)
try:
event = Event.objects.get(
project_id=project.id,
event_id=self.data['event_id'],
)
except Event.DoesNotExist:
pass
else:
self.logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': self.data['event_id'],
'project_id': project.id,
'model': Event.__name__,
})
return event
data = self.data.copy()
# First we pull out our top-level (non-data attr) kwargs
event_id = data.pop('event_id')
level = data.pop('level')
culprit = data.pop('transaction', None)
if not culprit:
culprit = data.pop('culprit', None)
logger_name = data.pop('logger', None)
server_name = data.pop('server_name', None)
site = data.pop('site', None)
checksum = data.pop('checksum', None)
fingerprint = data.pop('fingerprint', None)
platform = data.pop('platform', None)
release = data.pop('release', None)
dist = data.pop('dist', None)
environment = data.pop('environment', None)
# unused
time_spent = data.pop('time_spent', None)
message = data.pop('message', '')
if not culprit:
# if we generate an implicit culprit, lets not call it a
# transaction
transaction_name = None
culprit = generate_culprit(data, platform=platform)
else:
transaction_name = culprit
culprit = force_text(culprit)
recorded_timestamp = data.pop('timestamp')
date = datetime.fromtimestamp(recorded_timestamp)
date = date.replace(tzinfo=timezone.utc)
kwargs = {
'platform': platform,
}
event = Event(project_id=project.id,
event_id=event_id,
data=data,
time_spent=time_spent,
datetime=date,
**kwargs)
event._project_cache = project
# convert this to a dict to ensure we're only storing one value per key
# as most parts of Sentry dont currently play well with multiple values
tags = dict(data.get('tags') or [])
tags['level'] = LOG_LEVELS[level]
if logger_name:
tags['logger'] = logger_name
if server_name:
tags['server_name'] = server_name
if site:
tags['site'] = site
if environment:
tags['environment'] = environment
if transaction_name:
tags['transaction'] = transaction_name
if release:
# dont allow a conflicting 'release' tag
if 'release' in tags:
del tags['release']
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
tags['sentry:release'] = release.version
if dist and release:
dist = release.add_dist(dist, date)
tags['sentry:dist'] = dist.name
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
if 'user' in tags:
del tags['user']
tags['sentry:user'] = event_user.tag_value
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
normalize_in_app(data)
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags,
event,
_with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
tags.setdefault(key, value)
for path, iface in six.iteritems(event.interfaces):
for k, v in iface.iter_tags():
tags[k] = v
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.get_path(), None)
# tags are stored as a tuple
tags = tags.items()
# XXX(dcramer): we're relying on mutation of the data object to ensure
# this propagates into Event
data['tags'] = tags
data['fingerprint'] = fingerprint or ['{{ default }}']
# prioritize fingerprint over checksum as its likely the client defaulted
# a checksum whereas the fingerprint was explicit
if fingerprint:
hashes = [
md5_from_hash(h)
for h in get_hashes_from_fingerprint(event, fingerprint)
]
elif checksum:
if HASH_RE.match(checksum):
hashes = [checksum]
else:
hashes = [md5_from_hash([checksum]), checksum]
data['checksum'] = checksum
else:
hashes = [md5_from_hash(h) for h in get_hashes_for_event(event)]
# TODO(dcramer): temp workaround for complexity
data['message'] = message
event_type = eventtypes.get(data.get('type', 'default'))(data)
event_metadata = event_type.get_metadata()
# TODO(dcramer): temp workaround for complexity
del data['message']
data['type'] = event_type.key
data['metadata'] = event_metadata
# index components into ``Event.message``
# See GH-3248
if event_type.key != 'default':
if 'sentry.interfaces.Message' in data and \
data['sentry.interfaces.Message']['message'] != message:
message = u'{} {}'.format(
message,
data['sentry.interfaces.Message']['message'],
)
if not message:
message = ''
elif not isinstance(message, six.string_types):
message = force_text(message)
for value in six.itervalues(event_metadata):
value_u = force_text(value, errors='replace')
if value_u not in message:
message = u'{} {}'.format(message, value_u)
if culprit and culprit not in message:
culprit_u = force_text(culprit, errors='replace')
message = u'{} {}'.format(message, culprit_u)
message = trim(message.strip(), settings.SENTRY_MAX_MESSAGE_LENGTH)
event.message = message
kwargs['message'] = message
received_timestamp = event.data.get('received') or float(
event.datetime.strftime('%s'))
group_kwargs = kwargs.copy()
group_kwargs.update({
'culprit': culprit,
'logger': logger_name,
'level': level,
'last_seen': date,
'first_seen': date,
'active_at': date,
'data': {
'last_received': received_timestamp,
'type': event_type.key,
# we cache the events metadata on the group to ensure its
# accessible in the stream
'metadata': event_metadata,
},
})
if release:
group_kwargs['first_release'] = release
try:
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **group_kwargs)
except HashDiscarded:
event_discarded.send_robust(
project=project,
sender=EventManager,
)
metrics.incr(
'events.discarded',
skip_internal=True,
tags={
'organization_id': project.organization_id,
'platform': platform,
},
)
raise
else:
event_saved.send_robust(
project=project,
sender=EventManager,
)
event.group = group
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
# When an event was sampled, the canonical source of truth
# is the EventMapping table since we aren't going to be writing out an actual
# Event row. Otherwise, if the Event isn't being sampled, we can safely
# rely on the Event table itself as the source of truth and ignore
# EventMapping since it's redundant information.
if is_sample:
try:
with transaction.atomic(
using=router.db_for_write(EventMapping)):
EventMapping.objects.create(project=project,
group=group,
event_id=event_id)
except IntegrityError:
self.logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': EventMapping.__name__,
})
return event
environment = Environment.get_or_create(
project=project,
name=environment,
)
group_environment, is_new_group_environment = GroupEnvironment.get_or_create(
group_id=group.id,
environment_id=environment.id,
defaults={
'first_release_id': release.id if release else None,
},
)
if release:
ReleaseEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
ReleaseProjectEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date,
)
counters = [
(tsdb.models.group, group.id),
(tsdb.models.project, project.id),
]
if release:
counters.append((tsdb.models.release, release.id))
tsdb.incr_multi(counters,
timestamp=event.datetime,
environment_id=environment.id)
frequencies = [
# (tsdb.models.frequent_projects_by_organization, {
# project.organization_id: {
# project.id: 1,
# },
# }),
# (tsdb.models.frequent_issues_by_project, {
# project.id: {
# group.id: 1,
# },
# })
(tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1,
},
})
]
if release:
frequencies.append((tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1,
},
}))
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(
group=group,
environment=environment,
)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
self.logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': Event.__name__,
})
return event
index_event_tags.delay(
organization_id=project.organization_id,
project_id=project.id,
group_id=group.id,
environment_id=environment.id,
event_id=event.id,
tags=tags,
date_added=event.datetime,
)
if event_user:
tsdb.record_multi(
(
(tsdb.models.users_affected_by_group, group.id,
(event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id,
(event_user.tag_value, )),
),
timestamp=event.datetime,
environment_id=environment.id,
)
if release:
if is_new:
buffer.incr(ReleaseProject, {'new_groups': 1}, {
'release_id': release.id,
'project_id': project.id,
})
if is_new_group_environment:
buffer.incr(ReleaseProjectEnvironment, {'new_issues_count': 1},
{
'project_id': project.id,
'release_id': release.id,
'environment_id': environment.id,
})
safe_execute(Group.objects.add_tags,
group,
environment,
tags,
_with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send(project=project,
group=group,
sender=Project)
post_process_group.delay(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
is_new_group_environment=is_new_group_environment,
primary_hash=hashes[0],
)
else:
self.logger.info('post_process.skip.raw_event',
extra={'event_id': event.id})
# TODO: move this to the queue
if is_regression and not raw:
regression_signal.send_robust(sender=Group, instance=group)
metrics.timing(
'events.latency',
received_timestamp - recorded_timestamp,
tags={
'project_id': project.id,
},
)
return event
