def _do_process_event(cache_key,
start_time,
event_id,
process_task,
data=None):
from sentry.plugins import plugins
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'process'
},
skip_internal=False)
error_logger.error('process.failed.empty',
extra={'cache_key': cache_key})
return
data = CanonicalKeyDict(data)
project_id = data['project']
with configure_scope() as scope:
scope.set_tag("project", project_id)
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
enhancers = safe_execute(plugin.get_event_enhancers, data=data)
for enhancer in (enhancers or ()):
enhanced = safe_execute(enhancer, data)
if enhanced:
data = enhanced
has_changed = True
try:
# Stacktrace based event processors.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
except RetrySymbolication as e:
if start_time and (time() - start_time) > 3600:
raise RuntimeError('Event spent one hour in processing')
retry_process_event.apply_async(args=(),
kwargs={
'process_task_name':
process_task.__name__,
'task_kwargs': {
'cache_key': cache_key,
'event_id': event_id,
'start_time': start_time,
}
},
countdown=e.retry_after)
return
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(plugin.get_event_preprocessors,
data=data,
_with_transaction=False)
for processor in (processors or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data[
'project'] == project_id, 'Project cannot be mutated by preprocessor'
project = Project.objects.get_from_cache(id=project_id)
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
issues = data.get('processing_issues')
try:
if issues and create_failed_event(
cache_key,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
from_reprocessing = process_task is process_event_from_reprocessing
submit_process(project, from_reprocessing, cache_key, event_id,
start_time, data)
process_task.delay(cache_key,
start_time=start_time,
event_id=event_id)
return
default_cache.set(cache_key, data, 3600)
submit_save_event(project, cache_key, event_id, start_time, data)
def _do_save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
project_id=None,
**kwargs):
"""
Saves an event to the database.
"""
set_current_event_project(project_id)
from sentry.event_manager import EventManager, HashDiscarded
event_type = "none"
if cache_key and data is None:
with metrics.timer(
"tasks.store.do_save_event.get_cache") as metric_tags:
data = event_processing_store.get(cache_key)
if data is not None:
metric_tags["event_type"] = event_type = data.get(
"type") or "none"
with metrics.global_tags(event_type=event_type):
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data["event_id"]
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop("project")
set_current_event_project(project_id)
# We only need to delete raw events for events that support
# reprocessing. If the data cannot be found we want to assume
# that we need to delete the raw event.
if not data or reprocessing.event_supports_reprocessing(data):
with metrics.timer("tasks.store.do_save_event.delete_raw_event"):
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "post"
},
skip_internal=False)
return
try:
with metrics.timer("tasks.store.do_save_event.event_manager.save"):
manager = EventManager(data)
# event.project.organization is populated after this statement.
manager.save(project_id,
assume_normalized=True,
start_time=start_time,
cache_key=cache_key)
# Put the updated event back into the cache so that post_process
# has the most recent data.
data = manager.get_data()
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
with metrics.timer(
"tasks.store.do_save_event.write_processing_cache"):
event_processing_store.store(data)
except HashDiscarded:
# Delete the event payload from cache since it won't show up in post-processing.
if cache_key:
with metrics.timer("tasks.store.do_save_event.delete_cache"):
event_processing_store.delete_by_key(cache_key)
finally:
reprocessing2.mark_event_reprocessed(data)
if cache_key:
with metrics.timer(
"tasks.store.do_save_event.delete_attachment_cache"):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing("events.time-to-process",
time() - start_time,
instance=data["platform"])
time_synthetic_monitoring_event(data, project_id, start_time)
def _do_process_event(cache_key, start_time, event_id, process_task):
from sentry.plugins import plugins
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'process'
})
error_logger.error('process.failed.empty',
extra={'cache_key': cache_key})
return
data = CanonicalKeyDict(data)
project = data['project']
Raven.tags_context({
'project': project,
})
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project)
# Stacktrace based event processors. These run before anything else.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(plugin.get_event_preprocessors,
data=data,
_with_transaction=False)
for processor in (processors or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data[
'project'] == project, 'Project cannot be mutated by preprocessor'
if has_changed:
issues = data.get('processing_issues')
try:
if issues and create_failed_event(
cache_key,
project,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
process_task.delay(cache_key,
start_time=start_time,
event_id=event_id)
return
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
default_cache.set(cache_key, data, 3600)
save_event.delay(cache_key=cache_key,
data=None,
start_time=start_time,
event_id=event_id,
project_id=project)
def _do_symbolicate_event(cache_key,
start_time,
event_id,
symbolicate_task,
data=None):
from sentry.lang.native.processing import get_symbolication_function
if data is None:
data = event_processing_store.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "symbolicate"
},
skip_internal=False)
error_logger.error("symbolicate.failed.empty",
extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_event_project(project_id)
event_id = data["event_id"]
if killswitch_matches_context(
"store.load-shed-symbolicate-event-projects",
{
"project_id": project_id,
"event_id": event_id,
"platform": data.get("platform") or "null",
},
):
return
symbolication_function = get_symbolication_function(data)
has_changed = False
from_reprocessing = symbolicate_task is symbolicate_event_from_reprocessing
symbolication_start_time = time()
with sentry_sdk.start_span(
op="tasks.store.symbolicate_event.symbolication") as span:
span.set_data("symbolicaton_function", symbolication_function.__name__)
with metrics.timer(
"tasks.store.symbolicate_event.symbolication",
tags={
"symbolication_function": symbolication_function.__name__
},
):
while True:
try:
with sentry_sdk.start_span(
op="tasks.store.symbolicate_event.%s" %
symbolication_function.__name__) as span:
symbolicated_data = symbolication_function(data)
span.set_data("symbolicated_data",
bool(symbolicated_data))
if symbolicated_data:
data = symbolicated_data
has_changed = True
break
except RetrySymbolication as e:
if (time() - symbolication_start_time
) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning(
"symbolicate.slow",
extra={
"project_id": project_id,
"event_id": event_id
},
)
if (time() - symbolication_start_time
) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
metrics.incr(
"tasks.store.symbolicate_event.fatal",
tags={
"reason":
"timeout",
"symbolication_function":
symbolication_function.__name__,
},
)
error_logger.exception(
"symbolicate.failed.infinite_retry",
extra={
"project_id": project_id,
"event_id": event_id
},
)
data.setdefault("_metrics",
{})["flag.processing.error"] = True
data.setdefault("_metrics",
{})["flag.processing.fatal"] = True
has_changed = True
break
else:
# sleep for `retry_after` but max 5 seconds and try again
metrics.incr(
"tasks.store.symbolicate_event.retry",
tags={
"symbolication_function":
symbolication_function.__name__
},
)
sleep(min(e.retry_after, SYMBOLICATOR_MAX_RETRY_AFTER))
continue
except Exception:
metrics.incr(
"tasks.store.symbolicate_event.fatal",
tags={
"reason":
"error",
"symbolication_function":
symbolication_function.__name__,
},
)
error_logger.exception(
"tasks.store.symbolicate_event.symbolication")
data.setdefault("_metrics",
{})["flag.processing.error"] = True
data.setdefault("_metrics",
{})["flag.processing.fatal"] = True
has_changed = True
break
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
cache_key = event_processing_store.store(data)
process_task = process_event_from_reprocessing if from_reprocessing else process_event
_do_process_event(
cache_key=cache_key,
start_time=start_time,
event_id=event_id,
process_task=process_task,
data=data,
data_has_changed=has_changed,
from_symbolicate=True,
)
def _do_process_event(
cache_key,
start_time,
event_id,
process_task,
data=None,
data_has_changed=None,
from_symbolicate=False,
):
from sentry.plugins.base import plugins
if data is None:
data = event_processing_store.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "process"
},
skip_internal=False)
error_logger.error("process.failed.empty",
extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_event_project(project_id)
event_id = data["event_id"]
if killswitch_matches_context(
"store.load-shed-process-event-projects",
{
"project_id": project_id,
"event_id": event_id,
"platform": data.get("platform") or "null",
},
):
return
with sentry_sdk.start_span(
op="tasks.store.process_event.get_project_from_cache"):
project = Project.objects.get_from_cache(id=project_id)
with metrics.timer(
"tasks.store.process_event.organization.get_from_cache"):
project._organization_cache = Organization.objects.get_from_cache(
id=project.organization_id)
has_changed = bool(data_has_changed)
with sentry_sdk.start_span(
op="tasks.store.process_event.get_reprocessing_revision"):
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
# Stacktrace based event processors.
with sentry_sdk.start_span(op="task.store.process_event.stacktraces"):
with metrics.timer("tasks.store.process_event.stacktraces",
tags={"from_symbolicate": from_symbolicate}):
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
# Second round of datascrubbing after stacktrace and language-specific
# processing. First round happened as part of ingest.
#
# *Right now* the only sensitive data that is added in stacktrace
# processing are usernames in filepaths, so we run directly after
# stacktrace processors.
#
# We do not yet want to deal with context data produced by plugins like
# sessionstack or fullstory (which are in `get_event_preprocessors`), as
# this data is very unlikely to be sensitive data. This is why scrubbing
# happens somewhere in the middle of the pipeline.
#
# On the other hand, Javascript event error translation is happening after
# this block because it uses `get_event_preprocessors` instead of
# `get_event_enhancers`.
#
# We are fairly confident, however, that this should run *before*
# re-normalization as it is hard to find sensitive data in partially
# trimmed strings.
if has_changed and options.get("processing.can-use-scrubbers"):
with sentry_sdk.start_span(op="task.store.datascrubbers.scrub"):
with metrics.timer("tasks.store.datascrubbers.scrub",
tags={"from_symbolicate": from_symbolicate}):
new_data = safe_execute(scrub_data,
project=project,
event=data.data)
# XXX(markus): When datascrubbing is finally "totally stable", we might want
# to drop the event if it crashes to avoid saving PII
if new_data is not None:
data.data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
with sentry_sdk.start_span(
op="task.store.process_event.preprocessors") as span:
span.set_data("plugin", plugin.slug)
span.set_data("from_symbolicate", from_symbolicate)
with metrics.timer(
"tasks.store.process_event.preprocessors",
tags={
"plugin": plugin.slug,
"from_symbolicate": from_symbolicate
},
):
processors = safe_execute(plugin.get_event_preprocessors,
data=data,
_with_transaction=False)
for processor in processors or ():
try:
result = processor(data)
except Exception:
error_logger.exception(
"tasks.store.preprocessors.error")
data.setdefault("_metrics",
{})["flag.processing.error"] = True
has_changed = True
else:
if result:
data = result
has_changed = True
assert data[
"project"] == project_id, "Project cannot be mutated by plugins"
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
# Run some of normalization again such that we don't:
# - persist e.g. incredibly large stacktraces from minidumps
# - store event timestamps that are older than our retention window
# (also happening with minidumps)
normalizer = StoreNormalizer(remove_other=False,
is_renormalize=True,
**DEFAULT_STORE_NORMALIZER_ARGS)
data = normalizer.normalize_event(dict(data))
issues = data.get("processing_issues")
try:
if issues and create_failed_event(
cache_key,
data,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev,
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke ourselves again. This happens when the reprocessing
# revision changed while we were processing.
_do_preprocess_event(cache_key, data, start_time, event_id,
process_task, project)
return
cache_key = event_processing_store.store(data)
from_reprocessing = process_task is process_event_from_reprocessing
submit_save_event(project, from_reprocessing, cache_key, event_id,
start_time, data)
def _do_symbolicate_event(cache_key, start_time, event_id, symbolicate_task, data=None):
from sentry.lang.native.processing import get_symbolication_function
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr(
"events.failed", tags={"reason": "cache", "stage": "symbolicate"}, skip_internal=False
)
error_logger.error("symbolicate.failed.empty", extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_project(project_id)
event_id = data["event_id"]
symbolication_function = get_symbolication_function(data)
has_changed = False
from_reprocessing = symbolicate_task is symbolicate_event_from_reprocessing
try:
with sentry_sdk.start_span(op="tasks.store.symbolicate_event.symbolication") as span:
span.set_data("symbolicaton_function", symbolication_function.__name__)
with metrics.timer("tasks.store.symbolicate_event.symbolication"):
symbolicated_data = safe_execute(
symbolication_function, data, _passthrough_errors=(RetrySymbolication,)
)
span.set_data("symbolicated_data", bool(symbolicated_data))
if symbolicated_data:
data = symbolicated_data
has_changed = True
except RetrySymbolication as e:
if start_time and (time() - start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning(
"symbolicate.slow", extra={"project_id": project_id, "event_id": event_id}
)
if start_time and (time() - start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
error_logger.exception(
"symbolicate.failed.infinite_retry",
extra={"project_id": project_id, "event_id": event_id},
)
else:
# Requeue the task in the "sleep" queue
retry_symbolicate_event.apply_async(
args=(),
kwargs={
"symbolicate_task_name": symbolicate_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
default_cache.set(cache_key, data, 3600)
process_task = process_event_from_reprocessing if from_reprocessing else process_event
_do_process_event(
cache_key=cache_key,
start_time=start_time,
event_id=event_id,
process_task=process_task,
data=data,
data_has_changed=has_changed,
from_symbolicate=True,
)
def _do_process_event(cache_key,
start_time,
event_id,
process_task,
data=None):
from sentry.plugins.base import plugins
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "process"
},
skip_internal=False)
error_logger.error("process.failed.empty",
extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
event_id = data["event_id"]
with configure_scope() as scope:
scope.set_tag("project", project_id)
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
try:
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
enhancers = safe_execute(plugin.get_event_enhancers, data=data)
for enhancer in enhancers or ():
enhanced = safe_execute(
enhancer, data, _passthrough_errors=(RetrySymbolication, ))
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
except RetrySymbolication as e:
if start_time and (time() - start_time) > 120:
error_logger.warning("process.slow",
extra={
"project_id": project_id,
"event_id": event_id
})
if start_time and (time() - start_time) > 3600:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
error_logger.exception(
"process.failed.infinite_retry",
extra={
"project_id": project_id,
"event_id": event_id
},
)
else:
retry_process_event.apply_async(
args=(),
kwargs={
"process_task_name": process_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(plugin.get_event_preprocessors,
data=data,
_with_transaction=False)
for processor in processors or ():
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data[
"project"] == project_id, "Project cannot be mutated by preprocessor"
project = Project.objects.get_from_cache(id=project_id)
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
# Run some of normalization again such that we don't:
# - persist e.g. incredibly large stacktraces from minidumps
# - store event timestamps that are older than our retention window
# (also happening with minidumps)
normalizer = StoreNormalizer(remove_other=False,
is_renormalize=True,
**DEFAULT_STORE_NORMALIZER_ARGS)
data = normalizer.normalize_event(dict(data))
issues = data.get("processing_issues")
try:
if issues and create_failed_event(
cache_key,
data,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev,
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
from_reprocessing = process_task is process_event_from_reprocessing
submit_process(project, from_reprocessing, cache_key, event_id,
start_time, data)
process_task.delay(cache_key,
start_time=start_time,
event_id=event_id)
return
default_cache.set(cache_key, data, 3600)
submit_save_event(project, cache_key, event_id, start_time, data)
def _do_process_event(cache_key, start_time, event_id, process_task,
data=None):
from sentry.plugins import plugins
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr(
'events.failed',
tags={
'reason': 'cache',
'stage': 'process'},
skip_internal=False)
error_logger.error('process.failed.empty', extra={'cache_key': cache_key})
return
data = CanonicalKeyDict(data)
project_id = data['project']
with configure_scope() as scope:
scope.set_tag("project", project_id)
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
try:
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
enhancers = safe_execute(plugin.get_event_enhancers, data=data)
for enhancer in (enhancers or ()):
enhanced = safe_execute(enhancer, data, _passthrough_errors=(RetrySymbolication,))
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
except RetrySymbolication as e:
if start_time and (time() - start_time) > 3600:
raise RuntimeError('Event spent one hour in processing')
retry_process_event.apply_async(
args=(),
kwargs={
'process_task_name': process_task.__name__,
'task_kwargs': {
'cache_key': cache_key,
'event_id': event_id,
'start_time': start_time,
}
},
countdown=e.retry_after
)
return
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(
plugin.get_event_preprocessors, data=data, _with_transaction=False
)
for processor in (processors or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data['project'] == project_id, 'Project cannot be mutated by preprocessor'
project = Project.objects.get_from_cache(id=project_id)
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
issues = data.get('processing_issues')
try:
if issues and create_failed_event(
cache_key, project_id, list(issues.values()),
event_id=event_id, start_time=start_time,
reprocessing_rev=reprocessing_rev
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
from_reprocessing = process_task is process_event_from_reprocessing
submit_process(project, from_reprocessing, cache_key, event_id, start_time, data)
process_task.delay(cache_key, start_time=start_time,
event_id=event_id)
return
default_cache.set(cache_key, data, 3600)
submit_save_event(project, cache_key, event_id, start_time, data)
def _do_process_event(cache_key, start_time, event_id, process_task, data=None):
from sentry.plugins.base import plugins
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr(
"events.failed", tags={"reason": "cache", "stage": "process"}, skip_internal=False
)
error_logger.error("process.failed.empty", extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
event_id = data["event_id"]
project = Project.objects.get_from_cache(id=project_id)
with configure_scope() as scope:
scope.set_tag("project", project_id)
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
try:
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
enhancers = safe_execute(plugin.get_event_enhancers, data=data)
for enhancer in enhancers or ():
enhanced = safe_execute(enhancer, data, _passthrough_errors=(RetrySymbolication,))
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
except RetrySymbolication as e:
if start_time and (time() - start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning(
"process.slow", extra={"project_id": project_id, "event_id": event_id}
)
if start_time and (time() - start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
error_logger.exception(
"process.failed.infinite_retry",
extra={"project_id": project_id, "event_id": event_id},
)
else:
retry_process_event.apply_async(
args=(),
kwargs={
"process_task_name": process_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
# Second round of datascrubbing after stacktrace and language-specific
# processing. First round happened as part of ingest.
#
# We assume that all potential PII is produced as part of stacktrace
# processors and event enhancers.
#
# We assume that plugins for eg sessionstack (running via
# `plugin.get_event_preprocessors`) are not producing data that should be
# PII-stripped, ever.
#
# XXX(markus): Javascript event error translation is happening after this block
# because it uses `get_event_preprocessors` instead of
# `get_event_enhancers`, possibly move?
if has_changed and features.has(
"organizations:datascrubbers-v2", project.organization, actor=None
):
with metrics.timer("tasks.store.datascrubbers.scrub"):
project_config = get_project_config(project)
new_data = safe_execute(scrub_data, project_config=project_config, event=data.data)
# XXX(markus): When datascrubbing is finally "totally stable", we might want
# to drop the event if it crashes to avoid saving PII
if new_data is not None:
data.data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(
plugin.get_event_preprocessors, data=data, _with_transaction=False
)
for processor in processors or ():
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data["project"] == project_id, "Project cannot be mutated by plugins"
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
# Run some of normalization again such that we don't:
# - persist e.g. incredibly large stacktraces from minidumps
# - store event timestamps that are older than our retention window
# (also happening with minidumps)
normalizer = StoreNormalizer(
remove_other=False, is_renormalize=True, **DEFAULT_STORE_NORMALIZER_ARGS
)
data = normalizer.normalize_event(dict(data))
issues = data.get("processing_issues")
try:
if issues and create_failed_event(
cache_key,
data,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev,
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
from_reprocessing = process_task is process_event_from_reprocessing
submit_process(project, from_reprocessing, cache_key, event_id, start_time, data)
process_task.delay(cache_key, start_time=start_time, event_id=event_id)
return
default_cache.set(cache_key, data, 3600)
submit_save_event(project, cache_key, event_id, start_time, data)
def _do_process_event(cache_key, start_time, event_id, process_task):
from sentry.plugins import plugins
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'process'})
error_logger.error('process.failed.empty', extra={'cache_key': cache_key})
return
data = CanonicalKeyDict(data)
project = data['project']
Raven.tags_context({
'project': project,
})
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project)
# Stacktrace based event processors. These run before anything else.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(
plugin.get_event_preprocessors, data=data, _with_transaction=False
)
for processor in (processors or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data['project'] == project, 'Project cannot be mutated by preprocessor'
if has_changed:
issues = data.get('processing_issues')
try:
if issues and create_failed_event(
cache_key, project, list(issues.values()),
event_id=event_id, start_time=start_time,
reprocessing_rev=reprocessing_rev
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
process_task.delay(cache_key, start_time=start_time,
event_id=event_id)
return
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
default_cache.set(cache_key, data, 3600)
save_event.delay(
cache_key=cache_key, data=None, start_time=start_time, event_id=event_id,
project_id=project
)
def _do_symbolicate_event(
cache_key, start_time, event_id, symbolicate_task, data=None, queue_switches=0
):
from sentry.lang.native.processing import get_symbolication_function
if data is None:
data = event_processing_store.get(cache_key)
if data is None:
metrics.incr(
"events.failed", tags={"reason": "cache", "stage": "symbolicate"}, skip_internal=False
)
error_logger.error("symbolicate.failed.empty", extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_event_project(project_id)
event_id = data["event_id"]
from_reprocessing = (
symbolicate_task is symbolicate_event_from_reprocessing
or symbolicate_task is symbolicate_event_from_reprocessing_low_priority
)
# check whether the event is in the wrong queue and if so, move it to the other one.
# we do this at most SYMBOLICATOR_MAX_QUEUE_SWITCHES times.
if queue_switches >= SYMBOLICATOR_MAX_QUEUE_SWITCHES:
metrics.gauge("tasks.store.symbolicate_event.low_priority.max_queue_switches", 1)
else:
is_low_priority = symbolicate_task in [
symbolicate_event_low_priority,
symbolicate_event_from_reprocessing_low_priority,
]
should_be_low_priority = should_demote_symbolication(project_id)
if is_low_priority != should_be_low_priority:
metrics.gauge("tasks.store.symbolicate_event.low_priority.wrong_queue", 1)
submit_symbolicate(
should_be_low_priority,
from_reprocessing,
cache_key,
event_id,
start_time,
data,
queue_switches + 1,
)
return
def _continue_to_process_event():
process_task = process_event_from_reprocessing if from_reprocessing else process_event
_do_process_event(
cache_key=cache_key,
start_time=start_time,
event_id=event_id,
process_task=process_task,
data=data,
data_has_changed=has_changed,
from_symbolicate=True,
)
symbolication_function = get_symbolication_function(data)
symbolication_function_name = getattr(symbolication_function, "__name__", "none")
if killswitch_matches_context(
"store.load-shed-symbolicate-event-projects",
{
"project_id": project_id,
"event_id": event_id,
"platform": data.get("platform") or "null",
"symbolication_function": symbolication_function_name,
},
):
return _continue_to_process_event()
has_changed = False
symbolication_start_time = time()
submission_ratio = options.get("symbolicate-event.low-priority.metrics.submission-rate")
submit_realtime_metrics = not from_reprocessing and random.random() < submission_ratio
if submit_realtime_metrics:
with sentry_sdk.start_span(op="tasks.store.symbolicate_event.low_priority.metrics.counter"):
timestamp = int(symbolication_start_time)
try:
realtime_metrics.increment_project_event_counter(project_id, timestamp)
except Exception as e:
sentry_sdk.capture_exception(e)
with sentry_sdk.start_span(op="tasks.store.symbolicate_event.symbolication") as span:
span.set_data("symbolication_function", symbolication_function_name)
with metrics.timer(
"tasks.store.symbolicate_event.symbolication",
tags={"symbolication_function": symbolication_function_name},
):
while True:
try:
with sentry_sdk.start_span(
op="tasks.store.symbolicate_event.%s" % symbolication_function_name
) as span:
symbolicated_data = symbolication_function(data)
span.set_data("symbolicated_data", bool(symbolicated_data))
if symbolicated_data:
data = symbolicated_data
has_changed = True
break
except RetrySymbolication as e:
if (
time() - symbolication_start_time
) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning(
"symbolicate.slow",
extra={"project_id": project_id, "event_id": event_id},
)
if (
time() - symbolication_start_time
) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
metrics.incr(
"tasks.store.symbolicate_event.fatal",
tags={
"reason": "timeout",
"symbolication_function": symbolication_function_name,
},
)
error_logger.exception(
"symbolicate.failed.infinite_retry",
extra={"project_id": project_id, "event_id": event_id},
)
data.setdefault("_metrics", {})["flag.processing.error"] = True
data.setdefault("_metrics", {})["flag.processing.fatal"] = True
has_changed = True
break
else:
# sleep for `retry_after` but max 5 seconds and try again
metrics.incr(
"tasks.store.symbolicate_event.retry",
tags={"symbolication_function": symbolication_function_name},
)
sleep(min(e.retry_after, SYMBOLICATOR_MAX_RETRY_AFTER))
continue
except Exception:
metrics.incr(
"tasks.store.symbolicate_event.fatal",
tags={
"reason": "error",
"symbolication_function": symbolication_function_name,
},
)
error_logger.exception("tasks.store.symbolicate_event.symbolication")
data.setdefault("_metrics", {})["flag.processing.error"] = True
data.setdefault("_metrics", {})["flag.processing.fatal"] = True
has_changed = True
break
if submit_realtime_metrics:
with sentry_sdk.start_span(
op="tasks.store.symbolicate_event.low_priority.metrics.histogram"
):
symbolication_duration = int(time() - symbolication_start_time)
try:
realtime_metrics.increment_project_duration_counter(
project_id, timestamp, symbolication_duration
)
except Exception as e:
sentry_sdk.capture_exception(e)
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
cache_key = event_processing_store.store(data)
return _continue_to_process_event()
def _do_process_event(
cache_key,
start_time,
event_id,
process_task,
data=None,
data_has_changed=None,
new_process_behavior=None,
):
from sentry.plugins.base import plugins
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "process"
},
skip_internal=False)
error_logger.error("process.failed.empty",
extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_project(project_id)
event_id = data["event_id"]
project = Project.objects.get_from_cache(id=project_id)
has_changed = bool(data_has_changed)
new_process_behavior = bool(new_process_behavior)
metrics.incr("tasks.store.process_event.new_process_behavior",
tags={"value": new_process_behavior})
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
try:
if not new_process_behavior:
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
with metrics.timer("tasks.store.process_event.enhancers",
tags={"plugin": plugin.slug}):
enhancers = safe_execute(plugin.get_event_enhancers,
data=data)
for enhancer in enhancers or ():
enhanced = safe_execute(
enhancer,
data,
_passthrough_errors=(RetrySymbolication, ))
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
with metrics.timer("tasks.store.process_event.stacktraces"):
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
except RetrySymbolication as e:
if start_time and (
time() -
start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning("process.slow",
extra={
"project_id": project_id,
"event_id": event_id
})
if start_time and (
time() -
start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
error_logger.exception(
"process.failed.infinite_retry",
extra={
"project_id": project_id,
"event_id": event_id
},
)
else:
retry_process_event.apply_async(
args=(),
kwargs={
"process_task_name": process_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
# Second round of datascrubbing after stacktrace and language-specific
# processing. First round happened as part of ingest.
#
# *Right now* the only sensitive data that is added in stacktrace
# processing are usernames in filepaths, so we run directly after
# stacktrace processors and `get_event_enhancers`.
#
# We do not yet want to deal with context data produced by plugins like
# sessionstack or fullstory (which are in `get_event_preprocessors`), as
# this data is very unlikely to be sensitive data. This is why scrubbing
# happens somewhere in the middle of the pipeline.
#
# On the other hand, Javascript event error translation is happening after
# this block because it uses `get_event_preprocessors` instead of
# `get_event_enhancers`.
#
# We are fairly confident, however, that this should run *before*
# re-normalization as it is hard to find sensitive data in partially
# trimmed strings.
if (has_changed and options.get("processing.can-use-scrubbers")
and features.has("organizations:datascrubbers-v2",
project.organization,
actor=None)):
with metrics.timer("tasks.store.datascrubbers.scrub"):
project_config = get_project_config(project)
new_data = safe_execute(scrub_data,
project_config=project_config,
event=data.data)
# XXX(markus): When datascrubbing is finally "totally stable", we might want
# to drop the event if it crashes to avoid saving PII
if new_data is not None:
data.data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
with metrics.timer("tasks.store.process_event.preprocessors",
tags={"plugin": plugin.slug}):
processors = safe_execute(plugin.get_event_preprocessors,
data=data,
_with_transaction=False)
for processor in processors or ():
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data[
"project"] == project_id, "Project cannot be mutated by plugins"
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
# Run some of normalization again such that we don't:
# - persist e.g. incredibly large stacktraces from minidumps
# - store event timestamps that are older than our retention window
# (also happening with minidumps)
normalizer = StoreNormalizer(remove_other=False,
is_renormalize=True,
**DEFAULT_STORE_NORMALIZER_ARGS)
data = normalizer.normalize_event(dict(data))
issues = data.get("processing_issues")
try:
if issues and create_failed_event(
cache_key,
data,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev,
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
_do_preprocess_event(cache_key, data, start_time, event_id,
process_task, project)
return
default_cache.set(cache_key, data, 3600)
submit_save_event(project, cache_key, event_id, start_time, data)
