def post_process_group(is_new,
is_regression,
is_new_group_environment,
cache_key,
group_id=None,
**kwargs):
"""
Fires post processing hooks for a group.
"""
from sentry.eventstore.models import Event
from sentry.eventstore.processing import event_processing_store
from sentry.utils import snuba
from sentry.reprocessing2 import is_reprocessed_event
with snuba.options_override({"consistent": True}):
# We use the data being present/missing in the processing store
# to ensure that we don't duplicate work should the forwarding consumers
# need to rewind history.
data = event_processing_store.get(cache_key)
if not data:
logger.info(
"post_process.skipped",
extra={
"cache_key": cache_key,
"reason": "missing_cache"
},
)
return
event = Event(project_id=data["project"],
event_id=data["event_id"],
group_id=group_id,
data=data)
set_current_project(event.project_id)
is_reprocessed = is_reprocessed_event(event.data)
# NOTE: we must pass through the full Event object, and not an
# event_id since the Event object may not actually have been stored
# in the database due to sampling.
from sentry.models import (
Project,
Organization,
EventDict,
GroupInboxReason,
)
from sentry.models.groupinbox import add_group_to_inbox
from sentry.models.group import get_group_with_redirect
from sentry.rules.processor import RuleProcessor
from sentry.tasks.servicehooks import process_service_hook
# Re-bind node data to avoid renormalization. We only want to
# renormalize when loading old data from the database.
event.data = EventDict(event.data, skip_renormalization=True)
# Re-bind Project and Org since we're reading the Event object
# from cache which may contain stale parent models.
event.project = Project.objects.get_from_cache(id=event.project_id)
event.project._organization_cache = Organization.objects.get_from_cache(
id=event.project.organization_id)
if event.group_id:
# Re-bind Group since we're reading the Event object
# from cache, which may contain a stale group and project
event.group, _ = get_group_with_redirect(event.group_id)
event.group_id = event.group.id
event.group.project = event.project
event.group.project._organization_cache = event.project._organization_cache
bind_organization_context(event.project.organization)
_capture_stats(event, is_new)
if event.group_id and not is_reprocessed:
# we process snoozes before rules as it might create a regression
# but not if it's new because you can't immediately snooze a new group
has_reappeared = False if is_new else process_snoozes(event.group)
if not has_reappeared: # If true, we added the .UNIGNORED reason already
if is_new:
add_group_to_inbox(event.group, GroupInboxReason.NEW)
elif is_regression:
add_group_to_inbox(event.group,
GroupInboxReason.REGRESSION)
handle_owner_assignment(event.project, event.group, event)
rp = RuleProcessor(event, is_new, is_regression,
is_new_group_environment, has_reappeared)
has_alert = False
# TODO(dcramer): ideally this would fanout, but serializing giant
# objects back and forth isn't super efficient
for callback, futures in rp.apply():
has_alert = True
with sentry_sdk.start_transaction(op="post_process_group",
name="rule_processor_apply",
sampled=True):
safe_execute(callback, event, futures)
if features.has("projects:servicehooks", project=event.project):
allowed_events = set(["event.created"])
if has_alert:
allowed_events.add("event.alert")
if allowed_events:
for servicehook_id, events in _get_service_hooks(
project_id=event.project_id):
if any(e in allowed_events for e in events):
process_service_hook.delay(
servicehook_id=servicehook_id, event=event)
from sentry.tasks.sentry_apps import process_resource_change_bound
if event.get_event_type(
) == "error" and _should_send_error_created_hooks(event.project):
process_resource_change_bound.delay(action="created",
sender="Error",
instance_id=event.event_id,
instance=event)
if is_new:
process_resource_change_bound.delay(action="created",
sender="Group",
instance_id=event.group_id)
from sentry.plugins.base import plugins
for plugin in plugins.for_project(event.project):
plugin_post_process_group(plugin_slug=plugin.slug,
event=event,
is_new=is_new,
is_regresion=is_regression)
from sentry import similarity
safe_execute(similarity.record, event.project, [event])
if event.group_id:
# Patch attachments that were ingested on the standalone path.
update_existing_attachments(event)
if not is_reprocessed:
event_processed.send_robust(
sender=post_process_group,
project=event.project,
event=event,
primary_hash=kwargs.get("primary_hash"),
)
with metrics.timer("tasks.post_process.delete_event_cache"):
event_processing_store.delete_by_key(cache_key)
def _do_save_event(
cache_key=None, data=None, start_time=None, event_id=None, project_id=None, **kwargs
):
"""
Saves an event to the database.
"""
set_current_project(project_id)
from sentry.event_manager import EventManager, HashDiscarded
event_type = "none"
if cache_key and data is None:
with metrics.timer("tasks.store.do_save_event.get_cache") as metric_tags:
data = event_processing_store.get(cache_key)
if data is not None:
metric_tags["event_type"] = event_type = data.get("type") or "none"
with metrics.global_tags(event_type=event_type):
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data["event_id"]
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop("project")
set_current_project(project_id)
# We only need to delete raw events for events that support
# reprocessing. If the data cannot be found we want to assume
# that we need to delete the raw event.
if not data or reprocessing.event_supports_reprocessing(data):
with metrics.timer("tasks.store.do_save_event.delete_raw_event"):
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr(
"events.failed", tags={"reason": "cache", "stage": "post"}, skip_internal=False
)
return
try:
with metrics.timer("tasks.store.do_save_event.event_manager.save"):
manager = EventManager(data)
# event.project.organization is populated after this statement.
manager.save(
project_id, assume_normalized=True, start_time=start_time, cache_key=cache_key
)
except HashDiscarded:
pass
finally:
if cache_key:
with metrics.timer("tasks.store.do_save_event.delete_cache"):
event_processing_store.delete_by_key(cache_key)
with metrics.timer("tasks.store.do_save_event.delete_attachment_cache"):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing(
"events.time-to-process", time() - start_time, instance=data["platform"]
)
def post_process_group(is_new,
is_regression,
is_new_group_environment,
cache_key,
group_id=None,
**kwargs):
"""
Fires post processing hooks for a group.
"""
from sentry.eventstore.models import Event
from sentry.eventstore.processing import event_processing_store
from sentry.reprocessing2 import is_reprocessed_event
from sentry.utils import snuba
with snuba.options_override({"consistent": True}):
# We use the data being present/missing in the processing store
# to ensure that we don't duplicate work should the forwarding consumers
# need to rewind history.
data = event_processing_store.get(cache_key)
if not data:
logger.info(
"post_process.skipped",
extra={
"cache_key": cache_key,
"reason": "missing_cache"
},
)
return
event = Event(project_id=data["project"],
event_id=data["event_id"],
group_id=group_id,
data=data)
set_current_event_project(event.project_id)
is_transaction_event = not bool(event.group_id)
from sentry.models import EventDict, Organization, Project
# Re-bind node data to avoid renormalization. We only want to
# renormalize when loading old data from the database.
event.data = EventDict(event.data, skip_renormalization=True)
# Re-bind Project and Org since we're reading the Event object
# from cache which may contain stale parent models.
event.project = Project.objects.get_from_cache(id=event.project_id)
event.project.set_cached_field_value(
"organization",
Organization.objects.get_from_cache(
id=event.project.organization_id))
# Simplified post processing for transaction events.
# This should eventually be completely removed and transactions
# will not go through any post processing.
if is_transaction_event:
transaction_processed.send_robust(
sender=post_process_group,
project=event.project,
event=event,
)
event_processing_store.delete_by_key(cache_key)
return
is_reprocessed = is_reprocessed_event(event.data)
sentry_sdk.set_tag("is_reprocessed", is_reprocessed)
# NOTE: we must pass through the full Event object, and not an
# event_id since the Event object may not actually have been stored
# in the database due to sampling.
from sentry.models import Commit, GroupInboxReason
from sentry.models.group import get_group_with_redirect
from sentry.models.groupinbox import add_group_to_inbox
from sentry.rules.processor import RuleProcessor
from sentry.tasks.groupowner import process_suspect_commits
from sentry.tasks.servicehooks import process_service_hook
# Re-bind Group since we're reading the Event object
# from cache, which may contain a stale group and project
event.group, _ = get_group_with_redirect(event.group_id)
event.group_id = event.group.id
event.group.project = event.project
event.group.project.set_cached_field_value("organization",
event.project.organization)
bind_organization_context(event.project.organization)
_capture_stats(event, is_new)
with sentry_sdk.start_span(
op="tasks.post_process_group.add_group_to_inbox"):
try:
if is_reprocessed and is_new:
add_group_to_inbox(event.group,
GroupInboxReason.REPROCESSED)
except Exception:
logger.exception(
"Failed to add group to inbox for reprocessed groups")
if not is_reprocessed:
# we process snoozes before rules as it might create a regression
# but not if it's new because you can't immediately snooze a new group
has_reappeared = not is_new
try:
if has_reappeared:
has_reappeared = process_snoozes(event.group)
except Exception:
logger.exception("Failed to process snoozes for group")
try:
if not has_reappeared: # If true, we added the .UNIGNORED reason already
if is_new:
add_group_to_inbox(event.group, GroupInboxReason.NEW)
elif is_regression:
add_group_to_inbox(event.group,
GroupInboxReason.REGRESSION)
except Exception:
logger.exception(
"Failed to add group to inbox for non-reprocessed groups")
with sentry_sdk.start_span(
op="tasks.post_process_group.handle_owner_assignment"):
try:
handle_owner_assignment(event.project, event.group, event)
except Exception:
logger.exception("Failed to handle owner assignments")
rp = RuleProcessor(event, is_new, is_regression,
is_new_group_environment, has_reappeared)
has_alert = False
with sentry_sdk.start_span(
op="tasks.post_process_group.rule_processor_callbacks"):
# TODO(dcramer): ideally this would fanout, but serializing giant
# objects back and forth isn't super efficient
for callback, futures in rp.apply():
has_alert = True
safe_execute(callback,
event,
futures,
_with_transaction=False)
try:
lock = locks.get(
f"w-o:{event.group_id}-d-l",
duration=10,
)
with lock.acquire():
has_commit_key = f"w-o:{event.project.organization_id}-h-c"
org_has_commit = cache.get(has_commit_key)
if org_has_commit is None:
org_has_commit = Commit.objects.filter(
organization_id=event.project.organization_id
).exists()
cache.set(has_commit_key, org_has_commit, 3600)
if org_has_commit:
group_cache_key = f"w-o-i:g-{event.group_id}"
if cache.get(group_cache_key):
metrics.incr(
"sentry.tasks.process_suspect_commits.debounce",
tags={"detail": "w-o-i:g debounce"},
)
else:
from sentry.utils.committers import get_frame_paths
cache.set(group_cache_key, True,
604800) # 1 week in seconds
event_frames = get_frame_paths(event.data)
process_suspect_commits.delay(
event_id=event.event_id,
event_platform=event.platform,
event_frames=event_frames,
group_id=event.group_id,
project_id=event.project_id,
)
except UnableToAcquireLock:
pass
except Exception:
logger.exception("Failed to process suspect commits")
if features.has("projects:servicehooks", project=event.project):
allowed_events = {"event.created"}
if has_alert:
allowed_events.add("event.alert")
if allowed_events:
for servicehook_id, events in _get_service_hooks(
project_id=event.project_id):
if any(e in allowed_events for e in events):
process_service_hook.delay(
servicehook_id=servicehook_id, event=event)
from sentry.tasks.sentry_apps import process_resource_change_bound
if event.get_event_type(
) == "error" and _should_send_error_created_hooks(event.project):
process_resource_change_bound.delay(action="created",
sender="Error",
instance_id=event.event_id,
instance=event)
if is_new:
process_resource_change_bound.delay(action="created",
sender="Group",
instance_id=event.group_id)
from sentry.plugins.base import plugins
for plugin in plugins.for_project(event.project):
plugin_post_process_group(plugin_slug=plugin.slug,
event=event,
is_new=is_new,
is_regresion=is_regression)
from sentry import similarity
with sentry_sdk.start_span(
op="tasks.post_process_group.similarity"):
safe_execute(similarity.record,
event.project, [event],
_with_transaction=False)
# Patch attachments that were ingested on the standalone path.
with sentry_sdk.start_span(
op="tasks.post_process_group.update_existing_attachments"):
try:
update_existing_attachments(event)
except Exception:
logger.exception("Failed to update existing attachments")
if not is_reprocessed:
event_processed.send_robust(
sender=post_process_group,
project=event.project,
event=event,
primary_hash=kwargs.get("primary_hash"),
)
with metrics.timer("tasks.post_process.delete_event_cache"):
event_processing_store.delete_by_key(cache_key)
def _do_process_event(
cache_key,
start_time,
event_id,
process_task,
data=None,
data_has_changed=None,
from_symbolicate=False,
):
from sentry.plugins.base import plugins
if data is None:
data = event_processing_store.get(cache_key)
if data is None:
metrics.incr(
"events.failed", tags={"reason": "cache", "stage": "process"}, skip_internal=False
)
error_logger.error("process.failed.empty", extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_project(project_id)
event_id = data["event_id"]
with sentry_sdk.start_span(op="tasks.store.process_event.get_project_from_cache"):
project = Project.objects.get_from_cache(id=project_id)
with metrics.timer("tasks.store.process_event.organization.get_from_cache"):
project._organization_cache = Organization.objects.get_from_cache(
id=project.organization_id
)
has_changed = bool(data_has_changed)
with sentry_sdk.start_span(op="tasks.store.process_event.get_reprocessing_revision"):
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
# Stacktrace based event processors.
with sentry_sdk.start_span(op="task.store.process_event.stacktraces"):
with metrics.timer(
"tasks.store.process_event.stacktraces", tags={"from_symbolicate": from_symbolicate}
):
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
# Second round of datascrubbing after stacktrace and language-specific
# processing. First round happened as part of ingest.
#
# *Right now* the only sensitive data that is added in stacktrace
# processing are usernames in filepaths, so we run directly after
# stacktrace processors.
#
# We do not yet want to deal with context data produced by plugins like
# sessionstack or fullstory (which are in `get_event_preprocessors`), as
# this data is very unlikely to be sensitive data. This is why scrubbing
# happens somewhere in the middle of the pipeline.
#
# On the other hand, Javascript event error translation is happening after
# this block because it uses `get_event_preprocessors` instead of
# `get_event_enhancers`.
#
# We are fairly confident, however, that this should run *before*
# re-normalization as it is hard to find sensitive data in partially
# trimmed strings.
if has_changed and options.get("processing.can-use-scrubbers"):
with sentry_sdk.start_span(op="task.store.datascrubbers.scrub"):
with metrics.timer(
"tasks.store.datascrubbers.scrub", tags={"from_symbolicate": from_symbolicate}
):
new_data = safe_execute(scrub_data, project=project, event=data.data)
# XXX(markus): When datascrubbing is finally "totally stable", we might want
# to drop the event if it crashes to avoid saving PII
if new_data is not None and features.has(
"organizations:datascrubbers-v2", project.organization, actor=None
):
data.data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
with sentry_sdk.start_span(op="task.store.process_event.preprocessors") as span:
span.set_data("plugin", plugin.slug)
span.set_data("from_symbolicate", from_symbolicate)
with metrics.timer(
"tasks.store.process_event.preprocessors",
tags={"plugin": plugin.slug, "from_symbolicate": from_symbolicate},
):
processors = safe_execute(
plugin.get_event_preprocessors, data=data, _with_transaction=False
)
for processor in processors or ():
try:
result = processor(data)
except Exception:
error_logger.exception("tasks.store.preprocessors.error")
data.setdefault("_metrics", {})["flag.processing.error"] = True
has_changed = True
else:
if result:
data = result
has_changed = True
assert data["project"] == project_id, "Project cannot be mutated by plugins"
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
# Run some of normalization again such that we don't:
# - persist e.g. incredibly large stacktraces from minidumps
# - store event timestamps that are older than our retention window
# (also happening with minidumps)
normalizer = StoreNormalizer(
remove_other=False, is_renormalize=True, **DEFAULT_STORE_NORMALIZER_ARGS
)
data = normalizer.normalize_event(dict(data))
issues = data.get("processing_issues")
try:
if issues and create_failed_event(
cache_key,
data,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev,
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke ourselves again. This happens when the reprocessing
# revision changed while we were processing.
_do_preprocess_event(cache_key, data, start_time, event_id, process_task, project)
return
cache_key = event_processing_store.store(data)
submit_save_event(project, cache_key, event_id, start_time, data)
def create_failed_event(
cache_key, data, project_id, issues, event_id, start_time=None, reprocessing_rev=None
):
"""If processing failed we put the original data from the cache into a
raw event. Returns `True` if a failed event was inserted
"""
set_current_project(project_id)
# We can only create failed events for events that can potentially
# create failed events.
if not reprocessing.event_supports_reprocessing(data):
return False
reprocessing_active = ProjectOption.objects.get_value(
project_id, "sentry:reprocessing_active", REPROCESSING_DEFAULT
)
# In case there is reprocessing active but the current reprocessing
# revision is already different than when we started, we want to
# immediately retry the event. This resolves the problem when
# otherwise a concurrent change of debug symbols might leave a
# reprocessing issue stuck in the project forever.
if (
reprocessing_active
and reprocessing.get_reprocessing_revision(project_id, cached=False) != reprocessing_rev
):
raise RetryProcessing()
# The first time we encounter a failed event and the hint was cleared
# we send a notification.
sent_notification = ProjectOption.objects.get_value(
project_id, "sentry:sent_failed_event_hint", False
)
if not sent_notification:
project = Project.objects.get_from_cache(id=project_id)
Activity.objects.create(
type=Activity.NEW_PROCESSING_ISSUES,
project=project,
datetime=to_datetime(start_time),
data={"reprocessing_active": reprocessing_active, "issues": issues},
).send_notification()
ProjectOption.objects.set_value(project, "sentry:sent_failed_event_hint", True)
# If reprocessing is not active we bail now without creating the
# processing issues
if not reprocessing_active:
return False
# We need to get the original data here instead of passing the data in
# from the last processing step because we do not want any
# modifications to take place.
delete_raw_event(project_id, event_id)
data = event_processing_store.get(cache_key)
if data is None:
metrics.incr("events.failed", tags={"reason": "cache", "stage": "raw"}, skip_internal=False)
error_logger.error("process.failed_raw.empty", extra={"cache_key": cache_key})
return True
data = CanonicalKeyDict(data)
from sentry.models import RawEvent, ProcessingIssue
raw_event = RawEvent.objects.create(
project_id=project_id,
event_id=event_id,
datetime=datetime.utcfromtimestamp(data["timestamp"]).replace(tzinfo=timezone.utc),
data=data,
)
for issue in issues:
ProcessingIssue.objects.record_processing_issue(
raw_event=raw_event,
scope=issue["scope"],
object=issue["object"],
type=issue["type"],
data=issue["data"],
)
event_processing_store.delete_by_key(cache_key)
return True
def _do_symbolicate_event(cache_key, start_time, event_id, symbolicate_task, data=None):
from sentry.lang.native.processing import get_symbolication_function
if data is None:
data = event_processing_store.get(cache_key)
if data is None:
metrics.incr(
"events.failed", tags={"reason": "cache", "stage": "symbolicate"}, skip_internal=False
)
error_logger.error("symbolicate.failed.empty", extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_project(project_id)
event_id = data["event_id"]
symbolication_function = get_symbolication_function(data)
has_changed = False
from_reprocessing = symbolicate_task is symbolicate_event_from_reprocessing
try:
with sentry_sdk.start_span(op="tasks.store.symbolicate_event.symbolication") as span:
span.set_data("symbolicaton_function", symbolication_function.__name__)
with metrics.timer("tasks.store.symbolicate_event.symbolication"):
symbolicated_data = symbolication_function(data)
span.set_data("symbolicated_data", bool(symbolicated_data))
if symbolicated_data:
data = symbolicated_data
has_changed = True
except RetrySymbolication as e:
if start_time and (time() - start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning(
"symbolicate.slow", extra={"project_id": project_id, "event_id": event_id}
)
if start_time and (time() - start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
error_logger.exception(
"symbolicate.failed.infinite_retry",
extra={"project_id": project_id, "event_id": event_id},
)
data.setdefault("_metrics", {})["flag.processing.error"] = True
data.setdefault("_metrics", {})["flag.processing.fatal"] = True
has_changed = True
else:
# Requeue the task in the "sleep" queue
retry_symbolicate_event.apply_async(
args=(),
kwargs={
"symbolicate_task_name": symbolicate_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
except Exception:
error_logger.exception("tasks.store.symbolicate_event.symbolication")
data.setdefault("_metrics", {})["flag.processing.error"] = True
data.setdefault("_metrics", {})["flag.processing.fatal"] = True
has_changed = True
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
cache_key = event_processing_store.store(data)
process_task = process_event_from_reprocessing if from_reprocessing else process_event
_do_process_event(
cache_key=cache_key,
start_time=start_time,
event_id=event_id,
process_task=process_task,
data=data,
data_has_changed=has_changed,
from_symbolicate=True,
)
def _do_save_event(
cache_key=None, data=None, start_time=None, event_id=None, project_id=None, **kwargs
):
"""
Saves an event to the database.
"""
set_current_event_project(project_id)
from sentry.event_manager import EventManager, HashDiscarded
event_type = "none"
if cache_key and data is None:
with metrics.timer("tasks.store.do_save_event.get_cache") as metric_tags:
data = event_processing_store.get(cache_key)
if data is not None:
metric_tags["event_type"] = event_type = data.get("type") or "none"
with metrics.global_tags(event_type=event_type):
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data["event_id"]
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop("project")
set_current_event_project(project_id)
# We only need to delete raw events for events that support
# reprocessing. If the data cannot be found we want to assume
# that we need to delete the raw event.
if not data or reprocessing.event_supports_reprocessing(data):
with metrics.timer("tasks.store.do_save_event.delete_raw_event"):
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr(
"events.failed", tags={"reason": "cache", "stage": "post"}, skip_internal=False
)
return
try:
if killswitch_matches_context(
"store.load-shed-save-event-projects",
{
"project_id": project_id,
"event_type": event_type,
"platform": data.get("platform") or "none",
},
):
raise HashDiscarded("Load shedding save_event")
with metrics.timer("tasks.store.do_save_event.event_manager.save"):
manager = EventManager(data)
# event.project.organization is populated after this statement.
manager.save(
project_id, assume_normalized=True, start_time=start_time, cache_key=cache_key
)
# Put the updated event back into the cache so that post_process
# has the most recent data.
data = manager.get_data()
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
with metrics.timer("tasks.store.do_save_event.write_processing_cache"):
event_processing_store.store(data)
except HashDiscarded:
# Delete the event payload from cache since it won't show up in post-processing.
if cache_key:
with metrics.timer("tasks.store.do_save_event.delete_cache"):
event_processing_store.delete_by_key(cache_key)
finally:
reprocessing2.mark_event_reprocessed(data)
if cache_key:
with metrics.timer("tasks.store.do_save_event.delete_attachment_cache"):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing(
"events.time-to-process",
time() - start_time,
instance=data["platform"],
tags={
"is_reprocessing2": "true"
if reprocessing2.is_reprocessed_event(data)
else "false",
},
)
time_synthetic_monitoring_event(data, project_id, start_time)
def _do_symbolicate_event(
cache_key, start_time, event_id, symbolicate_task, data=None, queue_switches=0
):
from sentry.lang.native.processing import get_symbolication_function
if data is None:
data = event_processing_store.get(cache_key)
if data is None:
metrics.incr(
"events.failed", tags={"reason": "cache", "stage": "symbolicate"}, skip_internal=False
)
error_logger.error("symbolicate.failed.empty", extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_event_project(project_id)
event_id = data["event_id"]
from_reprocessing = (
symbolicate_task is symbolicate_event_from_reprocessing
or symbolicate_task is symbolicate_event_from_reprocessing_low_priority
)
# check whether the event is in the wrong queue and if so, move it to the other one.
# we do this at most SYMBOLICATOR_MAX_QUEUE_SWITCHES times.
if queue_switches >= SYMBOLICATOR_MAX_QUEUE_SWITCHES:
metrics.gauge("tasks.store.symbolicate_event.low_priority.max_queue_switches", 1)
else:
is_low_priority = symbolicate_task in [
symbolicate_event_low_priority,
symbolicate_event_from_reprocessing_low_priority,
]
should_be_low_priority = should_demote_symbolication(project_id)
if is_low_priority != should_be_low_priority:
metrics.gauge("tasks.store.symbolicate_event.low_priority.wrong_queue", 1)
submit_symbolicate(
should_be_low_priority,
from_reprocessing,
cache_key,
event_id,
start_time,
data,
queue_switches + 1,
)
return
def _continue_to_process_event():
process_task = process_event_from_reprocessing if from_reprocessing else process_event
_do_process_event(
cache_key=cache_key,
start_time=start_time,
event_id=event_id,
process_task=process_task,
data=data,
data_has_changed=has_changed,
from_symbolicate=True,
)
symbolication_function = get_symbolication_function(data)
symbolication_function_name = getattr(symbolication_function, "__name__", "none")
if killswitch_matches_context(
"store.load-shed-symbolicate-event-projects",
{
"project_id": project_id,
"event_id": event_id,
"platform": data.get("platform") or "null",
"symbolication_function": symbolication_function_name,
},
):
return _continue_to_process_event()
has_changed = False
symbolication_start_time = time()
submission_ratio = options.get("symbolicate-event.low-priority.metrics.submission-rate")
submit_realtime_metrics = not from_reprocessing and random.random() < submission_ratio
if submit_realtime_metrics:
with sentry_sdk.start_span(op="tasks.store.symbolicate_event.low_priority.metrics.counter"):
timestamp = int(symbolication_start_time)
try:
realtime_metrics.increment_project_event_counter(project_id, timestamp)
except Exception as e:
sentry_sdk.capture_exception(e)
with sentry_sdk.start_span(op="tasks.store.symbolicate_event.symbolication") as span:
span.set_data("symbolication_function", symbolication_function_name)
with metrics.timer(
"tasks.store.symbolicate_event.symbolication",
tags={"symbolication_function": symbolication_function_name},
):
while True:
try:
with sentry_sdk.start_span(
op="tasks.store.symbolicate_event.%s" % symbolication_function_name
) as span:
symbolicated_data = symbolication_function(data)
span.set_data("symbolicated_data", bool(symbolicated_data))
if symbolicated_data:
data = symbolicated_data
has_changed = True
break
except RetrySymbolication as e:
if (
time() - symbolication_start_time
) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning(
"symbolicate.slow",
extra={"project_id": project_id, "event_id": event_id},
)
if (
time() - symbolication_start_time
) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
metrics.incr(
"tasks.store.symbolicate_event.fatal",
tags={
"reason": "timeout",
"symbolication_function": symbolication_function_name,
},
)
error_logger.exception(
"symbolicate.failed.infinite_retry",
extra={"project_id": project_id, "event_id": event_id},
)
data.setdefault("_metrics", {})["flag.processing.error"] = True
data.setdefault("_metrics", {})["flag.processing.fatal"] = True
has_changed = True
break
else:
# sleep for `retry_after` but max 5 seconds and try again
metrics.incr(
"tasks.store.symbolicate_event.retry",
tags={"symbolication_function": symbolication_function_name},
)
sleep(min(e.retry_after, SYMBOLICATOR_MAX_RETRY_AFTER))
continue
except Exception:
metrics.incr(
"tasks.store.symbolicate_event.fatal",
tags={
"reason": "error",
"symbolication_function": symbolication_function_name,
},
)
error_logger.exception("tasks.store.symbolicate_event.symbolication")
data.setdefault("_metrics", {})["flag.processing.error"] = True
data.setdefault("_metrics", {})["flag.processing.fatal"] = True
has_changed = True
break
if submit_realtime_metrics:
with sentry_sdk.start_span(
op="tasks.store.symbolicate_event.low_priority.metrics.histogram"
):
symbolication_duration = int(time() - symbolication_start_time)
try:
realtime_metrics.increment_project_duration_counter(
project_id, timestamp, symbolication_duration
)
except Exception as e:
sentry_sdk.capture_exception(e)
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
cache_key = event_processing_store.store(data)
return _continue_to_process_event()
def _do_preprocess_event(cache_key, data, start_time, event_id, process_task,
project):
from sentry.lang.native.processing import should_process_with_symbolicator
if cache_key and data is None:
data = event_processing_store.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "pre"
},
skip_internal=False)
error_logger.error("preprocess.failed.empty",
extra={"cache_key": cache_key})
return
original_data = data
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_event_project(project_id)
if project is None:
project = Project.objects.get_from_cache(id=project_id)
else:
assert project.id == project_id, (project.id, project_id)
from_reprocessing = process_task is process_event_from_reprocessing
with metrics.timer(
"tasks.store.preprocess_event.organization.get_from_cache"):
project.set_cached_field_value(
"organization",
Organization.objects.get_from_cache(id=project.organization_id))
if should_process_with_symbolicator(data):
reprocessing2.backup_unprocessed_event(project=project,
data=original_data)
is_low_priority = should_demote_symbolication(project_id)
submit_symbolicate(
is_low_priority,
from_reprocessing,
cache_key,
event_id,
start_time,
original_data,
)
return
if should_process(data):
submit_process(
project,
from_reprocessing,
cache_key,
event_id,
start_time,
data_has_changed=False,
)
return
submit_save_event(project_id, from_reprocessing, cache_key, event_id,
start_time, original_data)
def _do_symbolicate_event(cache_key,
start_time,
event_id,
symbolicate_task,
data=None):
from sentry.lang.native.processing import get_symbolication_function
if data is None:
data = event_processing_store.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "symbolicate"
},
skip_internal=False)
error_logger.error("symbolicate.failed.empty",
extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_event_project(project_id)
event_id = data["event_id"]
if killswitch_matches_context(
"store.load-shed-symbolicate-event-projects",
{
"project_id": project_id,
"event_id": event_id,
"platform": data.get("platform") or "null",
},
):
return
symbolication_function = get_symbolication_function(data)
has_changed = False
from_reprocessing = symbolicate_task is symbolicate_event_from_reprocessing
symbolication_start_time = time()
with sentry_sdk.start_span(
op="tasks.store.symbolicate_event.symbolication") as span:
span.set_data("symbolicaton_function", symbolication_function.__name__)
with metrics.timer(
"tasks.store.symbolicate_event.symbolication",
tags={
"symbolication_function": symbolication_function.__name__
},
):
while True:
try:
with sentry_sdk.start_span(
op="tasks.store.symbolicate_event.%s" %
symbolication_function.__name__) as span:
symbolicated_data = symbolication_function(data)
span.set_data("symbolicated_data",
bool(symbolicated_data))
if symbolicated_data:
data = symbolicated_data
has_changed = True
break
except RetrySymbolication as e:
if (time() - symbolication_start_time
) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning(
"symbolicate.slow",
extra={
"project_id": project_id,
"event_id": event_id
},
)
if (time() - symbolication_start_time
) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
metrics.incr(
"tasks.store.symbolicate_event.fatal",
tags={
"reason":
"timeout",
"symbolication_function":
symbolication_function.__name__,
},
)
error_logger.exception(
"symbolicate.failed.infinite_retry",
extra={
"project_id": project_id,
"event_id": event_id
},
)
data.setdefault("_metrics",
{})["flag.processing.error"] = True
data.setdefault("_metrics",
{})["flag.processing.fatal"] = True
has_changed = True
break
else:
# sleep for `retry_after` but max 5 seconds and try again
metrics.incr(
"tasks.store.symbolicate_event.retry",
tags={
"symbolication_function":
symbolication_function.__name__
},
)
sleep(min(e.retry_after, SYMBOLICATOR_MAX_RETRY_AFTER))
continue
except Exception:
metrics.incr(
"tasks.store.symbolicate_event.fatal",
tags={
"reason":
"error",
"symbolication_function":
symbolication_function.__name__,
},
)
error_logger.exception(
"tasks.store.symbolicate_event.symbolication")
data.setdefault("_metrics",
{})["flag.processing.error"] = True
data.setdefault("_metrics",
{})["flag.processing.fatal"] = True
has_changed = True
break
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
cache_key = event_processing_store.store(data)
process_task = process_event_from_reprocessing if from_reprocessing else process_event
_do_process_event(
cache_key=cache_key,
start_time=start_time,
event_id=event_id,
process_task=process_task,
data=data,
data_has_changed=has_changed,
from_symbolicate=True,
)