def _normalize_impl(self):
if self._normalized:
raise RuntimeError('Already normalized')
self._normalized = True
from semaphore.processing import StoreNormalizer
rust_normalizer = StoreNormalizer(
geoip_lookup=rust_geoip,
project_id=self._project.id if self._project else None,
client_ip=self._client_ip,
client=self._auth.client if self._auth else None,
key_id=six.text_type(self._key.id) if self._key else None,
grouping_config=self._grouping_config,
protocol_version=six.text_type(self.version)
if self.version is not None else None,
stacktrace_frames_hard_limit=settings.
SENTRY_STACKTRACE_FRAMES_HARD_LIMIT,
max_stacktrace_frames=settings.SENTRY_MAX_STACKTRACE_FRAMES,
valid_platforms=list(VALID_PLATFORMS),
max_secs_in_future=MAX_SECS_IN_FUTURE,
max_secs_in_past=MAX_SECS_IN_PAST,
enable_trimming=True,
is_renormalize=self._is_renormalize)
self._data = CanonicalKeyDict(
rust_normalizer.normalize_event(dict(self._data)))
normalize_user_agent(self._data)
def _normalize_impl(self):
if self._normalized:
raise RuntimeError('Already normalized')
self._normalized = True
from semaphore.processing import StoreNormalizer
rust_normalizer = StoreNormalizer(
geoip_lookup=rust_geoip,
project_id=self._project.id if self._project else None,
client_ip=self._client_ip,
client=self._auth.client if self._auth else None,
key_id=six.text_type(self._key.id) if self._key else None,
grouping_config=self._grouping_config,
protocol_version=six.text_type(self.version) if self.version is not None else None,
stacktrace_frames_hard_limit=settings.SENTRY_STACKTRACE_FRAMES_HARD_LIMIT,
max_stacktrace_frames=settings.SENTRY_MAX_STACKTRACE_FRAMES,
valid_platforms=list(VALID_PLATFORMS),
max_secs_in_future=MAX_SECS_IN_FUTURE,
max_secs_in_past=MAX_SECS_IN_PAST,
enable_trimming=True,
is_renormalize=self._is_renormalize,
remove_other=self._remove_other,
)
self._data = CanonicalKeyDict(
rust_normalizer.normalize_event(dict(self._data))
)
normalize_user_agent(self._data)
def __init__(self, data, skip_renormalization=False, **kwargs):
is_renormalized = isinstance(data, EventDict) or (
isinstance(data, NodeData) and isinstance(data.data, EventDict)
)
if not skip_renormalization and not is_renormalized:
normalizer = StoreNormalizer(is_renormalize=True, enable_trimming=False)
data = normalizer.normalize_event(dict(data))
CanonicalKeyDict.__init__(self, data, **kwargs)
def __init__(self, data, **kwargs):
rust_renormalized = _should_skip_to_python(data.get('event_id'))
if rust_renormalized:
normalizer = StoreNormalizer(is_renormalize=True)
data = normalizer.normalize_event(dict(data))
metrics.incr('rust.renormalized', tags={'value': rust_renormalized})
with configure_scope() as scope:
scope.set_tag("rust.renormalized", rust_renormalized)
CanonicalKeyDict.__init__(self, data, **kwargs)
def _normalize_impl(self):
if self._normalized:
raise RuntimeError("Already normalized")
self._normalized = True
from semaphore.processing import StoreNormalizer
rust_normalizer = StoreNormalizer(
project_id=self._project.id if self._project else None,
client_ip=self._client_ip,
client=self._auth.client if self._auth else None,
key_id=six.text_type(self._key.id) if self._key else None,
grouping_config=self._grouping_config,
protocol_version=six.text_type(self.version) if self.version is not None else None,
is_renormalize=self._is_renormalize,
remove_other=self._remove_other,
normalize_user_agent=True,
**DEFAULT_STORE_NORMALIZER_ARGS
)
self._data = CanonicalKeyDict(rust_normalizer.normalize_event(dict(self._data)))
def __init__(self, data, skip_renormalization=False, **kwargs):
is_renormalized = (isinstance(data, EventDict)
or (isinstance(data, NodeData)
and isinstance(data.data, EventDict)))
with configure_scope() as scope:
scope.set_tag("rust.is_renormalized", is_renormalized)
scope.set_tag("rust.skip_renormalization", skip_renormalization)
scope.set_tag("rust.renormalized", "null")
if not skip_renormalization and not is_renormalized:
rust_renormalized = _should_skip_to_python(data.get('event_id'))
if rust_renormalized:
normalizer = StoreNormalizer(is_renormalize=True)
data = normalizer.normalize_event(dict(data))
metrics.incr('rust.renormalized',
tags={'value': rust_renormalized})
with configure_scope() as scope:
scope.set_tag("rust.renormalized", rust_renormalized)
CanonicalKeyDict.__init__(self, data, **kwargs)
def __init__(self, data, skip_renormalization=False, **kwargs):
is_renormalized = (
isinstance(data, EventDict) or
(isinstance(data, NodeData) and isinstance(data.data, EventDict))
)
with configure_scope() as scope:
scope.set_tag("rust.is_renormalized", is_renormalized)
scope.set_tag("rust.skip_renormalization", skip_renormalization)
scope.set_tag("rust.renormalized", "null")
if not skip_renormalization and not is_renormalized:
rust_renormalized = _should_skip_to_python(data.get('event_id'))
if rust_renormalized:
normalizer = StoreNormalizer(is_renormalize=True)
data = normalizer.normalize_event(dict(data))
metrics.incr('rust.renormalized',
tags={'value': rust_renormalized})
with configure_scope() as scope:
scope.set_tag("rust.renormalized", rust_renormalized)
CanonicalKeyDict.__init__(self, data, **kwargs)
def _do_process_event(cache_key, start_time, event_id, process_task, data=None):
from sentry.plugins.base import plugins
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr(
"events.failed", tags={"reason": "cache", "stage": "process"}, skip_internal=False
)
error_logger.error("process.failed.empty", extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
with configure_scope() as scope:
scope.set_tag("project", project_id)
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
try:
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
enhancers = safe_execute(plugin.get_event_enhancers, data=data)
for enhancer in enhancers or ():
enhanced = safe_execute(enhancer, data, _passthrough_errors=(RetrySymbolication,))
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
except RetrySymbolication as e:
if start_time and (time() - start_time) > 3600:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
error_logger.exception("process.failed.infinite_retry")
else:
retry_process_event.apply_async(
args=(),
kwargs={
"process_task_name": process_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(
plugin.get_event_preprocessors, data=data, _with_transaction=False
)
for processor in processors or ():
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data["project"] == project_id, "Project cannot be mutated by preprocessor"
project = Project.objects.get_from_cache(id=project_id)
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
# Run some of normalization again such that we don't:
# - persist e.g. incredibly large stacktraces from minidumps
# - store event timestamps that are older than our retention window
# (also happening with minidumps)
normalizer = StoreNormalizer(
remove_other=False, is_renormalize=True, **DEFAULT_STORE_NORMALIZER_ARGS
)
data = normalizer.normalize_event(dict(data))
issues = data.get("processing_issues")
try:
if issues and create_failed_event(
cache_key,
data,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev,
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
from_reprocessing = process_task is process_event_from_reprocessing
submit_process(project, from_reprocessing, cache_key, event_id, start_time, data)
process_task.delay(cache_key, start_time=start_time, event_id=event_id)
return
default_cache.set(cache_key, data, 3600)
submit_save_event(project, cache_key, event_id, start_time, data)
