def _normalize_impl(self): if self._normalized: raise RuntimeError('Already normalized') self._normalized = True from semaphore.processing import StoreNormalizer rust_normalizer = StoreNormalizer( geoip_lookup=rust_geoip, project_id=self._project.id if self._project else None, client_ip=self._client_ip, client=self._auth.client if self._auth else None, key_id=six.text_type(self._key.id) if self._key else None, grouping_config=self._grouping_config, protocol_version=six.text_type(self.version) if self.version is not None else None, stacktrace_frames_hard_limit=settings. SENTRY_STACKTRACE_FRAMES_HARD_LIMIT, max_stacktrace_frames=settings.SENTRY_MAX_STACKTRACE_FRAMES, valid_platforms=list(VALID_PLATFORMS), max_secs_in_future=MAX_SECS_IN_FUTURE, max_secs_in_past=MAX_SECS_IN_PAST, enable_trimming=True, is_renormalize=self._is_renormalize) self._data = CanonicalKeyDict( rust_normalizer.normalize_event(dict(self._data))) normalize_user_agent(self._data)
def _normalize_impl(self): if self._normalized: raise RuntimeError('Already normalized') self._normalized = True from semaphore.processing import StoreNormalizer rust_normalizer = StoreNormalizer( geoip_lookup=rust_geoip, project_id=self._project.id if self._project else None, client_ip=self._client_ip, client=self._auth.client if self._auth else None, key_id=six.text_type(self._key.id) if self._key else None, grouping_config=self._grouping_config, protocol_version=six.text_type(self.version) if self.version is not None else None, stacktrace_frames_hard_limit=settings.SENTRY_STACKTRACE_FRAMES_HARD_LIMIT, max_stacktrace_frames=settings.SENTRY_MAX_STACKTRACE_FRAMES, valid_platforms=list(VALID_PLATFORMS), max_secs_in_future=MAX_SECS_IN_FUTURE, max_secs_in_past=MAX_SECS_IN_PAST, enable_trimming=True, is_renormalize=self._is_renormalize, remove_other=self._remove_other, ) self._data = CanonicalKeyDict( rust_normalizer.normalize_event(dict(self._data)) ) normalize_user_agent(self._data)
def __init__(self, data, skip_renormalization=False, **kwargs): is_renormalized = isinstance(data, EventDict) or ( isinstance(data, NodeData) and isinstance(data.data, EventDict) ) if not skip_renormalization and not is_renormalized: normalizer = StoreNormalizer(is_renormalize=True, enable_trimming=False) data = normalizer.normalize_event(dict(data)) CanonicalKeyDict.__init__(self, data, **kwargs)
def __init__(self, data, **kwargs): rust_renormalized = _should_skip_to_python(data.get('event_id')) if rust_renormalized: normalizer = StoreNormalizer(is_renormalize=True) data = normalizer.normalize_event(dict(data)) metrics.incr('rust.renormalized', tags={'value': rust_renormalized}) with configure_scope() as scope: scope.set_tag("rust.renormalized", rust_renormalized) CanonicalKeyDict.__init__(self, data, **kwargs)
def _normalize_impl(self): if self._normalized: raise RuntimeError("Already normalized") self._normalized = True from semaphore.processing import StoreNormalizer rust_normalizer = StoreNormalizer( project_id=self._project.id if self._project else None, client_ip=self._client_ip, client=self._auth.client if self._auth else None, key_id=six.text_type(self._key.id) if self._key else None, grouping_config=self._grouping_config, protocol_version=six.text_type(self.version) if self.version is not None else None, is_renormalize=self._is_renormalize, remove_other=self._remove_other, normalize_user_agent=True, **DEFAULT_STORE_NORMALIZER_ARGS ) self._data = CanonicalKeyDict(rust_normalizer.normalize_event(dict(self._data)))
def __init__(self, data, skip_renormalization=False, **kwargs): is_renormalized = (isinstance(data, EventDict) or (isinstance(data, NodeData) and isinstance(data.data, EventDict))) with configure_scope() as scope: scope.set_tag("rust.is_renormalized", is_renormalized) scope.set_tag("rust.skip_renormalization", skip_renormalization) scope.set_tag("rust.renormalized", "null") if not skip_renormalization and not is_renormalized: rust_renormalized = _should_skip_to_python(data.get('event_id')) if rust_renormalized: normalizer = StoreNormalizer(is_renormalize=True) data = normalizer.normalize_event(dict(data)) metrics.incr('rust.renormalized', tags={'value': rust_renormalized}) with configure_scope() as scope: scope.set_tag("rust.renormalized", rust_renormalized) CanonicalKeyDict.__init__(self, data, **kwargs)
def __init__(self, data, skip_renormalization=False, **kwargs): is_renormalized = ( isinstance(data, EventDict) or (isinstance(data, NodeData) and isinstance(data.data, EventDict)) ) with configure_scope() as scope: scope.set_tag("rust.is_renormalized", is_renormalized) scope.set_tag("rust.skip_renormalization", skip_renormalization) scope.set_tag("rust.renormalized", "null") if not skip_renormalization and not is_renormalized: rust_renormalized = _should_skip_to_python(data.get('event_id')) if rust_renormalized: normalizer = StoreNormalizer(is_renormalize=True) data = normalizer.normalize_event(dict(data)) metrics.incr('rust.renormalized', tags={'value': rust_renormalized}) with configure_scope() as scope: scope.set_tag("rust.renormalized", rust_renormalized) CanonicalKeyDict.__init__(self, data, **kwargs)
def _do_process_event(cache_key, start_time, event_id, process_task, data=None): from sentry.plugins.base import plugins if data is None: data = default_cache.get(cache_key) if data is None: metrics.incr( "events.failed", tags={"reason": "cache", "stage": "process"}, skip_internal=False ) error_logger.error("process.failed.empty", extra={"cache_key": cache_key}) return data = CanonicalKeyDict(data) project_id = data["project"] with configure_scope() as scope: scope.set_tag("project", project_id) has_changed = False # Fetch the reprocessing revision reprocessing_rev = reprocessing.get_reprocessing_revision(project_id) try: # Event enhancers. These run before anything else. for plugin in plugins.all(version=2): enhancers = safe_execute(plugin.get_event_enhancers, data=data) for enhancer in enhancers or (): enhanced = safe_execute(enhancer, data, _passthrough_errors=(RetrySymbolication,)) if enhanced: data = enhanced has_changed = True # Stacktrace based event processors. new_data = process_stacktraces(data) if new_data is not None: has_changed = True data = new_data except RetrySymbolication as e: if start_time and (time() - start_time) > 3600: # Do not drop event but actually continue with rest of pipeline # (persisting unsymbolicated event) error_logger.exception("process.failed.infinite_retry") else: retry_process_event.apply_async( args=(), kwargs={ "process_task_name": process_task.__name__, "task_kwargs": { "cache_key": cache_key, "event_id": event_id, "start_time": start_time, }, }, countdown=e.retry_after, ) return # TODO(dcramer): ideally we would know if data changed by default # Default event processors. for plugin in plugins.all(version=2): processors = safe_execute( plugin.get_event_preprocessors, data=data, _with_transaction=False ) for processor in processors or (): result = safe_execute(processor, data) if result: data = result has_changed = True assert data["project"] == project_id, "Project cannot be mutated by preprocessor" project = Project.objects.get_from_cache(id=project_id) # We cannot persist canonical types in the cache, so we need to # downgrade this. if isinstance(data, CANONICAL_TYPES): data = dict(data.items()) if has_changed: # Run some of normalization again such that we don't: # - persist e.g. incredibly large stacktraces from minidumps # - store event timestamps that are older than our retention window # (also happening with minidumps) normalizer = StoreNormalizer( remove_other=False, is_renormalize=True, **DEFAULT_STORE_NORMALIZER_ARGS ) data = normalizer.normalize_event(dict(data)) issues = data.get("processing_issues") try: if issues and create_failed_event( cache_key, data, project_id, list(issues.values()), event_id=event_id, start_time=start_time, reprocessing_rev=reprocessing_rev, ): return except RetryProcessing: # If `create_failed_event` indicates that we need to retry we # invoke outselves again. This happens when the reprocessing # revision changed while we were processing. from_reprocessing = process_task is process_event_from_reprocessing submit_process(project, from_reprocessing, cache_key, event_id, start_time, data) process_task.delay(cache_key, start_time=start_time, event_id=event_id) return default_cache.set(cache_key, data, 3600) submit_save_event(project, cache_key, event_id, start_time, data)