def load_data( platform, default=None, sample_name=None, timestamp=None, start_timestamp=None, trace=None, span_id=None, spans=None, ): # NOTE: Before editing this data, make sure you understand the context # in which its being used. It is NOT only used for local development and # has production consequences. # * bin/load-mocks to generate fake data for local testing # * When a new project is created, a fake event is generated as a "starter" # event so it's not an empty project. # * When a user clicks Test Configuration from notification plugin settings page, # a fake event is generated to go through the pipeline. data = None language = None platform_data = INTEGRATION_ID_TO_PLATFORM_DATA.get(platform) if platform_data is not None and platform_data["type"] != "language": language = platform_data["language"] samples_root = os.path.join(DATA_ROOT, "samples") all_samples = {f for f in os.listdir(samples_root) if f.endswith(".json")} for platform in (platform, language, default): if not platform: continue # Verify by checking if the file is within our folder explicitly # avoids being able to have a name that invokes traversing directories. json_path = f"{platform}.json" if json_path not in all_samples: continue if not sample_name: try: sample_name = INTEGRATION_ID_TO_PLATFORM_DATA[platform]["name"] except KeyError: pass # XXX: At this point, it's assumed that `json_path` was safely found # within `samples_root` due to the check above and cannot traverse # into paths. with open(os.path.join(samples_root, json_path)) as fp: data = json.load(fp) break if data is None: return data = CanonicalKeyDict(data) if platform in ("csp", "hkpk", "expectct", "expectstaple"): return data # Generate a timestamp in the present. if timestamp is None: timestamp = timezone.now() else: timestamp = timestamp.replace(tzinfo=pytz.utc) data.setdefault("timestamp", to_timestamp(timestamp)) if data.get("type") == "transaction": if start_timestamp is None: start_timestamp = timestamp - timedelta(seconds=3) else: start_timestamp = start_timestamp.replace(tzinfo=pytz.utc) data["start_timestamp"] = to_timestamp(start_timestamp) if trace is None: trace = uuid4().hex if span_id is None: span_id = uuid4().hex[:16] for tag in data["tags"]: if tag[0] == "trace": tag[1] = trace elif tag[0] == "trace.span": tag[1] = span_id data["contexts"]["trace"]["trace_id"] = trace data["contexts"]["trace"]["span_id"] = span_id if spans: data["spans"] = spans for span in data.get("spans", []): # Use data to generate span timestamps consistently and based # on event timestamp duration = span.get("data", {}).get("duration", 10.0) offset = span.get("data", {}).get("offset", 0) # Span doesn't have a parent, make it the transaction if span.get("parent_span_id") is None: span["parent_span_id"] = span_id if span.get("span_id") is None: span["span_id"] = uuid4().hex[:16] span_start = data["start_timestamp"] + offset span["trace_id"] = trace span.setdefault("start_timestamp", span_start) span.setdefault("timestamp", span_start + duration) measurements = data.get("measurements") if measurements: measurement_markers = {} for key, entry in measurements.items(): if key in ["fp", "fcp", "lcp", "fid"]: measurement_markers[f"mark.{key}"] = { "value": round(data["start_timestamp"] + entry["value"] / 1000, 3) } measurements.update(measurement_markers) data["platform"] = platform # XXX: Message is a legacy alias for logentry. Do not overwrite if set. if "message" not in data: data[ "message"] = f"This is an example {sample_name or platform} exception" data.setdefault( "user", generate_user(ip_address="127.0.0.1", username="******", id=1, email="*****@*****.**"), ) data.setdefault( "extra", { "session": { "foo": "bar" }, "results": [1, 2, 3, 4, 5], "emptyList": [], "emptyMap": {}, "length": 10837790, "unauthorized": False, "url": "http://example.org/foo/bar/", }, ) data.setdefault("modules", {"my.package": "1.0.0"}) data.setdefault( "request", { "cookies": "foo=bar;biz=baz", "url": "http://example.com/foo", "headers": { "Referer": "http://example.com", "Content-Type": "application/json", "User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36", }, "env": { "ENV": "prod" }, "query_string": "foo=bar", "data": '{"hello": "world"}', "method": "GET", }, ) return data
def _do_symbolicate_event(cache_key, start_time, event_id, symbolicate_task, data=None): from sentry.lang.native.processing import get_symbolication_function if data is None: data = event_processing_store.get(cache_key) if data is None: metrics.incr("events.failed", tags={ "reason": "cache", "stage": "symbolicate" }, skip_internal=False) error_logger.error("symbolicate.failed.empty", extra={"cache_key": cache_key}) return data = CanonicalKeyDict(data) project_id = data["project"] set_current_event_project(project_id) event_id = data["event_id"] if killswitch_matches_context( "store.load-shed-symbolicate-event-projects", { "project_id": project_id, "event_id": event_id, "platform": data.get("platform") or "null", }, ): return symbolication_function = get_symbolication_function(data) has_changed = False from_reprocessing = symbolicate_task is symbolicate_event_from_reprocessing symbolication_start_time = time() with sentry_sdk.start_span( op="tasks.store.symbolicate_event.symbolication") as span: span.set_data("symbolicaton_function", symbolication_function.__name__) with metrics.timer( "tasks.store.symbolicate_event.symbolication", tags={ "symbolication_function": symbolication_function.__name__ }, ): while True: try: with sentry_sdk.start_span( op="tasks.store.symbolicate_event.%s" % symbolication_function.__name__) as span: symbolicated_data = symbolication_function(data) span.set_data("symbolicated_data", bool(symbolicated_data)) if symbolicated_data: data = symbolicated_data has_changed = True break except RetrySymbolication as e: if (time() - symbolication_start_time ) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT: error_logger.warning( "symbolicate.slow", extra={ "project_id": project_id, "event_id": event_id }, ) if (time() - symbolication_start_time ) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT: # Do not drop event but actually continue with rest of pipeline # (persisting unsymbolicated event) metrics.incr( "tasks.store.symbolicate_event.fatal", tags={ "reason": "timeout", "symbolication_function": symbolication_function.__name__, }, ) error_logger.exception( "symbolicate.failed.infinite_retry", extra={ "project_id": project_id, "event_id": event_id }, ) data.setdefault("_metrics", {})["flag.processing.error"] = True data.setdefault("_metrics", {})["flag.processing.fatal"] = True has_changed = True break else: # sleep for `retry_after` but max 5 seconds and try again metrics.incr( "tasks.store.symbolicate_event.retry", tags={ "symbolication_function": symbolication_function.__name__ }, ) sleep(min(e.retry_after, SYMBOLICATOR_MAX_RETRY_AFTER)) continue except Exception: metrics.incr( "tasks.store.symbolicate_event.fatal", tags={ "reason": "error", "symbolication_function": symbolication_function.__name__, }, ) error_logger.exception( "tasks.store.symbolicate_event.symbolication") data.setdefault("_metrics", {})["flag.processing.error"] = True data.setdefault("_metrics", {})["flag.processing.fatal"] = True has_changed = True break # We cannot persist canonical types in the cache, so we need to # downgrade this. if isinstance(data, CANONICAL_TYPES): data = dict(data.items()) if has_changed: cache_key = event_processing_store.store(data) process_task = process_event_from_reprocessing if from_reprocessing else process_event _do_process_event( cache_key=cache_key, start_time=start_time, event_id=event_id, process_task=process_task, data=data, data_has_changed=has_changed, from_symbolicate=True, )
def _do_process_event( cache_key, start_time, event_id, process_task, data=None, data_has_changed=None, from_symbolicate=False, ): from sentry.plugins.base import plugins if data is None: data = event_processing_store.get(cache_key) if data is None: metrics.incr("events.failed", tags={ "reason": "cache", "stage": "process" }, skip_internal=False) error_logger.error("process.failed.empty", extra={"cache_key": cache_key}) return data = CanonicalKeyDict(data) project_id = data["project"] set_current_event_project(project_id) event_id = data["event_id"] if killswitch_matches_context( "store.load-shed-process-event-projects", { "project_id": project_id, "event_id": event_id, "platform": data.get("platform") or "null", }, ): return with sentry_sdk.start_span( op="tasks.store.process_event.get_project_from_cache"): project = Project.objects.get_from_cache(id=project_id) with metrics.timer( "tasks.store.process_event.organization.get_from_cache"): project._organization_cache = Organization.objects.get_from_cache( id=project.organization_id) has_changed = bool(data_has_changed) with sentry_sdk.start_span( op="tasks.store.process_event.get_reprocessing_revision"): # Fetch the reprocessing revision reprocessing_rev = reprocessing.get_reprocessing_revision(project_id) # Stacktrace based event processors. with sentry_sdk.start_span(op="task.store.process_event.stacktraces"): with metrics.timer("tasks.store.process_event.stacktraces", tags={"from_symbolicate": from_symbolicate}): new_data = process_stacktraces(data) if new_data is not None: has_changed = True data = new_data # Second round of datascrubbing after stacktrace and language-specific # processing. First round happened as part of ingest. # # *Right now* the only sensitive data that is added in stacktrace # processing are usernames in filepaths, so we run directly after # stacktrace processors. # # We do not yet want to deal with context data produced by plugins like # sessionstack or fullstory (which are in `get_event_preprocessors`), as # this data is very unlikely to be sensitive data. This is why scrubbing # happens somewhere in the middle of the pipeline. # # On the other hand, Javascript event error translation is happening after # this block because it uses `get_event_preprocessors` instead of # `get_event_enhancers`. # # We are fairly confident, however, that this should run *before* # re-normalization as it is hard to find sensitive data in partially # trimmed strings. if has_changed and options.get("processing.can-use-scrubbers"): with sentry_sdk.start_span(op="task.store.datascrubbers.scrub"): with metrics.timer("tasks.store.datascrubbers.scrub", tags={"from_symbolicate": from_symbolicate}): new_data = safe_execute(scrub_data, project=project, event=data.data) # XXX(markus): When datascrubbing is finally "totally stable", we might want # to drop the event if it crashes to avoid saving PII if new_data is not None: data.data = new_data # TODO(dcramer): ideally we would know if data changed by default # Default event processors. for plugin in plugins.all(version=2): with sentry_sdk.start_span( op="task.store.process_event.preprocessors") as span: span.set_data("plugin", plugin.slug) span.set_data("from_symbolicate", from_symbolicate) with metrics.timer( "tasks.store.process_event.preprocessors", tags={ "plugin": plugin.slug, "from_symbolicate": from_symbolicate }, ): processors = safe_execute(plugin.get_event_preprocessors, data=data, _with_transaction=False) for processor in processors or (): try: result = processor(data) except Exception: error_logger.exception( "tasks.store.preprocessors.error") data.setdefault("_metrics", {})["flag.processing.error"] = True has_changed = True else: if result: data = result has_changed = True assert data[ "project"] == project_id, "Project cannot be mutated by plugins" # We cannot persist canonical types in the cache, so we need to # downgrade this. if isinstance(data, CANONICAL_TYPES): data = dict(data.items()) if has_changed: # Run some of normalization again such that we don't: # - persist e.g. incredibly large stacktraces from minidumps # - store event timestamps that are older than our retention window # (also happening with minidumps) normalizer = StoreNormalizer(remove_other=False, is_renormalize=True, **DEFAULT_STORE_NORMALIZER_ARGS) data = normalizer.normalize_event(dict(data)) issues = data.get("processing_issues") try: if issues and create_failed_event( cache_key, data, project_id, list(issues.values()), event_id=event_id, start_time=start_time, reprocessing_rev=reprocessing_rev, ): return except RetryProcessing: # If `create_failed_event` indicates that we need to retry we # invoke ourselves again. This happens when the reprocessing # revision changed while we were processing. _do_preprocess_event(cache_key, data, start_time, event_id, process_task, project) return cache_key = event_processing_store.store(data) from_reprocessing = process_task is process_event_from_reprocessing submit_save_event(project, from_reprocessing, cache_key, event_id, start_time, data)
def load_data(platform, default=None, sample_name=None): # NOTE: Before editing this data, make sure you understand the context # in which its being used. It is NOT only used for local development and # has production consequences. # * bin/load-mocks to generate fake data for local testing # * When a new project is created, a fake event is generated as a "starter" # event so it's not an empty project. # * When a user clicks Test Configuration from notification plugin settings page, # a fake event is generated to go through the pipeline. data = None language = None platform_data = INTEGRATION_ID_TO_PLATFORM_DATA.get(platform) if platform_data is not None and platform_data['type'] != 'language': language = platform_data['language'] for platform in (platform, language, default): if not platform: continue json_path = os.path.join(DATA_ROOT, 'samples', '%s.json' % (platform.encode('utf-8'), )) if not os.path.exists(json_path): continue if not sample_name: try: sample_name = INTEGRATION_ID_TO_PLATFORM_DATA[platform]['name'] except KeyError: pass with open(json_path) as fp: data = json.loads(fp.read()) break if data is None: return data = CanonicalKeyDict(data) if platform in ('csp', 'hkpk', 'expectct', 'expectstaple'): return data data['platform'] = platform # XXX: Message is a legacy alias for logentry. Do not overwrite if set. if 'message' not in data: data['message'] = 'This is an example %s exception' % (sample_name or platform, ) data.setdefault( 'user', generate_user( ip_address='127.0.0.1', username='******', id=1, email='*****@*****.**', )) data.setdefault( 'extra', { 'session': { 'foo': 'bar', }, 'results': [1, 2, 3, 4, 5], 'emptyList': [], 'emptyMap': {}, 'length': 10837790, 'unauthorized': False, 'url': 'http://example.org/foo/bar/', }) data.setdefault('modules', { 'my.package': '1.0.0', }) data.setdefault( 'request', { "cookies": 'foo=bar;biz=baz', "url": "http://example.com/foo", "headers": { "Referer": "http://example.com", "Content-Type": "application/json", "User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" }, "env": { 'ENV': 'prod', }, "query_string": "foo=bar", "data": '{"hello": "world"}', "method": "GET" }) return data
def load_data(platform, default=None, sample_name=None): # NOTE: Before editing this data, make sure you understand the context # in which its being used. It is NOT only used for local development and # has production consequences. # * bin/load-mocks to generate fake data for local testing # * When a new project is created, a fake event is generated as a "starter" # event so it's not an empty project. # * When a user clicks Test Configuration from notification plugin settings page, # a fake event is generated to go through the pipeline. data = None language = None platform_data = INTEGRATION_ID_TO_PLATFORM_DATA.get(platform) if platform_data is not None and platform_data["type"] != "language": language = platform_data["language"] for platform in (platform, language, default): if not platform: continue json_path = os.path.join(DATA_ROOT, "samples", "%s.json" % (platform.encode("utf-8"),)) if not os.path.exists(json_path): continue if not sample_name: try: sample_name = INTEGRATION_ID_TO_PLATFORM_DATA[platform]["name"] except KeyError: pass with open(json_path) as fp: data = json.loads(fp.read()) break if data is None: return data = CanonicalKeyDict(data) if platform in ("csp", "hkpk", "expectct", "expectstaple"): return data # Transaction events need timestamp data set to something current. if platform == "transaction": now = timezone.now() now_time = to_timestamp(now) start_time = to_timestamp(now - timedelta(seconds=-2)) data.setdefault("timestamp", now_time) data.setdefault("start_timestamp", start_time) for span in data["spans"]: span.setdefault("timestamp", now_time) span.setdefault("start_timestamp", start_time) data["platform"] = platform # XXX: Message is a legacy alias for logentry. Do not overwrite if set. if "message" not in data: data["message"] = "This is an example %s exception" % (sample_name or platform,) data.setdefault( "user", generate_user(ip_address="127.0.0.1", username="******", id=1, email="*****@*****.**"), ) data.setdefault( "extra", { "session": {"foo": "bar"}, "results": [1, 2, 3, 4, 5], "emptyList": [], "emptyMap": {}, "length": 10837790, "unauthorized": False, "url": "http://example.org/foo/bar/", }, ) data.setdefault("modules", {"my.package": "1.0.0"}) data.setdefault( "request", { "cookies": "foo=bar;biz=baz", "url": "http://example.com/foo", "headers": { "Referer": "http://example.com", "Content-Type": "application/json", "User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36", }, "env": {"ENV": "prod"}, "query_string": "foo=bar", "data": '{"hello": "world"}', "method": "GET", }, ) return data
def _do_symbolicate_event(cache_key, start_time, event_id, symbolicate_task, data=None): from sentry.lang.native.processing import get_symbolication_function if data is None: data = event_processing_store.get(cache_key) if data is None: metrics.incr( "events.failed", tags={"reason": "cache", "stage": "symbolicate"}, skip_internal=False ) error_logger.error("symbolicate.failed.empty", extra={"cache_key": cache_key}) return data = CanonicalKeyDict(data) project_id = data["project"] set_current_project(project_id) event_id = data["event_id"] symbolication_function = get_symbolication_function(data) has_changed = False from_reprocessing = symbolicate_task is symbolicate_event_from_reprocessing try: with sentry_sdk.start_span(op="tasks.store.symbolicate_event.symbolication") as span: span.set_data("symbolicaton_function", symbolication_function.__name__) with metrics.timer("tasks.store.symbolicate_event.symbolication"): symbolicated_data = symbolication_function(data) span.set_data("symbolicated_data", bool(symbolicated_data)) if symbolicated_data: data = symbolicated_data has_changed = True except RetrySymbolication as e: if start_time and (time() - start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT: error_logger.warning( "symbolicate.slow", extra={"project_id": project_id, "event_id": event_id} ) if start_time and (time() - start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT: # Do not drop event but actually continue with rest of pipeline # (persisting unsymbolicated event) error_logger.exception( "symbolicate.failed.infinite_retry", extra={"project_id": project_id, "event_id": event_id}, ) data.setdefault("_metrics", {})["flag.processing.error"] = True data.setdefault("_metrics", {})["flag.processing.fatal"] = True has_changed = True else: # Requeue the task in the "sleep" queue retry_symbolicate_event.apply_async( args=(), kwargs={ "symbolicate_task_name": symbolicate_task.__name__, "task_kwargs": { "cache_key": cache_key, "event_id": event_id, "start_time": start_time, }, }, countdown=e.retry_after, ) return except Exception: error_logger.exception("tasks.store.symbolicate_event.symbolication") data.setdefault("_metrics", {})["flag.processing.error"] = True data.setdefault("_metrics", {})["flag.processing.fatal"] = True has_changed = True # We cannot persist canonical types in the cache, so we need to # downgrade this. if isinstance(data, CANONICAL_TYPES): data = dict(data.items()) if has_changed: cache_key = event_processing_store.store(data) process_task = process_event_from_reprocessing if from_reprocessing else process_event _do_process_event( cache_key=cache_key, start_time=start_time, event_id=event_id, process_task=process_task, data=data, data_has_changed=has_changed, from_symbolicate=True, )
def load_data(platform, default=None, sample_name=None): # NOTE: Before editing this data, make sure you understand the context # in which its being used. It is NOT only used for local development and # has production consequences. # * bin/load-mocks to generate fake data for local testing # * When a new project is created, a fake event is generated as a "starter" # event so it's not an empty project. # * When a user clicks Test Configuration from notification plugin settings page, # a fake event is generated to go through the pipeline. data = None language = None platform_data = INTEGRATION_ID_TO_PLATFORM_DATA.get(platform) if platform_data is not None and platform_data['type'] != 'language': language = platform_data['language'] for platform in (platform, language, default): if not platform: continue json_path = os.path.join(DATA_ROOT, 'samples', '%s.json' % (platform.encode('utf-8'), )) if not os.path.exists(json_path): continue if not sample_name: try: sample_name = INTEGRATION_ID_TO_PLATFORM_DATA[platform]['name'] except KeyError: pass with open(json_path) as fp: data = json.loads(fp.read()) break if data is None: return data = CanonicalKeyDict(data) if platform in ('csp', 'hkpk', 'expectct', 'expectstaple'): return data data['platform'] = platform # XXX: Message is a legacy alias for logentry. Do not overwrite if set. if 'message' not in data: data['message'] = 'This is an example %s exception' % (sample_name or platform, ) data.setdefault('user', generate_user( ip_address='127.0.0.1', username='******', id=1, email='*****@*****.**', )) data.setdefault('extra', { 'session': { 'foo': 'bar', }, 'results': [1, 2, 3, 4, 5], 'emptyList': [], 'emptyMap': {}, 'length': 10837790, 'unauthorized': False, 'url': 'http://example.org/foo/bar/', }) data.setdefault('modules', { 'my.package': '1.0.0', }) data.setdefault('request', { "cookies": 'foo=bar;biz=baz', "url": "http://example.com/foo", "headers": { "Referer": "http://example.com", "Content-Type": "application/json", "User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" }, "env": { 'ENV': 'prod', }, "query_string": "foo=bar", "data": '{"hello": "world"}', "method": "GET" }) return data
def _do_symbolicate_event( cache_key, start_time, event_id, symbolicate_task, data=None, queue_switches=0 ): from sentry.lang.native.processing import get_symbolication_function if data is None: data = event_processing_store.get(cache_key) if data is None: metrics.incr( "events.failed", tags={"reason": "cache", "stage": "symbolicate"}, skip_internal=False ) error_logger.error("symbolicate.failed.empty", extra={"cache_key": cache_key}) return data = CanonicalKeyDict(data) project_id = data["project"] set_current_event_project(project_id) event_id = data["event_id"] from_reprocessing = ( symbolicate_task is symbolicate_event_from_reprocessing or symbolicate_task is symbolicate_event_from_reprocessing_low_priority ) # check whether the event is in the wrong queue and if so, move it to the other one. # we do this at most SYMBOLICATOR_MAX_QUEUE_SWITCHES times. if queue_switches >= SYMBOLICATOR_MAX_QUEUE_SWITCHES: metrics.gauge("tasks.store.symbolicate_event.low_priority.max_queue_switches", 1) else: is_low_priority = symbolicate_task in [ symbolicate_event_low_priority, symbolicate_event_from_reprocessing_low_priority, ] should_be_low_priority = should_demote_symbolication(project_id) if is_low_priority != should_be_low_priority: metrics.gauge("tasks.store.symbolicate_event.low_priority.wrong_queue", 1) submit_symbolicate( should_be_low_priority, from_reprocessing, cache_key, event_id, start_time, data, queue_switches + 1, ) return def _continue_to_process_event(): process_task = process_event_from_reprocessing if from_reprocessing else process_event _do_process_event( cache_key=cache_key, start_time=start_time, event_id=event_id, process_task=process_task, data=data, data_has_changed=has_changed, from_symbolicate=True, ) symbolication_function = get_symbolication_function(data) symbolication_function_name = getattr(symbolication_function, "__name__", "none") if killswitch_matches_context( "store.load-shed-symbolicate-event-projects", { "project_id": project_id, "event_id": event_id, "platform": data.get("platform") or "null", "symbolication_function": symbolication_function_name, }, ): return _continue_to_process_event() has_changed = False symbolication_start_time = time() submission_ratio = options.get("symbolicate-event.low-priority.metrics.submission-rate") submit_realtime_metrics = not from_reprocessing and random.random() < submission_ratio if submit_realtime_metrics: with sentry_sdk.start_span(op="tasks.store.symbolicate_event.low_priority.metrics.counter"): timestamp = int(symbolication_start_time) try: realtime_metrics.increment_project_event_counter(project_id, timestamp) except Exception as e: sentry_sdk.capture_exception(e) with sentry_sdk.start_span(op="tasks.store.symbolicate_event.symbolication") as span: span.set_data("symbolication_function", symbolication_function_name) with metrics.timer( "tasks.store.symbolicate_event.symbolication", tags={"symbolication_function": symbolication_function_name}, ): while True: try: with sentry_sdk.start_span( op="tasks.store.symbolicate_event.%s" % symbolication_function_name ) as span: symbolicated_data = symbolication_function(data) span.set_data("symbolicated_data", bool(symbolicated_data)) if symbolicated_data: data = symbolicated_data has_changed = True break except RetrySymbolication as e: if ( time() - symbolication_start_time ) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT: error_logger.warning( "symbolicate.slow", extra={"project_id": project_id, "event_id": event_id}, ) if ( time() - symbolication_start_time ) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT: # Do not drop event but actually continue with rest of pipeline # (persisting unsymbolicated event) metrics.incr( "tasks.store.symbolicate_event.fatal", tags={ "reason": "timeout", "symbolication_function": symbolication_function_name, }, ) error_logger.exception( "symbolicate.failed.infinite_retry", extra={"project_id": project_id, "event_id": event_id}, ) data.setdefault("_metrics", {})["flag.processing.error"] = True data.setdefault("_metrics", {})["flag.processing.fatal"] = True has_changed = True break else: # sleep for `retry_after` but max 5 seconds and try again metrics.incr( "tasks.store.symbolicate_event.retry", tags={"symbolication_function": symbolication_function_name}, ) sleep(min(e.retry_after, SYMBOLICATOR_MAX_RETRY_AFTER)) continue except Exception: metrics.incr( "tasks.store.symbolicate_event.fatal", tags={ "reason": "error", "symbolication_function": symbolication_function_name, }, ) error_logger.exception("tasks.store.symbolicate_event.symbolication") data.setdefault("_metrics", {})["flag.processing.error"] = True data.setdefault("_metrics", {})["flag.processing.fatal"] = True has_changed = True break if submit_realtime_metrics: with sentry_sdk.start_span( op="tasks.store.symbolicate_event.low_priority.metrics.histogram" ): symbolication_duration = int(time() - symbolication_start_time) try: realtime_metrics.increment_project_duration_counter( project_id, timestamp, symbolication_duration ) except Exception as e: sentry_sdk.capture_exception(e) # We cannot persist canonical types in the cache, so we need to # downgrade this. if isinstance(data, CANONICAL_TYPES): data = dict(data.items()) if has_changed: cache_key = event_processing_store.store(data) return _continue_to_process_event()