def test_get_sdk_from_event(): sdk_info = get_sdk_from_event( { 'debug_meta': { 'sdk_info': { 'sdk_name': 'iOS', 'version_major': 9, 'version_minor': 3, 'version_patchlevel': 0, } } } ) assert sdk_info['sdk_name'] == 'iOS' assert sdk_info['version_major'] == 9 assert sdk_info['version_minor'] == 3 assert sdk_info['version_patchlevel'] == 0 sdk_info = get_sdk_from_event( { 'contexts': { 'os': { 'type': 'os', 'name': 'iOS', 'version': '9.3.1.1234', } } } ) assert sdk_info['sdk_name'] == 'iOS' assert sdk_info['version_major'] == 9 assert sdk_info['version_minor'] == 3 assert sdk_info['version_patchlevel'] == 1
def test_get_sdk_from_event(): sdk_info = get_sdk_from_event({ "debug_meta": { "sdk_info": { "sdk_name": "iOS", "version_major": 9, "version_minor": 3, "version_patchlevel": 0, } } }) assert sdk_info["sdk_name"] == "iOS" assert sdk_info["version_major"] == 9 assert sdk_info["version_minor"] == 3 assert sdk_info["version_patchlevel"] == 0 sdk_info = get_sdk_from_event({ "contexts": { "os": { "type": "os", "name": "iOS", "version": "9.3.1.1234" } } }) assert sdk_info["sdk_name"] == "iOS" assert sdk_info["version_major"] == 9 assert sdk_info["version_minor"] == 3 assert sdk_info["version_patchlevel"] == 1
def __init__(self, *args, **kwargs): StacktraceProcessor.__init__(self, *args, **kwargs) # If true, the project has been opted into using the symbolicator # service for native symbolication, which also means symbolic is not # used at all anymore. # The (iOS) symbolserver is still used regardless of this value. self.use_symbolicator = _is_symbolicator_enabled( self.project, self.data) metrics.incr('native.use_symbolicator', tags={'value': self.use_symbolicator}) self.arch = cpu_name_from_data(self.data) self.signal = signal_from_data(self.data) self.sym = None self.difs_referenced = set() images = get_path(self.data, 'debug_meta', 'images', default=(), filter=self._is_valid_image) if images: self.available = True self.sdk_info = get_sdk_from_event(self.data) self.object_lookup = ObjectLookup(images) self.images = images else: self.available = False
def __init__(self, *args, **kwargs): StacktraceProcessor.__init__(self, *args, **kwargs) # If true, the project has been opted into using the symbolicator # service for native symbolication, which also means symbolic is not # used at all anymore. # The (iOS) symbolserver is still used regardless of this value. self.use_symbolicator = _is_symbolicator_enabled(self.project, self.data) metrics.incr('native.use_symbolicator', tags={'value': self.use_symbolicator}) self.arch = cpu_name_from_data(self.data) self.signal = signal_from_data(self.data) self.sym = None self.difs_referenced = set() images = get_path(self.data, 'debug_meta', 'images', default=(), filter=self._is_valid_image) if images: self.available = True self.sdk_info = get_sdk_from_event(self.data) self.object_lookup = ObjectLookup(images) self.images = images else: self.available = False
def _merge_minidump_response(data, response): data["platform"] = "native" if response.get("crashed") is not None: data["level"] = "fatal" if response["crashed"] else "info" validate_and_set_timestamp(data, response.get("timestamp")) if response.get("system_info"): _merge_system_info(data, response["system_info"]) sdk_info = get_sdk_from_event(data) images = [] set_path(data, "debug_meta", "images", value=images) for complete_image in response["modules"]: image = {} _merge_image(image, complete_image, sdk_info, lambda e: write_error(e, data)) images.append(image) # Extract the crash reason and infos data_exception = get_path(data, "exception", "values", 0) exc_value = ("Assertion Error: %s" % response.get("assertion") if response.get("assertion") else "Fatal Error: %s" % response.get("crash_reason")) data_exception["value"] = exc_value data_exception["type"] = response.get("crash_reason") data_threads = [] if response["stacktraces"]: data["threads"] = {"values": data_threads} else: error = SymbolicationFailed(message="minidump has no thread list", type=EventError.NATIVE_SYMBOLICATOR_FAILED) write_error(error, data) for complete_stacktrace in response["stacktraces"]: is_requesting = complete_stacktrace.get("is_requesting") thread_id = complete_stacktrace.get("thread_id") data_thread = {"id": thread_id, "crashed": is_requesting} data_threads.append(data_thread) if is_requesting: data_exception["thread_id"] = thread_id data_stacktrace = data_exception.setdefault("stacktrace", {}) data_stacktrace["frames"] = [] else: data_thread["stacktrace"] = data_stacktrace = {"frames": []} if complete_stacktrace.get("registers"): data_stacktrace["registers"] = complete_stacktrace["registers"] for complete_frame in reversed(complete_stacktrace["frames"]): new_frame = {} _merge_frame(new_frame, complete_frame) data_stacktrace["frames"].append(new_frame)
def __init__(self, *args, **kwargs): StacktraceProcessor.__init__(self, *args, **kwargs) debug_meta = self.data.get('debug_meta') self.sym = None if debug_meta: self.available = True self.debug_meta = debug_meta self.sdk_info = get_sdk_from_event(self.data) else: self.available = False
def merge_symbolicator_minidump_response(data, response): sdk_info = get_sdk_from_event(data) # TODO(markus): Add OS context here when `merge_process_state_event` is no # longer called for symbolicator projects images = [] set_path(data, 'debug_meta', 'images', value=images) for complete_image in response['modules']: image = {} merge_symbolicator_image( image, complete_image, sdk_info, lambda e: handle_symbolication_failed(e, data=data)) images.append(image) data_threads = [] data['threads'] = {'values': data_threads} data_exception = get_path(data, 'exception', 'values', 0) for complete_stacktrace in response['stacktraces']: is_requesting = complete_stacktrace.get('is_requesting') thread_id = complete_stacktrace.get('thread_id') data_thread = { 'id': thread_id, 'crashed': is_requesting, } data_threads.append(data_thread) if is_requesting: data_stacktrace = get_path(data_exception, 'stacktrace') assert isinstance(data_stacktrace, dict), data_stacktrace # Make exemption specifically for unreal portable callstacks # TODO(markus): Allow overriding stacktrace more generically # (without looking into unreal context) once we no longer parse # minidump in the endpoint (right now we can't distinguish that # from user json). if data_stacktrace['frames'] and is_unreal_exception_stacktrace( data): continue data_stacktrace['frames'] = [] else: data_thread['stacktrace'] = data_stacktrace = {'frames': []} if complete_stacktrace.get('registers'): data_stacktrace['registers'] = complete_stacktrace['registers'] for complete_frame in reversed(complete_stacktrace['frames']): new_frame = {} merge_symbolicated_frame(new_frame, complete_frame) data_stacktrace['frames'].append(new_frame)
def __init__(self, *args, **kwargs): StacktraceProcessor.__init__(self, *args, **kwargs) debug_meta = self.data.get('debug_meta') self.cpu_name = cpu_name_from_data(self.data) self.sym = None if debug_meta: self.available = True self.debug_meta = debug_meta self.sdk_info = get_sdk_from_event(self.data) self.image_lookup = ImageLookup(self.debug_meta['images']) else: self.available = False
def merge_symbolicator_minidump_response(data, response): sdk_info = get_sdk_from_event(data) # TODO(markus): Add OS context here when `merge_process_state_event` is no # longer called for symbolicator projects images = [] set_path(data, 'debug_meta', 'images', value=images) for complete_image in response['modules']: image = {} merge_symbolicator_image( image, complete_image, sdk_info, lambda e: handle_symbolication_failed(e, data=data) ) images.append(image) data_threads = [] data['threads'] = {'values': data_threads} data_exception = get_path(data, 'exception', 'values', 0) for complete_stacktrace in response['stacktraces']: is_requesting = complete_stacktrace.get('is_requesting') thread_id = complete_stacktrace.get('thread_id') data_thread = { 'id': thread_id, 'crashed': is_requesting, } data_threads.append(data_thread) if is_requesting: data_stacktrace = get_path(data_exception, 'stacktrace') assert isinstance(data_stacktrace, dict), data_stacktrace # Make exemption specifically for unreal portable callstacks # TODO(markus): Allow overriding stacktrace more generically # (without looking into unreal context) once we no longer parse # minidump in the endpoint (right now we can't distinguish that # from user json). if data_stacktrace['frames'] and is_unreal_exception_stacktrace(data): continue del data_stacktrace['frames'][:] else: data_thread['stacktrace'] = data_stacktrace = {'frames': []} if complete_stacktrace.get('registers'): data_stacktrace['registers'] = complete_stacktrace['registers'] for complete_frame in reversed(complete_stacktrace['frames']): new_frame = {} merge_symbolicated_frame(new_frame, complete_frame) data_stacktrace['frames'].append(new_frame)
def __init__(self, *args, **kwargs): StacktraceProcessor.__init__(self, *args, **kwargs) debug_meta = self.data.get('debug_meta') self.arch = cpu_name_from_data(self.data) self.sym = None self.difs_referenced = set() if debug_meta: self.available = True self.debug_meta = debug_meta self.sdk_info = get_sdk_from_event(self.data) self.object_lookup = ObjectLookup( [img for img in self.debug_meta['images'] if img['type'] in self.supported_images] ) else: self.available = False
def __init__(self, *args, **kwargs): StacktraceProcessor.__init__(self, *args, **kwargs) debug_meta = self.data.get('debug_meta') self.arch = cpu_name_from_data(self.data) self.sym = None self.dsyms_referenced = set() if debug_meta: self.available = True self.debug_meta = debug_meta self.sdk_info = get_sdk_from_event(self.data) self.object_lookup = ObjectLookup( [img for img in self.debug_meta['images'] if img['type'] in self.supported_images] ) else: self.available = False
def __init__(self, *args, **kwargs): StacktraceProcessor.__init__(self, *args, **kwargs) self.arch = cpu_name_from_data(self.data) self.sym = None self.difs_referenced = set() images = get_path(self.data, 'debug_meta', 'images', default=(), filter=self._is_valid_image) if images: self.available = True self.sdk_info = get_sdk_from_event(self.data) self.object_lookup = ObjectLookup(images) else: self.available = False
def __init__(self, *args, **kwargs): StacktraceProcessor.__init__(self, *args, **kwargs) self.arch = cpu_name_from_data(self.data) self.sym = None self.difs_referenced = set() images = get_path(self.data, 'debug_meta', 'images', default=(), filter=(lambda img: img and img.get('type') in self.supported_images)) if images: self.available = True self.sdk_info = get_sdk_from_event(self.data) self.object_lookup = ObjectLookup(images) else: self.available = False
def __init__(self, *args, **kwargs): StacktraceProcessor.__init__(self, *args, **kwargs) debug_meta = self.data.get('debug_meta') self.cpu_name = cpu_name_from_data(self.data) self.sym = None self.dsyms_referenced = set() if debug_meta: self.available = True self.debug_meta = debug_meta self.sdk_info = get_sdk_from_event(self.data) self.image_lookup = ImageLookup([ img for img in self.debug_meta['images'] if img['type'] == 'apple' ]) else: self.available = False
def normalize(self, request_env=None): request_env = request_env or {} data = self.data errors = data['errors'] = [] # Ignore event meta data for now. data.pop('_meta', None) # Before validating with a schema, attempt to cast values to their desired types # so that the schema doesn't have to take every type variation into account. text = six.text_type fp_types = six.string_types + six.integer_types + (float, ) def to_values(v): return {'values': v} if v and isinstance(v, (tuple, list)) else v def stringify(f): if isinstance(f, float): return text(int(f)) if abs(f) < (1 << 53) else None return text(f) casts = { 'environment': lambda v: text(v) if v is not None else v, 'fingerprint': lambda v: list(x for x in map(stringify, v) if x is not None) if isinstance(v, list) and all(isinstance(f, fp_types) for f in v) else v, 'release': lambda v: text(v) if v is not None else v, 'dist': lambda v: text(v).strip() if v is not None else v, 'time_spent': lambda v: int(v) if v is not None else v, 'tags': lambda v: [(text(v_k).replace(' ', '-').strip(), text(v_v).strip()) for (v_k, v_v) in dict(v).items()], 'timestamp': lambda v: process_timestamp(v), 'platform': lambda v: v if v in VALID_PLATFORMS else 'other', 'logentry': lambda v: v if isinstance(v, dict) else {'message': v}, # These can be sent as lists and need to be converted to {'values': [...]} 'exception': to_values, 'breadcrumbs': to_values, 'threads': to_values, } for c in casts: if c in data: try: data[c] = casts[c](data[c]) except InvalidTimestamp as it: errors.append({'type': it.args[0], 'name': c, 'value': data[c]}) del data[c] except Exception as e: errors.append({'type': EventError.INVALID_DATA, 'name': c, 'value': data[c]}) del data[c] # raw 'message' is coerced to the Message interface, as its used for pure index of # searchable strings. If both a raw 'message' and a Message interface exist, try and # add the former as the 'formatted' attribute of the latter. # See GH-3248 msg_str = data.pop('message', None) if msg_str: msg_if = data.get('logentry') msg_meta = data.get('_meta', {}).get('message') if not msg_if: msg_if = data['logentry'] = {'message': msg_str} if msg_meta: data.setdefault('_meta', {}).setdefault('logentry', {})['message'] = msg_meta if msg_if.get('message') != msg_str: if not msg_if.get('formatted'): msg_if['formatted'] = msg_str if msg_meta: data.setdefault('_meta', {}).setdefault( 'logentry', {})['formatted'] = msg_meta # Fill in ip addresses marked as {{auto}} client_ip = request_env.get('client_ip') if client_ip: if get_path(data, ['sentry.interfaces.Http', 'env', 'REMOTE_ADDR']) == '{{auto}}': data['sentry.interfaces.Http']['env']['REMOTE_ADDR'] = client_ip if get_path(data, ['request', 'env', 'REMOTE_ADDR']) == '{{auto}}': data['request']['env']['REMOTE_ADDR'] = client_ip if get_path(data, ['sentry.interfaces.User', 'ip_address']) == '{{auto}}': data['sentry.interfaces.User']['ip_address'] = client_ip if get_path(data, ['user', 'ip_address']) == '{{auto}}': data['user']['ip_address'] = client_ip # Validate main event body and tags against schema. # XXX(ja): jsonschema does not like CanonicalKeyDict, so we need to pass # in the inner data dict. is_valid, event_errors = validate_and_default_interface(data.data, 'event') errors.extend(event_errors) if 'tags' in data: is_valid, tag_errors = validate_and_default_interface(data['tags'], 'tags', name='tags') errors.extend(tag_errors) # Validate interfaces for k in list(iter(data)): if k in CLIENT_RESERVED_ATTRS: continue value = data.pop(k) if not value: self.logger.debug('Ignored empty interface value: %s', k) continue try: interface = get_interface(k) except ValueError: self.logger.debug('Ignored unknown attribute: %s', k) errors.append({'type': EventError.INVALID_ATTRIBUTE, 'name': k}) continue try: inst = interface.to_python(value) data[inst.get_path()] = inst.to_json() except Exception as e: log = self.logger.debug if isinstance( e, InterfaceValidationError) else self.logger.error log('Discarded invalid value for interface: %s (%r)', k, value, exc_info=True) errors.append({'type': EventError.INVALID_DATA, 'name': k, 'value': value}) # Additional data coercion and defaulting level = data.get('level') or DEFAULT_LOG_LEVEL if isinstance(level, int) or (isinstance(level, six.string_types) and level.isdigit()): level = LOG_LEVELS.get(int(level), DEFAULT_LOG_LEVEL) data['level'] = LOG_LEVELS_MAP.get(level, LOG_LEVELS_MAP[DEFAULT_LOG_LEVEL]) if data.get('dist') and not data.get('release'): data['dist'] = None timestamp = data.get('timestamp') if not timestamp: timestamp = timezone.now() # TODO (alex) can this all be replaced by utcnow? # it looks like the only time that this would even be hit is when timestamp # is not defined, as the earlier process_timestamp already converts existing # timestamps to floats. if isinstance(timestamp, datetime): # We must convert date to local time so Django doesn't mess it up # based on TIME_ZONE if settings.TIME_ZONE: if not timezone.is_aware(timestamp): timestamp = timestamp.replace(tzinfo=timezone.utc) elif timezone.is_aware(timestamp): timestamp = timestamp.replace(tzinfo=None) timestamp = float(timestamp.strftime('%s')) data['timestamp'] = timestamp data['received'] = float(timezone.now().strftime('%s')) data.setdefault('checksum', None) data.setdefault('culprit', None) data.setdefault('dist', None) data.setdefault('environment', None) data.setdefault('extra', {}) data.setdefault('fingerprint', None) data.setdefault('logger', DEFAULT_LOGGER_NAME) data.setdefault('platform', None) data.setdefault('server_name', None) data.setdefault('site', None) data.setdefault('tags', []) data.setdefault('transaction', None) # Fix case where legacy apps pass 'environment' as a tag # instead of a top level key. # TODO (alex) save() just reinserts the environment into the tags if not data.get('environment'): tagsdict = dict(data['tags']) if 'environment' in tagsdict: data['environment'] = tagsdict['environment'] del tagsdict['environment'] data['tags'] = tagsdict.items() # the SDKs currently do not describe event types, and we must infer # them from available attributes data['type'] = eventtypes.infer(data).key data['version'] = self.version exception = data.get('sentry.interfaces.Exception') stacktrace = data.get('sentry.interfaces.Stacktrace') if exception and len(exception['values']) == 1 and stacktrace: exception['values'][0]['stacktrace'] = stacktrace del data['sentry.interfaces.Stacktrace'] # Exception mechanism needs SDK information to resolve proper names in # exception meta (such as signal names). "SDK Information" really means # the operating system version the event was generated on. Some # normalization still works without sdk_info, such as mach_exception # names (they can only occur on macOS). if exception: sdk_info = get_sdk_from_event(data) for ex in exception['values']: if 'mechanism' in ex: normalize_mechanism_meta(ex['mechanism'], sdk_info) # If there is no User ip_addres, update it either from the Http interface # or the client_ip of the request. auth = request_env.get('auth') is_public = auth and auth.is_public add_ip_platforms = ('javascript', 'cocoa', 'objc') http_ip = data.get('sentry.interfaces.Http', {}).get('env', {}).get('REMOTE_ADDR') if http_ip: data.setdefault('sentry.interfaces.User', {}).setdefault('ip_address', http_ip) elif client_ip and (is_public or data.get('platform') in add_ip_platforms): data.setdefault('sentry.interfaces.User', {}).setdefault('ip_address', client_ip) # Trim values data['logger'] = trim(data['logger'].strip(), 64) trim_dict(data['extra'], max_size=settings.SENTRY_MAX_EXTRA_VARIABLE_SIZE) if data['culprit']: data['culprit'] = trim(data['culprit'], MAX_CULPRIT_LENGTH) if data['transaction']: data['transaction'] = trim(data['transaction'], MAX_CULPRIT_LENGTH) return data
def normalize(self, request_env=None): request_env = request_env or {} data = self.data errors = data['errors'] = [] # Before validating with a schema, attempt to cast values to their desired types # so that the schema doesn't have to take every type variation into account. text = six.text_type fp_types = six.string_types + six.integer_types + (float, ) def to_values(v): return {'values': v} if v and isinstance(v, (tuple, list)) else v def convert_fingerprint(values): rv = values[:] bad_float = False for idx, item in enumerate(rv): if isinstance(item, float) and \ (abs(item) >= (1 << 53) or int(item) != item): bad_float = True rv[idx] = text(item) if bad_float: metrics.incr( 'events.bad_float_fingerprint', skip_internal=True, tags={ 'project_id': data.get('project'), }, ) return rv casts = { 'environment': lambda v: text(v) if v is not None else v, 'fingerprint': lambda v: convert_fingerprint(v) if isinstance(v, list) and all(isinstance(f, fp_types) for f in v) else v, 'release': lambda v: text(v) if v is not None else v, 'dist': lambda v: text(v).strip() if v is not None else v, 'time_spent': lambda v: int(v) if v is not None else v, 'tags': lambda v: [(text(v_k).replace(' ', '-').strip(), text(v_v).strip()) for (v_k, v_v) in dict(v).items()], 'timestamp': lambda v: process_timestamp(v), 'platform': lambda v: v if v in VALID_PLATFORMS else 'other', 'sentry.interfaces.Message': lambda v: v if isinstance(v, dict) else { 'message': v }, # These can be sent as lists and need to be converted to {'values': [...]} 'exception': to_values, 'sentry.interfaces.Exception': to_values, 'breadcrumbs': to_values, 'sentry.interfaces.Breadcrumbs': to_values, 'threads': to_values, 'sentry.interfaces.Threads': to_values, } for c in casts: if c in data: try: data[c] = casts[c](data[c]) except InvalidTimestamp as it: errors.append({ 'type': it.args[0], 'name': c, 'value': data[c] }) del data[c] except Exception as e: errors.append({ 'type': EventError.INVALID_DATA, 'name': c, 'value': data[c] }) del data[c] # raw 'message' is coerced to the Message interface, as its used for pure index of # searchable strings. If both a raw 'message' and a Message interface exist, try and # add the former as the 'formatted' attribute of the latter. # See GH-3248 msg_str = data.pop('message', None) if msg_str: msg_if = data.setdefault('sentry.interfaces.Message', {'message': msg_str}) if msg_if.get('message') != msg_str: msg_if.setdefault('formatted', msg_str) # Fill in ip addresses marked as {{auto}} client_ip = request_env.get('client_ip') if client_ip: if get_path(data, ['sentry.interfaces.Http', 'env', 'REMOTE_ADDR' ]) == '{{auto}}': data['sentry.interfaces.Http']['env'][ 'REMOTE_ADDR'] = client_ip if get_path(data, ['request', 'env', 'REMOTE_ADDR']) == '{{auto}}': data['request']['env']['REMOTE_ADDR'] = client_ip if get_path( data, ['sentry.interfaces.User', 'ip_address']) == '{{auto}}': data['sentry.interfaces.User']['ip_address'] = client_ip if get_path(data, ['user', 'ip_address']) == '{{auto}}': data['user']['ip_address'] = client_ip # Validate main event body and tags against schema is_valid, event_errors = validate_and_default_interface(data, 'event') errors.extend(event_errors) if 'tags' in data: is_valid, tag_errors = validate_and_default_interface(data['tags'], 'tags', name='tags') errors.extend(tag_errors) # Validate interfaces for k in list(iter(data)): if k in CLIENT_RESERVED_ATTRS: continue value = data.pop(k) if not value: self.logger.debug('Ignored empty interface value: %s', k) continue try: interface = get_interface(k) except ValueError: self.logger.debug('Ignored unknown attribute: %s', k) errors.append({ 'type': EventError.INVALID_ATTRIBUTE, 'name': k }) continue try: inst = interface.to_python(value) data[inst.get_path()] = inst.to_json() except Exception as e: log = self.logger.debug if isinstance( e, InterfaceValidationError) else self.logger.error log('Discarded invalid value for interface: %s (%r)', k, value, exc_info=True) errors.append({ 'type': EventError.INVALID_DATA, 'name': k, 'value': value }) # Additional data coercion and defaulting level = data.get('level') or DEFAULT_LOG_LEVEL if isinstance(level, int) or (isinstance(level, six.string_types) and level.isdigit()): level = LOG_LEVELS.get(int(level), DEFAULT_LOG_LEVEL) data['level'] = LOG_LEVELS_MAP.get(level, LOG_LEVELS_MAP[DEFAULT_LOG_LEVEL]) if data.get('dist') and not data.get('release'): data['dist'] = None timestamp = data.get('timestamp') if not timestamp: timestamp = timezone.now() # TODO (alex) can this all be replaced by utcnow? # it looks like the only time that this would even be hit is when timestamp # is not defined, as the earlier process_timestamp already converts existing # timestamps to floats. if isinstance(timestamp, datetime): # We must convert date to local time so Django doesn't mess it up # based on TIME_ZONE if settings.TIME_ZONE: if not timezone.is_aware(timestamp): timestamp = timestamp.replace(tzinfo=timezone.utc) elif timezone.is_aware(timestamp): timestamp = timestamp.replace(tzinfo=None) timestamp = float(timestamp.strftime('%s')) data['timestamp'] = timestamp data['received'] = float(timezone.now().strftime('%s')) data.setdefault('checksum', None) data.setdefault('culprit', None) data.setdefault('dist', None) data.setdefault('environment', None) data.setdefault('extra', {}) data.setdefault('fingerprint', None) data.setdefault('logger', DEFAULT_LOGGER_NAME) data.setdefault('platform', None) data.setdefault('server_name', None) data.setdefault('site', None) data.setdefault('tags', []) data.setdefault('transaction', None) # Fix case where legacy apps pass 'environment' as a tag # instead of a top level key. # TODO (alex) save() just reinserts the environment into the tags if not data.get('environment'): tagsdict = dict(data['tags']) if 'environment' in tagsdict: data['environment'] = tagsdict['environment'] del tagsdict['environment'] data['tags'] = tagsdict.items() # the SDKs currently do not describe event types, and we must infer # them from available attributes data['type'] = eventtypes.infer(data).key data['version'] = self.version exception = data.get('sentry.interfaces.Exception') stacktrace = data.get('sentry.interfaces.Stacktrace') if exception and len(exception['values']) == 1 and stacktrace: exception['values'][0]['stacktrace'] = stacktrace del data['sentry.interfaces.Stacktrace'] # Exception mechanism needs SDK information to resolve proper names in # exception meta (such as signal names). "SDK Information" really means # the operating system version the event was generated on. Some # normalization still works without sdk_info, such as mach_exception # names (they can only occur on macOS). if exception: sdk_info = get_sdk_from_event(data) for ex in exception['values']: if 'mechanism' in ex: normalize_mechanism_meta(ex['mechanism'], sdk_info) # If there is no User ip_addres, update it either from the Http interface # or the client_ip of the request. auth = request_env.get('auth') is_public = auth and auth.is_public add_ip_platforms = ('javascript', 'cocoa', 'objc') http_ip = data.get('sentry.interfaces.Http', {}).get('env', {}).get('REMOTE_ADDR') if http_ip: data.setdefault('sentry.interfaces.User', {}).setdefault('ip_address', http_ip) elif client_ip and (is_public or data.get('platform') in add_ip_platforms): data.setdefault('sentry.interfaces.User', {}).setdefault('ip_address', client_ip) # Trim values data['logger'] = trim(data['logger'].strip(), 64) trim_dict(data['extra'], max_size=settings.SENTRY_MAX_EXTRA_VARIABLE_SIZE) if data['culprit']: data['culprit'] = trim(data['culprit'], MAX_CULPRIT_LENGTH) if data['transaction']: data['transaction'] = trim(data['transaction'], MAX_CULPRIT_LENGTH) return data
def resolve_frame_symbols(data): debug_meta = data['debug_meta'] debug_images = debug_meta['images'] sdk_info = get_sdk_from_event(data) stacktraces = find_all_stacktraces(data) if not stacktraces: return project = Project.objects.get_from_cache( id=data['project'], ) errors = [] referenced_images = find_stacktrace_referenced_images( debug_images, [x[0] for x in stacktraces]) sym = Symbolizer(project, debug_images, referenced_images=referenced_images) frame = None idx = -1 def report_error(exc_type, exc_value, tb): if exc_value.is_user_fixable or exc_value.is_sdk_failure: errors.append({ 'type': EventError.NATIVE_INTERNAL_FAILURE, 'frame': frame, 'error': u'frame #%d: %s' % (idx, exc_value) }) if not exc_value.is_user_fixable: logger.debug('Failed to symbolicate', exc_info=(exc_type, exc_value, tb)) with sym: for stacktrace, container in stacktraces: store_raw = False new_frames = list(stacktrace['frames']) for idx, frame in enumerate(stacktrace['frames']): if 'image_addr' not in frame or \ 'instruction_addr' not in frame or \ 'symbol_addr' not in frame: continue try: # Construct a raw frame that is used by the symbolizer # backend. raw_frame = { 'object_name': frame.get('package'), 'object_addr': frame['image_addr'], 'instruction_addr': frame['instruction_addr'], 'symbol_addr': frame['symbol_addr'], } new_frame = dict(frame) try: sfrm = sym.symbolize_frame(raw_frame, sdk_info) except SymbolicationFailed: report_error(*sys.exc_info()) else: symbol = sfrm.get('symbol_name') or \ new_frame.get('function') or '<unknown>' function = demangle_symbol(symbol, simplified=True) new_frame['function'] = function # If we demangled something, store the original in the # symbol portion of the frame if function != symbol: new_frame['symbol'] = symbol new_frame['abs_path'] = sfrm.get('filename') or None if new_frame['abs_path']: new_frame['filename'] = posixpath.basename( new_frame['abs_path']) if sfrm.get('line') is not None: new_frame['lineno'] = sfrm['line'] else: new_frame['instruction_offset'] = \ parse_addr(sfrm['instruction_addr']) - \ parse_addr(sfrm['symbol_addr']) if sfrm.get('column') is not None: new_frame['colno'] = sfrm['column'] new_frame['package'] = sfrm['object_name'] \ or new_frame.get('package') new_frame['symbol_addr'] = '0x%x' % \ parse_addr(sfrm['symbol_addr']) new_frame['instruction_addr'] = '0x%x' % parse_addr( sfrm['instruction_addr']) new_frame['in_app'] = sym.is_in_app(raw_frame) if new_frame != frame: new_frames[idx] = new_frame store_raw = True except Exception: logger.exception('Failed to symbolicate') errors.append({ 'type': EventError.NATIVE_INTERNAL_FAILURE, 'error': 'The symbolicator encountered an internal failure', }) # Remember the raw stacktrace. if store_raw and container is not None: container['raw_stacktrace'] = { 'frames': stacktrace['frames'], } # Put the new frames in stacktrace['frames'] = new_frames if errors: data.setdefault('errors', []).extend(errors) return data
def process_payload(data): project = Project.objects.get_from_cache(id=data["project"]) symbolicator = Symbolicator(project=project, event_id=data["event_id"]) stacktrace_infos = [ stacktrace for stacktrace in find_stacktraces_in_data(data) if any( is_native_platform(x) for x in stacktrace.platforms) ] modules = native_images_from_data(data) stacktraces = [{ "registers": sinfo.stacktrace.get("registers") or {}, "frames": get_frames_for_symbolication( sinfo.stacktrace.get("frames") or (), data, modules), } for sinfo in stacktrace_infos] if not any(stacktrace["frames"] for stacktrace in stacktraces): return signal = signal_from_data(data) response = symbolicator.process_payload(stacktraces=stacktraces, modules=modules, signal=signal) if not _handle_response_status(data, response): return data assert len(modules) == len(response["modules"]), (modules, response) sdk_info = get_sdk_from_event(data) for raw_image, complete_image in zip(modules, response["modules"]): _merge_image(raw_image, complete_image, sdk_info, data) assert len(stacktraces) == len(response["stacktraces"]), (stacktraces, response) for sinfo, complete_stacktrace in zip(stacktrace_infos, response["stacktraces"]): complete_frames_by_idx = {} for complete_frame in complete_stacktrace.get("frames") or (): complete_frames_by_idx.setdefault(complete_frame["original_index"], []).append(complete_frame) new_frames = [] native_frames_idx = 0 for raw_frame in reversed(sinfo.stacktrace["frames"]): if not _handles_frame(data, raw_frame): new_frames.append(raw_frame) continue for complete_frame in complete_frames_by_idx.get( native_frames_idx) or (): merged_frame = dict(raw_frame) _merge_frame(merged_frame, complete_frame) if merged_frame.get("package"): raw_frame["package"] = merged_frame["package"] new_frames.append(merged_frame) native_frames_idx += 1 if sinfo.container is not None and native_frames_idx > 0: sinfo.container["raw_stacktrace"] = { "frames": list(sinfo.stacktrace["frames"]), "registers": sinfo.stacktrace.get("registers"), } new_frames.reverse() sinfo.stacktrace["frames"] = new_frames return data
def _merge_full_response(data, response): data["platform"] = "native" if response.get("crashed") is not None: data["level"] = "fatal" if response["crashed"] else "info" if response.get("system_info"): _merge_system_info(data, response["system_info"]) sdk_info = get_sdk_from_event(data) images = [] set_path(data, "debug_meta", "images", value=images) for complete_image in response["modules"]: image = {} _merge_image(image, complete_image, sdk_info, data) images.append(image) # Extract the crash reason and infos data_exception = get_path(data, "exception", "values", 0) if response.get("assertion"): data_exception["value"] = "Assertion Error: {}".format( response["assertion"]) elif response.get("crash_details"): data_exception["value"] = response["crash_details"] elif response.get("crash_reason"): data_exception["value"] = "Fatal Error: {}".format( response["crash_reason"]) else: # We're merging a full response, so there was no initial payload # submitted. Assuming that this still contains the placeholder, remove # it rather than showing a default value. data_exception.pop("value", None) if response.get("crash_reason"): data_exception["type"] = response["crash_reason"] data_threads = [] if response["stacktraces"]: data["threads"] = {"values": data_threads} else: error = SymbolicationFailed(message="minidump has no thread list", type=EventError.NATIVE_SYMBOLICATOR_FAILED) write_error(error, data) for complete_stacktrace in response["stacktraces"]: is_requesting = complete_stacktrace.get("is_requesting") thread_id = complete_stacktrace.get("thread_id") data_thread = {"id": thread_id, "crashed": is_requesting} data_threads.append(data_thread) if is_requesting: data_exception["thread_id"] = thread_id data_stacktrace = data_exception.setdefault("stacktrace", {}) data_stacktrace["frames"] = [] else: data_thread["stacktrace"] = data_stacktrace = {"frames": []} if complete_stacktrace.get("registers"): data_stacktrace["registers"] = complete_stacktrace["registers"] for complete_frame in reversed(complete_stacktrace["frames"]): new_frame = {} _merge_frame(new_frame, complete_frame) data_stacktrace["frames"].append(new_frame)
def _merge_minidump_response(data, response): data['platform'] = 'native' if response.get('crashed') is not None: data['level'] = 'fatal' if response['crashed'] else 'info' validate_and_set_timestamp(data, response.get('timestamp')) if response.get('system_info'): _merge_system_info(data, response['system_info']) sdk_info = get_sdk_from_event(data) images = [] set_path(data, 'debug_meta', 'images', value=images) for complete_image in response['modules']: image = {} _merge_image(image, complete_image, sdk_info, lambda e: write_error(e, data)) images.append(image) # Extract the crash reason and infos data_exception = get_path(data, 'exception', 'values', 0) exc_value = ('Assertion Error: %s' % response.get('assertion') if response.get('assertion') else 'Fatal Error: %s' % response.get('crash_reason')) data_exception['value'] = exc_value data_exception['type'] = response.get('crash_reason') data_threads = [] if response['stacktraces']: data['threads'] = {'values': data_threads} else: error = SymbolicationFailed(message='minidump has no thread list', type=EventError.NATIVE_SYMBOLICATOR_FAILED) write_error(error, data) for complete_stacktrace in response['stacktraces']: is_requesting = complete_stacktrace.get('is_requesting') thread_id = complete_stacktrace.get('thread_id') data_thread = { 'id': thread_id, 'crashed': is_requesting, } data_threads.append(data_thread) if is_requesting: data_exception['thread_id'] = thread_id data_stacktrace = data_exception.setdefault('stacktrace', {}) data_stacktrace['frames'] = [] else: data_thread['stacktrace'] = data_stacktrace = {'frames': []} if complete_stacktrace.get('registers'): data_stacktrace['registers'] = complete_stacktrace['registers'] for complete_frame in reversed(complete_stacktrace['frames']): new_frame = {} _merge_frame(new_frame, complete_frame) data_stacktrace['frames'].append(new_frame)
def resolve_frame_symbols(data): debug_meta = data.get('debug_meta') if not debug_meta: return debug_images = debug_meta['images'] sdk_info = get_sdk_from_event(data) stacktraces = find_all_stacktraces(data) if not stacktraces: return project = Project.objects.get_from_cache( id=data['project'], ) errors = [] referenced_images = find_stacktrace_referenced_images( debug_images, stacktraces) sym = Symbolizer(project, debug_images, referenced_images=referenced_images) frame = None idx = -1 def report_error(e): errors.append({ 'type': EventError.NATIVE_INTERNAL_FAILURE, 'frame': frame, 'error': 'frame #%d: %s: %s' % ( idx, e.__class__.__name__, six.text_type(e), ) }) longest_addr = 0 processed_frames = [] with sym: for stacktrace in stacktraces: for idx, frame in enumerate(stacktrace['frames']): if 'image_addr' not in frame or \ 'instruction_addr' not in frame or \ 'symbol_addr' not in frame: continue try: sfrm = sym.symbolize_frame({ 'object_name': frame.get('package'), 'object_addr': frame['image_addr'], 'instruction_addr': frame['instruction_addr'], 'symbol_addr': frame['symbol_addr'], }, sdk_info, report_error=report_error) if not sfrm: continue # XXX: log here if symbol could not be found? frame['function'] = sfrm.get('symbol_name') or \ frame.get('function') or '<unknown>' frame['abs_path'] = sfrm.get('filename') or None if frame['abs_path']: frame['filename'] = posixpath.basename(frame['abs_path']) if sfrm.get('line') is not None: frame['lineno'] = sfrm['line'] else: frame['instruction_offset'] = \ parse_addr(sfrm['instruction_addr']) - \ parse_addr(sfrm['symbol_addr']) if sfrm.get('column') is not None: frame['colno'] = sfrm['column'] frame['package'] = sfrm['object_name'] or frame.get('package') frame['symbol_addr'] = '0x%x' % parse_addr(sfrm['symbol_addr']) frame['instruction_addr'] = '0x%x' % parse_addr( sfrm['instruction_addr']) frame['in_app'] = is_in_app(frame) longest_addr = max(longest_addr, len(frame['symbol_addr']), len(frame['instruction_addr'])) processed_frames.append(frame) except Exception: logger.exception('Failed to symbolicate') errors.append({ 'type': EventError.NATIVE_INTERNAL_FAILURE, 'error': 'The symbolicator encountered an internal failure', }) # Pad out addresses to be of the same length for frame in processed_frames: for key in 'symbol_addr', 'instruction_addr': frame[key] = '0x' + frame[key][2:].rjust(longest_addr - 2, '0') if errors: data.setdefault('errors', []).extend(errors) return data
def process_payload(data): project = Project.objects.get_from_cache(id=data['project']) symbolicator = Symbolicator(project=project, event_id=data['event_id']) stacktrace_infos = [ stacktrace for stacktrace in find_stacktraces_in_data(data) if any( is_native_platform(x) for x in stacktrace.platforms) ] stacktraces = [{ 'registers': sinfo.stacktrace.get('registers') or {}, 'frames': [ f for f in reversed(sinfo.stacktrace.get('frames') or ()) if _handles_frame(data, f) ] } for sinfo in stacktrace_infos] if not any(stacktrace['frames'] for stacktrace in stacktraces): return modules = native_images_from_data(data) signal = signal_from_data(data) response = symbolicator.process_payload( stacktraces=stacktraces, modules=modules, signal=signal, ) if not _handle_response_status(data, response): return data assert len(modules) == len(response['modules']), (modules, response) sdk_info = get_sdk_from_event(data) for raw_image, complete_image in zip(modules, response['modules']): _merge_image(raw_image, complete_image, sdk_info, lambda e: write_error(e, data)) assert len(stacktraces) == len(response['stacktraces']), (stacktraces, response) for sinfo, complete_stacktrace in zip(stacktrace_infos, response['stacktraces']): complete_frames_by_idx = {} for complete_frame in complete_stacktrace.get('frames') or (): complete_frames_by_idx \ .setdefault(complete_frame['original_index'], []) \ .append(complete_frame) new_frames = [] native_frames_idx = 0 for raw_frame in reversed(sinfo.stacktrace['frames']): if not _handles_frame(data, raw_frame): new_frames.append(raw_frame) continue for complete_frame in complete_frames_by_idx.get( native_frames_idx) or (): merged_frame = dict(raw_frame) _merge_frame(merged_frame, complete_frame) if merged_frame.get('package'): raw_frame['package'] = merged_frame['package'] new_frames.append(merged_frame) native_frames_idx += 1 if sinfo.container is not None and native_frames_idx > 0: sinfo.container['raw_stacktrace'] = { 'frames': list(sinfo.stacktrace['frames']), 'registers': sinfo.stacktrace.get('registers') } new_frames.reverse() sinfo.stacktrace['frames'] = new_frames return data
def resolve_frame_symbols(data): debug_meta = data['debug_meta'] debug_images = debug_meta['images'] sdk_info = get_sdk_from_event(data) stacktraces = find_all_stacktraces(data) if not stacktraces: return project = Project.objects.get_from_cache( id=data['project'], ) errors = [] referenced_images = find_stacktrace_referenced_images( debug_images, [x[0] for x in stacktraces]) sym = Symbolizer(project, debug_images, referenced_images=referenced_images) frame = None idx = -1 def report_error(e): errors.append({ 'type': EventError.NATIVE_INTERNAL_FAILURE, 'frame': frame, 'error': 'frame #%d: %s: %s' % ( idx, e.__class__.__name__, six.text_type(e), ) }) with sym: for stacktrace, container in stacktraces: store_raw = False new_frames = list(stacktrace['frames']) for idx, frame in enumerate(stacktrace['frames']): if 'image_addr' not in frame or \ 'instruction_addr' not in frame or \ 'symbol_addr' not in frame: continue try: sfrm = sym.symbolize_frame({ 'object_name': frame.get('package'), 'object_addr': frame['image_addr'], 'instruction_addr': frame['instruction_addr'], 'symbol_addr': frame['symbol_addr'], }, sdk_info, report_error=report_error) if not sfrm: continue new_frame = dict(frame) # XXX: log here if symbol could not be found? symbol = sfrm.get('symbol_name') or \ new_frame.get('function') or '<unknown>' function = demangle_symbol(symbol, simplified=True) new_frame['function'] = function # If we demangled something, store the original in the # symbol portion of the frame if function != symbol: new_frame['symbol'] = symbol new_frame['abs_path'] = sfrm.get('filename') or None if new_frame['abs_path']: new_frame['filename'] = posixpath.basename(new_frame['abs_path']) if sfrm.get('line') is not None: new_frame['lineno'] = sfrm['line'] else: new_frame['instruction_offset'] = \ parse_addr(sfrm['instruction_addr']) - \ parse_addr(sfrm['symbol_addr']) if sfrm.get('column') is not None: new_frame['colno'] = sfrm['column'] new_frame['package'] = sfrm['object_name'] or new_frame.get('package') new_frame['symbol_addr'] = '0x%x' % parse_addr(sfrm['symbol_addr']) new_frame['instruction_addr'] = '0x%x' % parse_addr( sfrm['instruction_addr']) new_frame['in_app'] = is_in_app(new_frame) if new_frame != frame: new_frames[idx] = new_frame store_raw = True except Exception: logger.exception('Failed to symbolicate') errors.append({ 'type': EventError.NATIVE_INTERNAL_FAILURE, 'error': 'The symbolicator encountered an internal failure', }) # Remember the raw stacktrace. if store_raw and container is not None: container['raw_stacktrace'] = { 'frames': stacktrace['frames'], } # Put the new frames in stacktrace['frames'] = new_frames if errors: data.setdefault('errors', []).extend(errors) return data
def merge_symbolicator_minidump_response(data, response): sdk_info = get_sdk_from_event(data) data['platform'] = 'native' if response.get('crashed') is not None: data['level'] = 'fatal' if response['crashed'] else 'info' if response.get('timestamp'): data['timestamp'] = float(response['timestamp']) if response.get('system_info'): merge_symbolicator_minidump_system_info(data, response['system_info']) images = [] set_path(data, 'debug_meta', 'images', value=images) for complete_image in response['modules']: image = {} merge_symbolicator_image( image, complete_image, sdk_info, lambda e: handle_symbolication_failed(e, data=data)) images.append(image) # Extract the crash reason and infos data_exception = get_path(data, 'exception', 'values', 0) exc_value = ('Assertion Error: %s' % response.get('assertion') if response.get('assertion') else 'Fatal Error: %s' % response.get('crash_reason')) data_exception['value'] = exc_value data_exception['type'] = response.get('crash_reason') data_threads = [] if response['stacktraces']: data['threads'] = {'values': data_threads} else: error = SymbolicationFailed(message='minidump has no thread list', type=EventError.NATIVE_SYMBOLICATOR_FAILED) handle_symbolication_failed(error, data=data) for complete_stacktrace in response['stacktraces']: is_requesting = complete_stacktrace.get('is_requesting') thread_id = complete_stacktrace.get('thread_id') data_thread = { 'id': thread_id, 'crashed': is_requesting, } data_threads.append(data_thread) if is_requesting: data_exception['thread_id'] = thread_id data_stacktrace = data_exception.setdefault('stacktrace', {}) # Make exemption specifically for unreal portable callstacks # TODO(markus): Allow overriding stacktrace more generically # (without looking into unreal context) once we no longer parse # minidump in the endpoint (right now we can't distinguish that # from user json). if data_stacktrace.get( 'frames') and is_unreal_exception_stacktrace(data): continue data_stacktrace['frames'] = [] else: data_thread['stacktrace'] = data_stacktrace = {'frames': []} if complete_stacktrace.get('registers'): data_stacktrace['registers'] = complete_stacktrace['registers'] for complete_frame in reversed(complete_stacktrace['frames']): new_frame = {} merge_symbolicated_frame(new_frame, complete_frame) data_stacktrace['frames'].append(new_frame)