def test_get_path_list(self): arr = [1, 2] assert get_path(arr, 1) == 2 assert get_path(arr, -1) == 2 assert get_path(arr, 2) is None assert get_path(arr, '1') is None assert get_path([], 1) is None
def real_message(self): # XXX(mitsuhiko): this is a transitional attribute that should be # removed. `message` will be renamed to `search_message` and this # will become `message`. return get_path(self.data, 'logentry', 'formatted') \ or get_path(self.data, 'logentry', 'message') \ or ''
def signal_from_data(data): exceptions = get_path(data, 'exception', 'values', filter=True) signal = get_path(exceptions, 0, 'mechanism', 'meta', 'signal', 'number') if signal is not None: return int(signal) return None
def _get_meta_header(self): return 'OS Version: %s %s (%s)\nReport Version: %s' % ( get_path(self.context, 'os', 'name'), get_path(self.context, 'os', 'version'), get_path(self.context, 'os', 'build'), REPORT_VERSION, )
def get_frames(self, with_functions=False): from sentry.stacktraces.functions import get_function_name_for_frame if self._frames is None: self._frames = [] def _push_frame(frame): platform = frame.get('platform') or self.event.get('platform') func = get_function_name_for_frame(frame, platform) self._frames.append({ 'function': func or '<unknown>', 'path': frame.get('abs_path') or frame.get('filename'), 'module': frame.get('module'), 'family': get_behavior_family_for_platform(platform), 'package': frame.get('package'), }) have_errors = False for exc in get_path(self.event, 'exception', 'values', filter=True) or (): for frame in get_path(exc, 'stacktrace', 'frames', filter=True) or (): _push_frame(frame) have_errors = True if not have_errors: frames = get_path(self.event, 'stacktrace', 'frames', filter=True) if not frames: threads = get_path(self.event, 'threads', 'values', filter=True) if threads and len(threads) == 1: frames = get_path(threads, 0, 'stacktrace', 'frames') for frame in frames or (): _push_frame(frame) return self._frames
def find_stacktraces_in_data(data, include_raw=False): """Finds all stracktraces in a given data blob and returns it together with some meta information. If `include_raw` is True, then also raw stacktraces are included. """ rv = [] def _report_stack(stacktrace, container): if not stacktrace or not get_path(stacktrace, 'frames', filter=True): return platforms = set( frame.get('platform') or data.get('platform') for frame in get_path(stacktrace, 'frames', filter=True, default=()) ) rv.append(StacktraceInfo(stacktrace=stacktrace, container=container, platforms=platforms)) for exc in get_path(data, 'exception', 'values', filter=True, default=()): _report_stack(exc.get('stacktrace'), exc) _report_stack(data.get('stacktrace'), None) for thread in get_path(data, 'threads', 'values', filter=True, default=()): _report_stack(thread.get('stacktrace'), thread) if include_raw: for info in rv[:]: if info.container is not None: _report_stack(info.container.get('raw_stacktrace'), info.container) return rv
def apply(self, data): # TODO(dcramer): move this into each interface if data.get('stacktrace'): self.filter_stacktrace(data['stacktrace']) for exc in get_path(data, 'exception', 'values', filter=True) or (): if exc.get('stacktrace'): self.filter_stacktrace(exc['stacktrace']) for exc in get_path(data, 'threads', 'values', filter=True) or (): if exc.get('stacktrace'): self.filter_stacktrace(exc['stacktrace']) for crumb in get_path(data, 'breadcrumbs', 'values', filter=True) or (): self.filter_crumb(crumb) if data.get('request'): self.filter_http(data['request']) if data.get('user'): self.filter_user(data['user']) if data.get('csp'): self.filter_csp(data['csp']) if data.get('extra'): data['extra'] = varmap(self.sanitize, data['extra']) if data.get('contexts'): for key, value in six.iteritems(data['contexts']): if value: data['contexts'][key] = varmap(self.sanitize, value)
def get_thread_apple_string(self, thread_info): rv = [] stacktrace = get_path(thread_info, 'stacktrace') if stacktrace is None: return None if stacktrace: frames = get_path(stacktrace, 'frames', filter=True) if frames: for i, frame in enumerate(reversed(frames)): frame_string = self._convert_frame_to_apple_string( frame=frame, next=frames[len(frames) - i - 2] if i < len(frames) - 1 else None, number=i ) if frame_string is not None: rv.append(frame_string) if len(rv) == 0: return None # No frames in thread, so we remove thread is_exception = bool(thread_info.get('mechanism')) thread_id = thread_info.get('id') or thread_info.get('thread_id') or '0' thread_name = thread_info.get('name') thread_name_string = ' name: %s' % (thread_name) if thread_name else '' thread_crashed = thread_info.get('crashed') or is_exception thread_crashed_thread = ' Crashed:' if thread_crashed else '' thread_string = 'Thread %s%s%s\n' % ( thread_id, thread_name_string, thread_crashed_thread ) return thread_string + '\n'.join(rv)
def _get_frame_paths(event): data = event.data frames = get_path(data, 'stacktrace', 'frames', filter=True) if frames: return frames return get_path(data, 'exception', 'values', 0, 'stacktrace', 'frames', filter=True) or []
def find_best_instruction(self, processable_frame): """Given a frame, stacktrace info and frame index this returns the interpolated instruction address we then use for symbolication later. """ if self.arch is None: return parse_addr(processable_frame['instruction_addr']) crashing_frame = False signal = None ip_reg = None # We only need to provide meta information for frame zero if processable_frame.idx == 0: # The signal is useful information for symbolic in some situations # to disambiugate the first frame. If we can get this information # from the mechanism we want to pass it onwards. exceptions = get_path(self.data, 'exception', 'values', filter=True) signal = get_path(exceptions, 0, 'mechanism', 'meta', 'signal', 'number') if signal is not None: signal = int(signal) registers = processable_frame.stacktrace_info.stacktrace.get('registers') if registers: ip_reg_name = arch_get_ip_reg_name(self.arch) if ip_reg_name: ip_reg = registers.get(ip_reg_name) crashing_frame = True return find_best_instruction( processable_frame['instruction_addr'], arch=self.arch, crashing_frame=crashing_frame, signal=signal, ip_reg=ip_reg )
def get_legacy_message(self): # TODO(mitsuhiko): remove this code once it's unused. It's still # being used by plugin code and once the message rename is through # plugins should instead swithc to the actual message attribute or # this method could return what currently is real_message. return get_path(self.data, 'logentry', 'formatted') \ or get_path(self.data, 'logentry', 'message') \ or self.message
def get_sdk_from_event(event): sdk_info = get_path(event, 'debug_meta', 'sdk_info') if sdk_info: return sdk_info os = get_path(event, 'contexts', 'os') if os and os.get('type') == 'os': return get_sdk_from_os(os)
def to_python(cls, data, slim_frames=True, rust_renormalized=RUST_RENORMALIZED_DEFAULT): if not rust_renormalized: is_valid, errors = validate_and_default_interface(data, cls.path) if not is_valid: raise InterfaceValidationError("Invalid exception") if not (data.get('type') or data.get('value')): raise InterfaceValidationError("No 'type' or 'value' present") if get_path(data, 'stacktrace', 'frames', filter=True): stacktrace = Stacktrace.to_python( data['stacktrace'], slim_frames=slim_frames, rust_renormalized=rust_renormalized ) else: stacktrace = None if get_path(data, 'raw_stacktrace', 'frames', filter=True): raw_stacktrace = Stacktrace.to_python( data['raw_stacktrace'], slim_frames=slim_frames, raw=True, rust_renormalized=rust_renormalized ) else: raw_stacktrace = None type = data.get('type') value = data.get('value') if not rust_renormalized: if isinstance(value, six.string_types): if type is None: m = _type_value_re.match(value) if m: type = m.group(1) value = m.group(2).strip() elif value is not None: value = json.dumps(value) value = trim(value, 4096) if data.get('mechanism'): mechanism = Mechanism.to_python(data['mechanism'], rust_renormalized=rust_renormalized) else: mechanism = None kwargs = { 'type': trim(type, 128), 'value': value, 'module': trim(data.get('module'), 128), 'mechanism': mechanism, 'stacktrace': stacktrace, 'thread_id': trim(data.get('thread_id'), 40), 'raw_stacktrace': raw_stacktrace, } return cls(**kwargs)
def _report_stack(stacktrace, container): if not stacktrace or not get_path(stacktrace, 'frames', filter=True): return platforms = set( frame.get('platform') or data.get('platform') for frame in get_path(stacktrace, 'frames', filter=True, default=()) ) rv.append(StacktraceInfo(stacktrace=stacktrace, container=container, platforms=platforms))
def ip_address(self): ip_address = get_path(self.data, 'user', 'ip_address') if ip_address: return ip_address remote_addr = get_path(self.data, 'request', 'env', 'REMOTE_ADDR') if remote_addr: return remote_addr return None
def has_sourcemap(event): if event.platform not in ('javascript', 'node'): return False for exception in get_path(event.data, 'exception', 'values', filter=True, default=()): for frame in get_path(exception, 'stacktrace', 'frames', filter=True, default=()): if 'sourcemap' in (frame.get('data') or ()): return True return False
def get(self, request, project, event_id): """ Retrieve an Apple Crash Report from an event ````````````````````````````````````````````` This endpoint returns the an apple crash report for a specific event. The event ID is either the event as it appears in the Sentry database or the event ID that is reported by the client upon submission. This works only if the event.platform == cocoa """ use_snuba = options.get('snuba.events-queries.enabled') event_cls = event_cls = SnubaEvent if use_snuba else Event event = event_cls.objects.from_event_id(event_id, project_id=project.id) if event is None: raise ResourceDoesNotExist Event.objects.bind_nodes([event], 'data') if event.platform not in ('cocoa', 'native'): return HttpResponse( { 'message': 'Only cocoa events can return an apple crash report', }, status=403 ) symbolicated = (request.GET.get('minified') not in ('1', 'true')) apple_crash_report_string = six.text_type( AppleCrashReport( threads=get_path(event.data, 'threads', 'values', filter=True), context=event.data.get('contexts'), debug_images=get_path(event.data, 'debug_meta', 'images', filter=True), exceptions=get_path(event.data, 'exception', 'values', filter=True), symbolicated=symbolicated, ) ) response = HttpResponse(apple_crash_report_string, content_type='text/plain') if request.GET.get('download') is not None: filename = u"{}{}.crash".format( event.event_id, symbolicated and '-symbolicated' or '') response = StreamingHttpResponse( apple_crash_report_string, content_type='text/plain', ) response['Content-Length'] = len(apple_crash_report_string) response['Content-Disposition'] = 'attachment; filename="%s"' % filename return response
def get_metadata(self): message = strip(get_path(self.data, 'logentry', 'formatted') or get_path(self.data, 'logentry', 'message')) if message: title = truncatechars(message.splitlines()[0], 100) else: title = '<unlabeled event>' return { 'title': title, }
def merge_symbolicator_minidump_response(data, response): sdk_info = get_sdk_from_event(data) # TODO(markus): Add OS context here when `merge_process_state_event` is no # longer called for symbolicator projects images = [] set_path(data, 'debug_meta', 'images', value=images) for complete_image in response['modules']: image = {} merge_symbolicator_image( image, complete_image, sdk_info, lambda e: handle_symbolication_failed(e, data=data) ) images.append(image) data_threads = [] data['threads'] = {'values': data_threads} data_exception = get_path(data, 'exception', 'values', 0) for complete_stacktrace in response['stacktraces']: is_requesting = complete_stacktrace.get('is_requesting') thread_id = complete_stacktrace.get('thread_id') data_thread = { 'id': thread_id, 'crashed': is_requesting, } data_threads.append(data_thread) if is_requesting: data_stacktrace = get_path(data_exception, 'stacktrace') assert isinstance(data_stacktrace, dict), data_stacktrace # Make exemption specifically for unreal portable callstacks # TODO(markus): Allow overriding stacktrace more generically # (without looking into unreal context) once we no longer parse # minidump in the endpoint (right now we can't distinguish that # from user json). if data_stacktrace['frames'] and is_unreal_exception_stacktrace(data): continue del data_stacktrace['frames'][:] else: data_thread['stacktrace'] = data_stacktrace = {'frames': []} if complete_stacktrace.get('registers'): data_stacktrace['registers'] = complete_stacktrace['registers'] for complete_frame in reversed(complete_stacktrace['frames']): new_frame = {} merge_symbolicated_frame(new_frame, complete_frame) data_stacktrace['frames'].append(new_frame)
def ensure_does_not_have_ip(self, data): env = get_path(data, 'request', 'env') if env: env.pop('REMOTE_ADDR', None) user = get_path(data, 'user') if user: user.pop('ip_address', None) sdk = get_path(data, 'sdk') if sdk: sdk.pop('client_ip', None)
def translate_exception(data): message = get_path(data, 'logentry', 'message') if message: data['logentry']['message'] = translate_message(message) formatted = get_path(data, 'logentry', 'formatted') if formatted: data['logentry']['formatted'] = translate_message(formatted) for entry in get_path(data, 'exception', 'values', filter=True, default=()): if 'value' in entry: entry['value'] = translate_message(entry['value']) return data
def _get_legacy_message_with_meta(self, event): meta = event.data.get('_meta') message = get_path(event.data, 'logentry', 'formatted') msg_meta = get_path(meta, 'logentry', 'formatted') if not message: message = get_path(event.data, 'logentry', 'message') msg_meta = get_path(meta, 'logentry', 'message') if not message: message = event.message msg_meta = None return (message, meta_with_chunks(message, msg_meta))
def __init__(self, *args, **kwargs): StacktraceProcessor.__init__(self, *args, **kwargs) # If true, the project has been opted into using the symbolicator # service for native symbolication, which also means symbolic is not # used at all anymore. # The (iOS) symbolserver is still used regardless of this value. self.use_symbolicator = _is_symbolicator_enabled(self.project, self.data) metrics.incr('native.use_symbolicator', tags={'value': self.use_symbolicator}) self.arch = cpu_name_from_data(self.data) self.signal = signal_from_data(self.data) self.sym = None self.difs_referenced = set() images = get_path(self.data, 'debug_meta', 'images', default=(), filter=self._is_valid_image) if images: self.available = True self.sdk_info = get_sdk_from_event(self.data) self.object_lookup = ObjectLookup(images) self.images = images else: self.available = False
def to_python(cls, data): if data and 'values' not in data and 'exc_omitted' not in data: data = {"values": [data]} values = get_path(data, 'values', default=[]) if not isinstance(values, list): raise InterfaceValidationError("Invalid value for 'values'") kwargs = { 'values': [ v and SingleException.to_python(v, slim_frames=False) for v in values ], } if data.get('exc_omitted'): if len(data['exc_omitted']) != 2: raise InterfaceValidationError("Invalid value for 'exc_omitted'") kwargs['exc_omitted'] = data['exc_omitted'] else: kwargs['exc_omitted'] = None instance = cls(**kwargs) # we want to wait to slim things til we've reconciled in_app slim_exception_data(instance) return instance
def get_user_agent(self, data): try: for key, value in get_path(data, 'request', 'headers', filter=True) or (): if key.lower() == 'user-agent': return value except LookupError: return ''
def primary_value_for_data(cls, data): val = get_path(data, 'contexts', cls.type) if val and val.get('type') == cls.type: return val rv = cls.values_for_data(data) if len(rv) == 1: return rv[0]
def iter_threads(self): """Returns an iterator over all threads of the process at the time of the crash, including the crashing thread. The values are of type ``ThreadRef``.""" for thread in get_path(self.data, 'threads', 'values', filter=True, default=()): if thread.get('crashed'): # XXX: Assumes that the full list of threads is present in the # original crash report. This is guaranteed by KSCrash and our # minidump utility. exceptions = get_path(self.data, 'exception', 'values', filter=True) frames = get_path(exceptions, 0, 'stacktrace', 'frames') else: frames = get_path(thread, 'stacktrace', 'frames') tid = thread.get('id') if tid and frames: yield tid, ThreadRef(frames, self.modules)
def full_url(self): url = self.url if url: if self.query_string: url = url + '?' + urlencode(get_path(self.query_string, filter=True)) if self.fragment: url = url + '#' + self.fragment return url
def get_stacktraces(self, data): exceptions = get_path(data, 'exception', 'values', filter=True, default=()) stacktraces = [e['stacktrace'] for e in exceptions if e.get('stacktrace')] if 'stacktrace' in data: stacktraces.append(data['stacktrace']) return [(s, Stacktrace.to_python(s)) for s in stacktraces]
def tags(self): try: rv = sorted([(t, v) for t, v in get_path( self.data, 'tags', filter=True) or () if t is not None and v is not None]) return rv except ValueError: # at one point Sentry allowed invalid tag sets such as (foo, bar) # vs ((tag, foo), (tag, bar)) return []
def get_related(data: JSONData, relation: str) -> Union[None, JSONData]: """Returns related data by looking it up in all the related data included in the page. This first looks up the related object in the data provided under the `relationships` key. Then it uses the `type` and `id` of this key to look up the actual data in `included_relations` which is an index of all related data returned with the page. If the `relation` does not exist in `data` then `None` is returned. """ rel_ptr_data = safe.get_path(data, "relationships", relation, "data") if rel_ptr_data is None: # The query asks for both the appStoreVersion and preReleaseVersion # relations to be included. However for each build there could be only one # of these that will have the data with type and id, the other will have # None for data. return None rel_type = rel_ptr_data["type"] rel_id = rel_ptr_data["id"] return included_relations[(rel_type, rel_id)]
def _send(self, project_id, _type, extra_data=(), asynchronous=True): data = (self.EVENT_PROTOCOL_VERSION, _type) + extra_data # TODO remove this once the unified dataset is available. # Inserting into both events and transactions datasets lets us # simulate what is currently happening via kafka when both the events # and transactions consumers are running. datasets = ["events"] if get_path(extra_data, 0, "data", "type") == "transaction": datasets.append("transactions") try: for dataset in datasets: resp = snuba._snuba_pool.urlopen( "POST", "/tests/{}/eventstream".format(dataset), body=json.dumps(data)) if resp.status != 200: raise snuba.SnubaError("HTTP %s response from Snuba!" % resp.status) return resp except urllib3.exceptions.HTTPError as err: raise snuba.SnubaError(err)
def get_dsym_url(session: Session, app_id: str, bundle_short_version: str, bundle_version: str, platform: str) -> Optional[str]: """ Returns the url for a dsyms bundle. The session must be logged in. :return: The url to use for downloading the dsyms bundle """ with sentry_sdk.start_span(op="itunes-dsym-url", description="Request iTunes dSYM download URL"): details_url = ( f"https://appstoreconnect.apple.com/WebObjects/iTunesConnect.woa/ra/apps/" f"{app_id}/platforms/{platform}/trains/{bundle_short_version}/builds/" f"{bundle_version}/details") logger.debug(f"GET {details_url}") details_response = session.get(details_url) # A non-OK status code will probably mean an expired token/session if details_response.status_code == HTTPStatus.UNAUTHORIZED: raise ITunesSessionExpiredException if details_response.status_code == HTTPStatus.OK: try: data = details_response.json() dsym_url: Optional[str] = safe.get_path( data, "data", "dsymurl") return dsym_url except Exception as e: logger.info( "Could not obtain dSYM info for " "app id=%s, bundle_short=%s, bundle=%s, platform=%s", app_id, bundle_short_version, bundle_version, platform, exc_info=True, ) raise e return None
def cpu_name_from_data(data): """Returns the CPU name from the given data if it exists.""" device = DeviceContextType.primary_value_for_data(data) if device and device.get('arch'): return device['arch'] # TODO: kill this here. we want to not support that going forward unique_cpu_name = None for img in get_path(data, 'debug_meta', 'images', filter=True, default=()): if img.get('arch') and arch_is_known(img['arch']): cpu_name = img['arch'] elif img.get('cpu_type') is not None \ and img.get('cpu_subtype') is not None: cpu_name = arch_from_macho(img['cpu_type'], img['cpu_subtype']) else: cpu_name = None if unique_cpu_name is None: unique_cpu_name = cpu_name elif unique_cpu_name != cpu_name: unique_cpu_name = None break return unique_cpu_name
def delete_old_primary_hash(event): """In case the primary hash changed during reprocessing, we need to tell Snuba before reinserting the event. Snuba may then insert a tombstone row depending on whether the primary_hash is part of the PK/sortkey or not. Only when the primary_hash changed and is part of the sortkey, we need to explicitly tombstone the old row. If the primary_hash is not part of the PK/sortkey, or if the primary_hash did not change, nothing needs to be done as ClickHouse's table merge will merge the two rows together. """ old_primary_hash = get_path(event.data, "contexts", "reprocessing", "original_primary_hash") if old_primary_hash is not None and old_primary_hash != event.get_primary_hash(): from sentry import eventstream eventstream.tombstone_events_unsafe( event.project_id, [event.event_id], old_primary_hash=old_primary_hash, )
def normalize_mechanism_meta(mechanism, sdk_info=None): meta = get_path(mechanism, 'meta') if not meta: return sdk_name = sdk_info['sdk_name'].lower() if sdk_info else '' if sdk_name in ('ios', 'watchos', 'tvos', 'macos'): sdk = 'darwin' elif sdk_name in ('linux', 'android'): sdk = 'linux' elif sdk_name in ('windows', ): sdk = 'windows' else: sdk = None if 'errno' in meta: normalize_mechanism_errno(meta['errno'], sdk) if 'signal' in meta: normalize_mechanism_signal(meta['signal'], sdk) if 'mach_exception' in meta: normalize_mechanism_mach_exception(meta['mach_exception'])
def is_valid_error_message(project_config, message): """ Verify that an error message is not being filtered for the given project. """ filtered_errors = get_path(project_config.config, "filter_settings", FilterTypes.ERROR_MESSAGES, "patterns") if not filtered_errors: return True message = force_text(message).lower() for error in filtered_errors: try: if fnmatch.fnmatch(message, error.lower()): return False except Exception: # fnmatch raises a string when the pattern is bad. # Patterns come from end users and can be full of mistakes. pass return True
def to_python(cls, data, rust_renormalized=RUST_RENORMALIZED_DEFAULT): if not rust_renormalized: if data and 'values' not in data and 'exc_omitted' not in data: data = {"values": [data]} values = get_path(data, 'values', default=[]) if not rust_renormalized: if not isinstance(values, list): raise InterfaceValidationError("Invalid value for 'values'") kwargs = { 'values': [ v and SingleException.to_python( v, slim_frames=False, rust_renormalized=rust_renormalized) for v in values ], } if not rust_renormalized: if data.get('exc_omitted'): if len(data['exc_omitted']) != 2: raise InterfaceValidationError( "Invalid value for 'exc_omitted'") kwargs['exc_omitted'] = data['exc_omitted'] else: kwargs['exc_omitted'] = None else: kwargs.setdefault('exc_omitted', None) instance = cls(**kwargs) if not rust_renormalized: # we want to wait to slim things til we've reconciled in_app slim_exception_data(instance) return instance
def normalize_stacktraces_for_grouping(data, grouping_config=None): def _has_system_frames(frames): system_frames = 0 for frame in frames: if not frame.get('in_app'): system_frames += 1 return bool(system_frames) and len(frames) != system_frames stacktraces = [] for stacktrace_info in find_stacktraces_in_data(data, include_raw=True): frames = get_path(stacktrace_info.stacktrace, 'frames', filter=True, default=()) if frames: stacktraces.append(frames) if not stacktraces: return # If a grouping config is available, run grouping enhancers if grouping_config is not None: platform = data.get('platform') for frames in stacktraces: grouping_config.enhancements.apply_modifications_to_frame( frames, platform) # normalize in-app for frames in stacktraces: has_system_frames = _has_system_frames(frames) for frame in frames: if not has_system_frames: frame['in_app'] = False elif frame.get('in_app') is None: frame['in_app'] = False
def normalize_stacktraces_for_grouping(data, grouping_config=None): """ Applies grouping enhancement rules and ensure in_app is set on all frames. """ stacktraces = [] for stacktrace_info in find_stacktraces_in_data(data, include_raw=True): frames = get_path(stacktrace_info.stacktrace, 'frames', filter=True, default=()) if frames: stacktraces.append(frames) if not stacktraces: return # If a grouping config is available, run grouping enhancers platform = data.get('platform') if grouping_config is not None: for frames in stacktraces: grouping_config.enhancements.apply_modifications_to_frame(frames, platform) # normalize in-app for stacktrace in stacktraces: _normalize_in_app(stacktrace, platform=platform)
def get_multiple_related(self, data: JSONData, relation: str) -> Optional[List[JSONData]]: """Returns a list of all the related objects of the named relation type. This is like :meth:`get_related` but is for relation types which have a list of related objects instead of exactly one. An example of this is a ``build`` can have multiple ``buildBundles`` related to it. Having this as a separate method makes it easier to handle the type checking. """ rel_ptr_data = safe.get_path(data, "relationships", relation, "data") if rel_ptr_data is None: # Because the related information was requested in the query does not mean a # relation of that type did exist. return None assert isinstance(rel_ptr_data, list) all_related = [] for relationship in rel_ptr_data: rel_type = _RelType(relationship["type"]) rel_id = _RelId(relationship["id"]) related_item = self._items[(rel_type, rel_id)] if related_item: all_related.append(related_item) return all_related
def insert(self, group, event, is_new, is_sample, is_regression, is_new_group_environment, primary_hash, skip_consume=False): project = event.project retention_days = quotas.get_event_retention( organization=project.organization, ) event_data = event.get_raw_data() unexpected_tags = set([ k for (k, v) in (get_path(event_data, 'tags', filter=True) or []) if k in self.UNEXPECTED_TAG_KEYS ]) if unexpected_tags: logger.error('%r received unexpected tags: %r', self, unexpected_tags) self._send(project.id, 'insert', extra_data=({ 'group_id': event.group_id, 'event_id': event.event_id, 'organization_id': project.organization_id, 'project_id': event.project_id, # TODO(mitsuhiko): We do not want to send this incorrect # message but this is what snuba needs at the moment. 'message': event.message, 'platform': event.platform, 'datetime': event.datetime, 'data': event_data, 'primary_hash': primary_hash, 'retention_days': retention_days, }, { 'is_new': is_new, 'is_sample': is_sample, 'is_regression': is_regression, 'is_new_group_environment': is_new_group_environment, 'skip_consume': skip_consume, },))
def get_pre_release_version_info(session: Session, credentials: AppConnectCredentials, app_id: str): """ Get all prerelease builds version information for an application The release build version information has the following structure: platform: str - the platform for the build (e.g. IOS, MAC_OS ...) short_version: str - the short version build info ( e.g. '1.0.1'), also called "train" in starship documentation id: str - the IID of the version versions: vec - a vector with builds version: str - the version of the build (e.g. '101'), looks like the build number id: str - the IID of the build NOTE: the pre release version information is identical to the release version information :return: a list of prerelease builds version information (see above) """ url = f"v1/apps/{app_id}/preReleaseVersions" data = _get_appstore_info_paged_data(session, credentials, url) result = [] for d in data: versions = [] v = { "platform": safe.get_path(d, "attributes", "platform"), "short_version": safe.get_path(d, "attributes", "version"), "id": safe.get_path(d, "id"), "versions": versions, } builds_url = safe.get_path(d, "relationships", "builds", "links", "related") for build in _get_appstore_info_paged_data(session, credentials, builds_url): b = { "version": safe.get_path(build, "attributes", "version"), "id": safe.get_path(build, "id"), } versions.append(b) result.append(v) return result
def _get_exception_info(self): rv = [] # We only have one exception at a time exception = get_path(self.exceptions, 0) if not exception: return '' mechanism = upgrade_legacy_mechanism(exception.get('mechanism')) or {} mechanism_meta = get_path(mechanism, 'meta', default={}) signal = get_path(mechanism_meta, 'signal', 'name') name = get_path(mechanism_meta, 'mach_exception', 'name') if name or signal: rv.append('Exception Type: %s%s' % ( name or 'Unknown', signal and (' (%s)' % signal) or '', )) exc_name = get_path(mechanism_meta, 'signal', 'code_name') exc_addr = get_path(mechanism, 'data', 'relevant_address') if exc_name: rv.append('Exception Codes: %s%s' % ( exc_name, exc_addr is not None and (' at %s' % exc_addr) or '', )) if exception.get('thread_id') is not None: rv.append('Crashed Thread: %s' % exception['thread_id']) if exception.get('value'): rv.append('\nApplication Specific Information:\n%s' % exception['value']) return '\n'.join(rv)
def _get_exception_info(self): rv = [] # We only have one exception at a time exception = get_path(self.exceptions, 0) if not exception: return "" mechanism = upgrade_legacy_mechanism(exception.get("mechanism")) or {} mechanism_meta = get_path(mechanism, "meta", default={}) signal = get_path(mechanism_meta, "signal", "name") name = get_path(mechanism_meta, "mach_exception", "name") if name or signal: rv.append( "Exception Type: {}{}".format( name or "Unknown", signal and (" (%s)" % signal) or "" ) ) exc_name = get_path(mechanism_meta, "signal", "code_name") exc_addr = get_path(mechanism, "data", "relevant_address") if exc_name: rv.append( "Exception Codes: %s%s" % (exc_name, exc_addr is not None and (" at %s" % exc_addr) or "") ) if exception.get("thread_id") is not None: rv.append("Crashed Thread: %s" % exception["thread_id"]) if exception.get("value"): rv.append("\nApplication Specific Information:\n%s" % exception["value"]) return "\n".join(rv)
def _localhost_filter(project_config, data): ip_address = get_path(data, "user", "ip_address") or "" url = get_path(data, "request", "url") or "" domain = urlparse(url).hostname return ip_address in _LOCAL_IPS or domain in _LOCAL_DOMAINS
def serialize(self, obj, attrs, user): errors = [ EventError(error).get_api_context() for error in get_path(obj.data, "errors", filter=True, default=()) # TODO(ja): Temporary workaround to hide certain normalization errors. # Remove this and the test in tests/sentry/api/serializers/test_event.py if self.should_display_error(error) ] (message, message_meta) = self._get_legacy_message_with_meta(obj) (tags, tags_meta) = get_tags_with_meta(obj) (context, context_meta) = self._get_attr_with_meta(obj, "extra", {}) (packages, packages_meta) = self._get_attr_with_meta(obj, "modules", {}) received = obj.data.get("received") if received: # Sentry at one point attempted to record invalid types here. # Remove after June 2 2016 try: received = datetime.utcfromtimestamp(received).replace( tzinfo=timezone.utc) except TypeError: received = None d = { "id": obj.event_id, "groupID": str(obj.group_id) if obj.group_id else None, "eventID": obj.event_id, "projectID": str(obj.project_id), "size": obj.size, "entries": attrs["entries"], "dist": obj.dist, # See GH-3248 "message": message, "title": obj.title, "location": obj.location, "user": attrs["user"], "contexts": attrs["contexts"], "sdk": attrs["sdk"], # TODO(dcramer): move into contexts['extra'] "context": context, "packages": packages, "type": obj.get_event_type(), "metadata": obj.get_event_metadata(), "tags": tags, "platform": obj.platform, "dateReceived": received, "errors": errors, "_meta": { "entries": attrs["_meta"]["entries"], "message": message_meta, "user": attrs["_meta"]["user"], "contexts": attrs["_meta"]["contexts"], "sdk": attrs["_meta"]["sdk"], "context": context_meta, "packages": packages_meta, "tags": tags_meta, }, } # Serialize attributes that are specific to different types of events. if obj.get_event_type() == "transaction": d.update(self.__serialize_transaction_attrs(attrs, obj)) else: d.update(self.__serialize_error_attrs(attrs, obj)) return d
def _get_attr_with_meta(self, event, attr, default=None): value = event.data.get(attr, default) meta = get_path(event.data, "_meta", attr) return (value, meta_with_chunks(value, meta))
def has_metadata(self, data): exception = get_path(data, 'exception', 'values', -1) return exception and any(v is not None for v in six.itervalues(exception))
def exceptions(self): return get_path(self.values, filter=True)
def normalize(self, request_env=None): request_env = request_env or {} data = self.data errors = data['errors'] = [] # Before validating with a schema, attempt to cast values to their desired types # so that the schema doesn't have to take every type variation into account. text = six.text_type fp_types = six.string_types + six.integer_types + (float, ) def to_values(v): return {'values': v} if v and isinstance(v, (tuple, list)) else v def convert_fingerprint(values): rv = values[:] bad_float = False for idx, item in enumerate(rv): if isinstance(item, float) and \ (abs(item) >= (1 << 53) or int(item) != item): bad_float = True rv[idx] = text(item) if bad_float: metrics.incr( 'events.bad_float_fingerprint', skip_internal=True, tags={ 'project_id': data.get('project'), }, ) return rv casts = { 'environment': lambda v: text(v) if v is not None else v, 'fingerprint': lambda v: convert_fingerprint(v) if isinstance(v, list) and all(isinstance(f, fp_types) for f in v) else v, 'release': lambda v: text(v) if v is not None else v, 'dist': lambda v: text(v).strip() if v is not None else v, 'time_spent': lambda v: int(v) if v is not None else v, 'tags': lambda v: [(text(v_k).replace(' ', '-').strip(), text(v_v).strip()) for (v_k, v_v) in dict(v).items()], 'timestamp': lambda v: process_timestamp(v), 'platform': lambda v: v if v in VALID_PLATFORMS else 'other', 'sentry.interfaces.Message': lambda v: v if isinstance(v, dict) else { 'message': v }, # These can be sent as lists and need to be converted to {'values': [...]} 'exception': to_values, 'sentry.interfaces.Exception': to_values, 'breadcrumbs': to_values, 'sentry.interfaces.Breadcrumbs': to_values, 'threads': to_values, 'sentry.interfaces.Threads': to_values, } for c in casts: if c in data: try: data[c] = casts[c](data[c]) except InvalidTimestamp as it: errors.append({ 'type': it.args[0], 'name': c, 'value': data[c] }) del data[c] except Exception as e: errors.append({ 'type': EventError.INVALID_DATA, 'name': c, 'value': data[c] }) del data[c] # raw 'message' is coerced to the Message interface, as its used for pure index of # searchable strings. If both a raw 'message' and a Message interface exist, try and # add the former as the 'formatted' attribute of the latter. # See GH-3248 msg_str = data.pop('message', None) if msg_str: msg_if = data.setdefault('sentry.interfaces.Message', {'message': msg_str}) if msg_if.get('message') != msg_str: msg_if.setdefault('formatted', msg_str) # Fill in ip addresses marked as {{auto}} client_ip = request_env.get('client_ip') if client_ip: if get_path(data, ['sentry.interfaces.Http', 'env', 'REMOTE_ADDR' ]) == '{{auto}}': data['sentry.interfaces.Http']['env'][ 'REMOTE_ADDR'] = client_ip if get_path(data, ['request', 'env', 'REMOTE_ADDR']) == '{{auto}}': data['request']['env']['REMOTE_ADDR'] = client_ip if get_path( data, ['sentry.interfaces.User', 'ip_address']) == '{{auto}}': data['sentry.interfaces.User']['ip_address'] = client_ip if get_path(data, ['user', 'ip_address']) == '{{auto}}': data['user']['ip_address'] = client_ip # Validate main event body and tags against schema is_valid, event_errors = validate_and_default_interface(data, 'event') errors.extend(event_errors) if 'tags' in data: is_valid, tag_errors = validate_and_default_interface(data['tags'], 'tags', name='tags') errors.extend(tag_errors) # Validate interfaces for k in list(iter(data)): if k in CLIENT_RESERVED_ATTRS: continue value = data.pop(k) if not value: self.logger.debug('Ignored empty interface value: %s', k) continue try: interface = get_interface(k) except ValueError: self.logger.debug('Ignored unknown attribute: %s', k) errors.append({ 'type': EventError.INVALID_ATTRIBUTE, 'name': k }) continue try: inst = interface.to_python(value) data[inst.get_path()] = inst.to_json() except Exception as e: log = self.logger.debug if isinstance( e, InterfaceValidationError) else self.logger.error log('Discarded invalid value for interface: %s (%r)', k, value, exc_info=True) errors.append({ 'type': EventError.INVALID_DATA, 'name': k, 'value': value }) # Additional data coercion and defaulting level = data.get('level') or DEFAULT_LOG_LEVEL if isinstance(level, int) or (isinstance(level, six.string_types) and level.isdigit()): level = LOG_LEVELS.get(int(level), DEFAULT_LOG_LEVEL) data['level'] = LOG_LEVELS_MAP.get(level, LOG_LEVELS_MAP[DEFAULT_LOG_LEVEL]) if data.get('dist') and not data.get('release'): data['dist'] = None timestamp = data.get('timestamp') if not timestamp: timestamp = timezone.now() # TODO (alex) can this all be replaced by utcnow? # it looks like the only time that this would even be hit is when timestamp # is not defined, as the earlier process_timestamp already converts existing # timestamps to floats. if isinstance(timestamp, datetime): # We must convert date to local time so Django doesn't mess it up # based on TIME_ZONE if settings.TIME_ZONE: if not timezone.is_aware(timestamp): timestamp = timestamp.replace(tzinfo=timezone.utc) elif timezone.is_aware(timestamp): timestamp = timestamp.replace(tzinfo=None) timestamp = float(timestamp.strftime('%s')) data['timestamp'] = timestamp data['received'] = float(timezone.now().strftime('%s')) data.setdefault('checksum', None) data.setdefault('culprit', None) data.setdefault('dist', None) data.setdefault('environment', None) data.setdefault('extra', {}) data.setdefault('fingerprint', None) data.setdefault('logger', DEFAULT_LOGGER_NAME) data.setdefault('platform', None) data.setdefault('server_name', None) data.setdefault('site', None) data.setdefault('tags', []) data.setdefault('transaction', None) # Fix case where legacy apps pass 'environment' as a tag # instead of a top level key. # TODO (alex) save() just reinserts the environment into the tags if not data.get('environment'): tagsdict = dict(data['tags']) if 'environment' in tagsdict: data['environment'] = tagsdict['environment'] del tagsdict['environment'] data['tags'] = tagsdict.items() # the SDKs currently do not describe event types, and we must infer # them from available attributes data['type'] = eventtypes.infer(data).key data['version'] = self.version exception = data.get('sentry.interfaces.Exception') stacktrace = data.get('sentry.interfaces.Stacktrace') if exception and len(exception['values']) == 1 and stacktrace: exception['values'][0]['stacktrace'] = stacktrace del data['sentry.interfaces.Stacktrace'] # Exception mechanism needs SDK information to resolve proper names in # exception meta (such as signal names). "SDK Information" really means # the operating system version the event was generated on. Some # normalization still works without sdk_info, such as mach_exception # names (they can only occur on macOS). if exception: sdk_info = get_sdk_from_event(data) for ex in exception['values']: if 'mechanism' in ex: normalize_mechanism_meta(ex['mechanism'], sdk_info) # If there is no User ip_addres, update it either from the Http interface # or the client_ip of the request. auth = request_env.get('auth') is_public = auth and auth.is_public add_ip_platforms = ('javascript', 'cocoa', 'objc') http_ip = data.get('sentry.interfaces.Http', {}).get('env', {}).get('REMOTE_ADDR') if http_ip: data.setdefault('sentry.interfaces.User', {}).setdefault('ip_address', http_ip) elif client_ip and (is_public or data.get('platform') in add_ip_platforms): data.setdefault('sentry.interfaces.User', {}).setdefault('ip_address', client_ip) # Trim values data['logger'] = trim(data['logger'].strip(), 64) trim_dict(data['extra'], max_size=settings.SENTRY_MAX_EXTRA_VARIABLE_SIZE) if data['culprit']: data['culprit'] = trim(data['culprit'], MAX_CULPRIT_LENGTH) if data['transaction']: data['transaction'] = trim(data['transaction'], MAX_CULPRIT_LENGTH) return data
def test_trusted_external_relays_should_receive_minimal_configs( relay, add_org_key, call_endpoint, default_project, default_projectkey): relay.is_internal = False relay.save() result, status_code = call_endpoint(full_config=False) assert status_code < 400 cfg = safe.get_path(result, "configs", six.text_type(default_project.id)) assert safe.get_path(cfg, "disabled") is False (public_key, ) = cfg["publicKeys"] assert public_key["publicKey"] == default_projectkey.public_key assert public_key["numericId"] == default_projectkey.id assert public_key["isEnabled"] assert "quotas" not in public_key assert safe.get_path(cfg, "slug") == default_project.slug last_change = safe.get_path(cfg, "lastChange") assert _date_regex.match(last_change) is not None last_fetch = safe.get_path(cfg, "lastFetch") assert _date_regex.match(last_fetch) is not None assert safe.get_path(cfg, "organizationId") == default_project.organization.id assert safe.get_path(cfg, "projectId") == default_project.id assert safe.get_path(cfg, "slug") == default_project.slug assert safe.get_path(cfg, "rev") is not None assert safe.get_path(cfg, "config", "trustedRelays") == [relay.public_key] assert safe.get_path(cfg, "config", "filterSettings") is None assert safe.get_path(cfg, "config", "groupingConfig") is None assert safe.get_path(cfg, "config", "datascrubbingSettings", "scrubData") is not None assert safe.get_path(cfg, "config", "datascrubbingSettings", "scrubIpAddresses") is not None assert safe.get_path(cfg, "config", "piiConfig", "rules") is None assert safe.get_path(cfg, "config", "piiConfig", "applications") is None assert safe.get_path(cfg, "config", "quotas") is None
def get_tag(data, key): for k, v in get_path(data, "tags", filter=True): if k == key: return v
def serialize(self, obj, attrs, user): errors = [ EventError(error).get_api_context() for error in get_path(obj.data, 'errors', filter=True, default=()) # TODO(ja): Temporary workaround to hide certain normalization errors. # Remove this and the test in tests/sentry/api/serializers/test_event.py if self.should_display_error(error) ] (message, message_meta) = self._get_legacy_message_with_meta(obj) (tags, tags_meta) = self._get_tags_with_meta(obj) (context, context_meta) = self._get_attr_with_meta(obj, 'extra', {}) (packages, packages_meta) = self._get_attr_with_meta(obj, 'modules', {}) received = obj.data.get('received') if received: # Sentry at one point attempted to record invalid types here. # Remove after June 2 2016 try: received = datetime.utcfromtimestamp(received).replace( tzinfo=timezone.utc, ) except TypeError: received = None d = { 'id': six.text_type(obj.id), 'groupID': six.text_type(obj.group_id), 'eventID': six.text_type(obj.event_id), 'size': obj.size, 'entries': attrs['entries'], 'dist': obj.dist, # See GH-3248 'message': message, 'title': obj.title, 'location': obj.location, 'culprit': obj.culprit, 'user': attrs['user'], 'contexts': attrs['contexts'], 'crashFile': attrs['crash_file'], 'sdk': attrs['sdk'], # TODO(dcramer): move into contexts['extra'] 'context': context, 'packages': packages, 'type': obj.get_event_type(), 'metadata': obj.get_event_metadata(), 'tags': tags, 'platform': obj.platform, 'dateCreated': obj.datetime, 'dateReceived': received, 'errors': errors, 'fingerprints': obj.get_hashes(), '_meta': { 'entries': attrs['_meta']['entries'], 'message': message_meta, 'user': attrs['_meta']['user'], 'contexts': attrs['_meta']['contexts'], 'sdk': attrs['_meta']['sdk'], 'context': context_meta, 'packages': packages_meta, 'tags': tags_meta, }, } return d
def message(self): return (get_path(self.data, "logentry", "formatted") or get_path(self.data, "logentry", "message") or "")
def real_message(self): # XXX(mitsuhiko): this is a transitional attribute that should be # removed. `message` will be renamed to `search_message` and this # will become `message`. return (get_path(self.data, "logentry", "formatted") or get_path(self.data, "logentry", "message") or "")
def native_images_from_data(data): return get_path(data, "debug_meta", "images", default=(), filter=is_native_image)
def _localhost_filter(project_config, data): ip_address = get_path(data, 'user', 'ip_address') or '' url = get_path(data, 'request', 'url') or '' domain = urlparse(url).hostname return ip_address in _LOCAL_IPS or domain in _LOCAL_DOMAINS
def _get_next_page(response_json) -> str: """ Gets the next page url from a app store connect paged response """ return safe.get_path(response_json, "links", "next")
def test_internal_relays_should_receive_full_configs(call_endpoint, default_project, default_projectkey): result, status_code = call_endpoint(full_config=True) assert status_code < 400 # Sweeping assertion that we do not have any snake_case in that config. # Might need refining. assert not set(x for x in _get_all_keys(result) if "-" in x or "_" in x) cfg = safe.get_path(result, "configs", six.text_type(default_project.id)) assert safe.get_path(cfg, "disabled") is False (public_key, ) = cfg["publicKeys"] assert public_key["publicKey"] == default_projectkey.public_key assert public_key["isEnabled"] assert "quotas" in public_key assert safe.get_path(cfg, "slug") == default_project.slug last_change = safe.get_path(cfg, "lastChange") assert _date_regex.match(last_change) is not None last_fetch = safe.get_path(cfg, "lastFetch") assert _date_regex.match(last_fetch) is not None assert safe.get_path(cfg, "organizationId") == default_project.organization.id assert safe.get_path(cfg, "projectId") == default_project.id assert safe.get_path(cfg, "slug") == default_project.slug assert safe.get_path(cfg, "rev") is not None assert safe.get_path(cfg, "config", "trustedRelays") == [] assert safe.get_path(cfg, "config", "filterSettings") is not None assert safe.get_path(cfg, "config", "groupingConfig", "enhancements") is not None assert safe.get_path(cfg, "config", "groupingConfig", "id") is not None assert safe.get_path(cfg, "config", "piiConfig", "applications") is None assert safe.get_path(cfg, "config", "piiConfig", "rules") is None assert safe.get_path(cfg, "config", "datascrubbingSettings", "scrubData") is True assert safe.get_path(cfg, "config", "datascrubbingSettings", "scrubDefaults") is True assert safe.get_path(cfg, "config", "datascrubbingSettings", "scrubIpAddresses") is True assert safe.get_path(cfg, "config", "datascrubbingSettings", "sensitiveFields") == [] assert safe.get_path(cfg, "config", "quotas") == [] # Event retention depends on settings, so assert the actual value. Likely # `None` in dev, but must not be missing. assert cfg["config"]["eventRetention"] == quotas.get_event_retention( default_project.organization)