def test_get_path_list(self):
arr = [1, 2]
assert get_path(arr, 1) == 2
assert get_path(arr, -1) == 2
assert get_path(arr, 2) is None
assert get_path(arr, '1') is None
assert get_path([], 1) is None
def real_message(self):
# XXX(mitsuhiko): this is a transitional attribute that should be
# removed. `message` will be renamed to `search_message` and this
# will become `message`.
return get_path(self.data, 'logentry', 'formatted') \
or get_path(self.data, 'logentry', 'message') \
or ''
def signal_from_data(data):
exceptions = get_path(data, 'exception', 'values', filter=True)
signal = get_path(exceptions, 0, 'mechanism', 'meta', 'signal', 'number')
if signal is not None:
return int(signal)
return None
def _get_meta_header(self):
return 'OS Version: %s %s (%s)\nReport Version: %s' % (
get_path(self.context, 'os', 'name'),
get_path(self.context, 'os', 'version'),
get_path(self.context, 'os', 'build'),
REPORT_VERSION,
)
def get_frames(self, with_functions=False):
from sentry.stacktraces.functions import get_function_name_for_frame
if self._frames is None:
self._frames = []
def _push_frame(frame):
platform = frame.get('platform') or self.event.get('platform')
func = get_function_name_for_frame(frame, platform)
self._frames.append({
'function': func or '<unknown>',
'path': frame.get('abs_path') or frame.get('filename'),
'module': frame.get('module'),
'family': get_behavior_family_for_platform(platform),
'package': frame.get('package'),
})
have_errors = False
for exc in get_path(self.event, 'exception', 'values', filter=True) or ():
for frame in get_path(exc, 'stacktrace', 'frames', filter=True) or ():
_push_frame(frame)
have_errors = True
if not have_errors:
frames = get_path(self.event, 'stacktrace', 'frames', filter=True)
if not frames:
threads = get_path(self.event, 'threads', 'values', filter=True)
if threads and len(threads) == 1:
frames = get_path(threads, 0, 'stacktrace', 'frames')
for frame in frames or ():
_push_frame(frame)
return self._frames
def find_stacktraces_in_data(data, include_raw=False):
"""Finds all stracktraces in a given data blob and returns it
together with some meta information.
If `include_raw` is True, then also raw stacktraces are included.
"""
rv = []
def _report_stack(stacktrace, container):
if not stacktrace or not get_path(stacktrace, 'frames', filter=True):
return
platforms = set(
frame.get('platform') or data.get('platform')
for frame in get_path(stacktrace, 'frames', filter=True, default=())
)
rv.append(StacktraceInfo(stacktrace=stacktrace, container=container, platforms=platforms))
for exc in get_path(data, 'exception', 'values', filter=True, default=()):
_report_stack(exc.get('stacktrace'), exc)
_report_stack(data.get('stacktrace'), None)
for thread in get_path(data, 'threads', 'values', filter=True, default=()):
_report_stack(thread.get('stacktrace'), thread)
if include_raw:
for info in rv[:]:
if info.container is not None:
_report_stack(info.container.get('raw_stacktrace'), info.container)
return rv
def apply(self, data):
# TODO(dcramer): move this into each interface
if data.get('stacktrace'):
self.filter_stacktrace(data['stacktrace'])
for exc in get_path(data, 'exception', 'values', filter=True) or ():
if exc.get('stacktrace'):
self.filter_stacktrace(exc['stacktrace'])
for exc in get_path(data, 'threads', 'values', filter=True) or ():
if exc.get('stacktrace'):
self.filter_stacktrace(exc['stacktrace'])
for crumb in get_path(data, 'breadcrumbs', 'values', filter=True) or ():
self.filter_crumb(crumb)
if data.get('request'):
self.filter_http(data['request'])
if data.get('user'):
self.filter_user(data['user'])
if data.get('csp'):
self.filter_csp(data['csp'])
if data.get('extra'):
data['extra'] = varmap(self.sanitize, data['extra'])
if data.get('contexts'):
for key, value in six.iteritems(data['contexts']):
if value:
data['contexts'][key] = varmap(self.sanitize, value)
def get_thread_apple_string(self, thread_info):
rv = []
stacktrace = get_path(thread_info, 'stacktrace')
if stacktrace is None:
return None
if stacktrace:
frames = get_path(stacktrace, 'frames', filter=True)
if frames:
for i, frame in enumerate(reversed(frames)):
frame_string = self._convert_frame_to_apple_string(
frame=frame,
next=frames[len(frames) - i -
2] if i < len(frames) - 1 else None,
number=i
)
if frame_string is not None:
rv.append(frame_string)
if len(rv) == 0:
return None # No frames in thread, so we remove thread
is_exception = bool(thread_info.get('mechanism'))
thread_id = thread_info.get('id') or thread_info.get('thread_id') or '0'
thread_name = thread_info.get('name')
thread_name_string = ' name: %s' % (thread_name) if thread_name else ''
thread_crashed = thread_info.get('crashed') or is_exception
thread_crashed_thread = ' Crashed:' if thread_crashed else ''
thread_string = 'Thread %s%s%s\n' % (
thread_id, thread_name_string, thread_crashed_thread
)
return thread_string + '\n'.join(rv)
def _get_frame_paths(event):
data = event.data
frames = get_path(data, 'stacktrace', 'frames', filter=True)
if frames:
return frames
return get_path(data, 'exception', 'values', 0, 'stacktrace', 'frames', filter=True) or []
def find_best_instruction(self, processable_frame):
"""Given a frame, stacktrace info and frame index this returns the
interpolated instruction address we then use for symbolication later.
"""
if self.arch is None:
return parse_addr(processable_frame['instruction_addr'])
crashing_frame = False
signal = None
ip_reg = None
# We only need to provide meta information for frame zero
if processable_frame.idx == 0:
# The signal is useful information for symbolic in some situations
# to disambiugate the first frame. If we can get this information
# from the mechanism we want to pass it onwards.
exceptions = get_path(self.data, 'exception', 'values', filter=True)
signal = get_path(exceptions, 0, 'mechanism', 'meta', 'signal', 'number')
if signal is not None:
signal = int(signal)
registers = processable_frame.stacktrace_info.stacktrace.get('registers')
if registers:
ip_reg_name = arch_get_ip_reg_name(self.arch)
if ip_reg_name:
ip_reg = registers.get(ip_reg_name)
crashing_frame = True
return find_best_instruction(
processable_frame['instruction_addr'],
arch=self.arch,
crashing_frame=crashing_frame,
signal=signal,
ip_reg=ip_reg
)
def get_legacy_message(self):
# TODO(mitsuhiko): remove this code once it's unused. It's still
# being used by plugin code and once the message rename is through
# plugins should instead swithc to the actual message attribute or
# this method could return what currently is real_message.
return get_path(self.data, 'logentry', 'formatted') \
or get_path(self.data, 'logentry', 'message') \
or self.message
def get_sdk_from_event(event):
sdk_info = get_path(event, 'debug_meta', 'sdk_info')
if sdk_info:
return sdk_info
os = get_path(event, 'contexts', 'os')
if os and os.get('type') == 'os':
return get_sdk_from_os(os)
def to_python(cls, data, slim_frames=True, rust_renormalized=RUST_RENORMALIZED_DEFAULT):
if not rust_renormalized:
is_valid, errors = validate_and_default_interface(data, cls.path)
if not is_valid:
raise InterfaceValidationError("Invalid exception")
if not (data.get('type') or data.get('value')):
raise InterfaceValidationError("No 'type' or 'value' present")
if get_path(data, 'stacktrace', 'frames', filter=True):
stacktrace = Stacktrace.to_python(
data['stacktrace'],
slim_frames=slim_frames,
rust_renormalized=rust_renormalized
)
else:
stacktrace = None
if get_path(data, 'raw_stacktrace', 'frames', filter=True):
raw_stacktrace = Stacktrace.to_python(
data['raw_stacktrace'], slim_frames=slim_frames, raw=True,
rust_renormalized=rust_renormalized
)
else:
raw_stacktrace = None
type = data.get('type')
value = data.get('value')
if not rust_renormalized:
if isinstance(value, six.string_types):
if type is None:
m = _type_value_re.match(value)
if m:
type = m.group(1)
value = m.group(2).strip()
elif value is not None:
value = json.dumps(value)
value = trim(value, 4096)
if data.get('mechanism'):
mechanism = Mechanism.to_python(data['mechanism'],
rust_renormalized=rust_renormalized)
else:
mechanism = None
kwargs = {
'type': trim(type, 128),
'value': value,
'module': trim(data.get('module'), 128),
'mechanism': mechanism,
'stacktrace': stacktrace,
'thread_id': trim(data.get('thread_id'), 40),
'raw_stacktrace': raw_stacktrace,
}
return cls(**kwargs)
def _report_stack(stacktrace, container):
if not stacktrace or not get_path(stacktrace, 'frames', filter=True):
return
platforms = set(
frame.get('platform') or data.get('platform')
for frame in get_path(stacktrace, 'frames', filter=True, default=())
)
rv.append(StacktraceInfo(stacktrace=stacktrace, container=container, platforms=platforms))
def ip_address(self):
ip_address = get_path(self.data, 'user', 'ip_address')
if ip_address:
return ip_address
remote_addr = get_path(self.data, 'request', 'env', 'REMOTE_ADDR')
if remote_addr:
return remote_addr
return None
def has_sourcemap(event):
if event.platform not in ('javascript', 'node'):
return False
for exception in get_path(event.data, 'exception', 'values', filter=True, default=()):
for frame in get_path(exception, 'stacktrace', 'frames', filter=True, default=()):
if 'sourcemap' in (frame.get('data') or ()):
return True
return False
def get(self, request, project, event_id):
"""
Retrieve an Apple Crash Report from an event
`````````````````````````````````````````````
This endpoint returns the an apple crash report for a specific event.
The event ID is either the event as it appears in the Sentry database
or the event ID that is reported by the client upon submission.
This works only if the event.platform == cocoa
"""
use_snuba = options.get('snuba.events-queries.enabled')
event_cls = event_cls = SnubaEvent if use_snuba else Event
event = event_cls.objects.from_event_id(event_id, project_id=project.id)
if event is None:
raise ResourceDoesNotExist
Event.objects.bind_nodes([event], 'data')
if event.platform not in ('cocoa', 'native'):
return HttpResponse(
{
'message': 'Only cocoa events can return an apple crash report',
}, status=403
)
symbolicated = (request.GET.get('minified') not in ('1', 'true'))
apple_crash_report_string = six.text_type(
AppleCrashReport(
threads=get_path(event.data, 'threads', 'values', filter=True),
context=event.data.get('contexts'),
debug_images=get_path(event.data, 'debug_meta', 'images', filter=True),
exceptions=get_path(event.data, 'exception', 'values', filter=True),
symbolicated=symbolicated,
)
)
response = HttpResponse(apple_crash_report_string,
content_type='text/plain')
if request.GET.get('download') is not None:
filename = u"{}{}.crash".format(
event.event_id, symbolicated and '-symbolicated' or '')
response = StreamingHttpResponse(
apple_crash_report_string,
content_type='text/plain',
)
response['Content-Length'] = len(apple_crash_report_string)
response['Content-Disposition'] = 'attachment; filename="%s"' % filename
return response
def get_metadata(self):
message = strip(get_path(self.data, 'logentry', 'formatted') or
get_path(self.data, 'logentry', 'message'))
if message:
title = truncatechars(message.splitlines()[0], 100)
else:
title = '<unlabeled event>'
return {
'title': title,
}
def merge_symbolicator_minidump_response(data, response):
sdk_info = get_sdk_from_event(data)
# TODO(markus): Add OS context here when `merge_process_state_event` is no
# longer called for symbolicator projects
images = []
set_path(data, 'debug_meta', 'images', value=images)
for complete_image in response['modules']:
image = {}
merge_symbolicator_image(
image, complete_image, sdk_info,
lambda e: handle_symbolication_failed(e, data=data)
)
images.append(image)
data_threads = []
data['threads'] = {'values': data_threads}
data_exception = get_path(data, 'exception', 'values', 0)
for complete_stacktrace in response['stacktraces']:
is_requesting = complete_stacktrace.get('is_requesting')
thread_id = complete_stacktrace.get('thread_id')
data_thread = {
'id': thread_id,
'crashed': is_requesting,
}
data_threads.append(data_thread)
if is_requesting:
data_stacktrace = get_path(data_exception, 'stacktrace')
assert isinstance(data_stacktrace, dict), data_stacktrace
# Make exemption specifically for unreal portable callstacks
# TODO(markus): Allow overriding stacktrace more generically
# (without looking into unreal context) once we no longer parse
# minidump in the endpoint (right now we can't distinguish that
# from user json).
if data_stacktrace['frames'] and is_unreal_exception_stacktrace(data):
continue
del data_stacktrace['frames'][:]
else:
data_thread['stacktrace'] = data_stacktrace = {'frames': []}
if complete_stacktrace.get('registers'):
data_stacktrace['registers'] = complete_stacktrace['registers']
for complete_frame in reversed(complete_stacktrace['frames']):
new_frame = {}
merge_symbolicated_frame(new_frame, complete_frame)
data_stacktrace['frames'].append(new_frame)
def ensure_does_not_have_ip(self, data):
env = get_path(data, 'request', 'env')
if env:
env.pop('REMOTE_ADDR', None)
user = get_path(data, 'user')
if user:
user.pop('ip_address', None)
sdk = get_path(data, 'sdk')
if sdk:
sdk.pop('client_ip', None)
def translate_exception(data):
message = get_path(data, 'logentry', 'message')
if message:
data['logentry']['message'] = translate_message(message)
formatted = get_path(data, 'logentry', 'formatted')
if formatted:
data['logentry']['formatted'] = translate_message(formatted)
for entry in get_path(data, 'exception', 'values', filter=True, default=()):
if 'value' in entry:
entry['value'] = translate_message(entry['value'])
return data
def _get_legacy_message_with_meta(self, event):
meta = event.data.get('_meta')
message = get_path(event.data, 'logentry', 'formatted')
msg_meta = get_path(meta, 'logentry', 'formatted')
if not message:
message = get_path(event.data, 'logentry', 'message')
msg_meta = get_path(meta, 'logentry', 'message')
if not message:
message = event.message
msg_meta = None
return (message, meta_with_chunks(message, msg_meta))
def __init__(self, *args, **kwargs):
StacktraceProcessor.__init__(self, *args, **kwargs)
# If true, the project has been opted into using the symbolicator
# service for native symbolication, which also means symbolic is not
# used at all anymore.
# The (iOS) symbolserver is still used regardless of this value.
self.use_symbolicator = _is_symbolicator_enabled(self.project, self.data)
metrics.incr('native.use_symbolicator', tags={'value': self.use_symbolicator})
self.arch = cpu_name_from_data(self.data)
self.signal = signal_from_data(self.data)
self.sym = None
self.difs_referenced = set()
images = get_path(self.data, 'debug_meta', 'images', default=(),
filter=self._is_valid_image)
if images:
self.available = True
self.sdk_info = get_sdk_from_event(self.data)
self.object_lookup = ObjectLookup(images)
self.images = images
else:
self.available = False
def to_python(cls, data):
if data and 'values' not in data and 'exc_omitted' not in data:
data = {"values": [data]}
values = get_path(data, 'values', default=[])
if not isinstance(values, list):
raise InterfaceValidationError("Invalid value for 'values'")
kwargs = {
'values': [
v and SingleException.to_python(v, slim_frames=False)
for v in values
],
}
if data.get('exc_omitted'):
if len(data['exc_omitted']) != 2:
raise InterfaceValidationError("Invalid value for 'exc_omitted'")
kwargs['exc_omitted'] = data['exc_omitted']
else:
kwargs['exc_omitted'] = None
instance = cls(**kwargs)
# we want to wait to slim things til we've reconciled in_app
slim_exception_data(instance)
return instance
def get_user_agent(self, data):
try:
for key, value in get_path(data, 'request', 'headers', filter=True) or ():
if key.lower() == 'user-agent':
return value
except LookupError:
return ''
def primary_value_for_data(cls, data):
val = get_path(data, 'contexts', cls.type)
if val and val.get('type') == cls.type:
return val
rv = cls.values_for_data(data)
if len(rv) == 1:
return rv[0]
def iter_threads(self):
"""Returns an iterator over all threads of the process at the time of
the crash, including the crashing thread. The values are of type
``ThreadRef``."""
for thread in get_path(self.data, 'threads', 'values', filter=True, default=()):
if thread.get('crashed'):
# XXX: Assumes that the full list of threads is present in the
# original crash report. This is guaranteed by KSCrash and our
# minidump utility.
exceptions = get_path(self.data, 'exception', 'values', filter=True)
frames = get_path(exceptions, 0, 'stacktrace', 'frames')
else:
frames = get_path(thread, 'stacktrace', 'frames')
tid = thread.get('id')
if tid and frames:
yield tid, ThreadRef(frames, self.modules)
def full_url(self):
url = self.url
if url:
if self.query_string:
url = url + '?' + urlencode(get_path(self.query_string, filter=True))
if self.fragment:
url = url + '#' + self.fragment
return url
def get_stacktraces(self, data):
exceptions = get_path(data, 'exception', 'values', filter=True, default=())
stacktraces = [e['stacktrace'] for e in exceptions if e.get('stacktrace')]
if 'stacktrace' in data:
stacktraces.append(data['stacktrace'])
return [(s, Stacktrace.to_python(s)) for s in stacktraces]
def tags(self):
try:
rv = sorted([(t, v) for t, v in get_path(
self.data, 'tags', filter=True) or () if t is not None and v is not None])
return rv
except ValueError:
# at one point Sentry allowed invalid tag sets such as (foo, bar)
# vs ((tag, foo), (tag, bar))
return []
def get_related(data: JSONData,
relation: str) -> Union[None, JSONData]:
"""Returns related data by looking it up in all the related data included in the page.
This first looks up the related object in the data provided under the
`relationships` key. Then it uses the `type` and `id` of this key to look up
the actual data in `included_relations` which is an index of all related data
returned with the page.
If the `relation` does not exist in `data` then `None` is returned.
"""
rel_ptr_data = safe.get_path(data, "relationships", relation,
"data")
if rel_ptr_data is None:
# The query asks for both the appStoreVersion and preReleaseVersion
# relations to be included. However for each build there could be only one
# of these that will have the data with type and id, the other will have
# None for data.
return None
rel_type = rel_ptr_data["type"]
rel_id = rel_ptr_data["id"]
return included_relations[(rel_type, rel_id)]
def _send(self, project_id, _type, extra_data=(), asynchronous=True):
data = (self.EVENT_PROTOCOL_VERSION, _type) + extra_data
# TODO remove this once the unified dataset is available.
# Inserting into both events and transactions datasets lets us
# simulate what is currently happening via kafka when both the events
# and transactions consumers are running.
datasets = ["events"]
if get_path(extra_data, 0, "data", "type") == "transaction":
datasets.append("transactions")
try:
for dataset in datasets:
resp = snuba._snuba_pool.urlopen(
"POST",
"/tests/{}/eventstream".format(dataset),
body=json.dumps(data))
if resp.status != 200:
raise snuba.SnubaError("HTTP %s response from Snuba!" %
resp.status)
return resp
except urllib3.exceptions.HTTPError as err:
raise snuba.SnubaError(err)
def get_dsym_url(session: Session, app_id: str, bundle_short_version: str,
bundle_version: str, platform: str) -> Optional[str]:
"""
Returns the url for a dsyms bundle. The session must be logged in.
:return: The url to use for downloading the dsyms bundle
"""
with sentry_sdk.start_span(op="itunes-dsym-url",
description="Request iTunes dSYM download URL"):
details_url = (
f"https://appstoreconnect.apple.com/WebObjects/iTunesConnect.woa/ra/apps/"
f"{app_id}/platforms/{platform}/trains/{bundle_short_version}/builds/"
f"{bundle_version}/details")
logger.debug(f"GET {details_url}")
details_response = session.get(details_url)
# A non-OK status code will probably mean an expired token/session
if details_response.status_code == HTTPStatus.UNAUTHORIZED:
raise ITunesSessionExpiredException
if details_response.status_code == HTTPStatus.OK:
try:
data = details_response.json()
dsym_url: Optional[str] = safe.get_path(
data, "data", "dsymurl")
return dsym_url
except Exception as e:
logger.info(
"Could not obtain dSYM info for "
"app id=%s, bundle_short=%s, bundle=%s, platform=%s",
app_id,
bundle_short_version,
bundle_version,
platform,
exc_info=True,
)
raise e
return None
def cpu_name_from_data(data):
"""Returns the CPU name from the given data if it exists."""
device = DeviceContextType.primary_value_for_data(data)
if device and device.get('arch'):
return device['arch']
# TODO: kill this here. we want to not support that going forward
unique_cpu_name = None
for img in get_path(data, 'debug_meta', 'images', filter=True, default=()):
if img.get('arch') and arch_is_known(img['arch']):
cpu_name = img['arch']
elif img.get('cpu_type') is not None \
and img.get('cpu_subtype') is not None:
cpu_name = arch_from_macho(img['cpu_type'], img['cpu_subtype'])
else:
cpu_name = None
if unique_cpu_name is None:
unique_cpu_name = cpu_name
elif unique_cpu_name != cpu_name:
unique_cpu_name = None
break
return unique_cpu_name
def delete_old_primary_hash(event):
"""In case the primary hash changed during reprocessing, we need to tell
Snuba before reinserting the event. Snuba may then insert a tombstone row
depending on whether the primary_hash is part of the PK/sortkey or not.
Only when the primary_hash changed and is part of the sortkey, we need to
explicitly tombstone the old row.
If the primary_hash is not part of the PK/sortkey, or if the primary_hash
did not change, nothing needs to be done as ClickHouse's table merge will
merge the two rows together.
"""
old_primary_hash = get_path(event.data, "contexts", "reprocessing", "original_primary_hash")
if old_primary_hash is not None and old_primary_hash != event.get_primary_hash():
from sentry import eventstream
eventstream.tombstone_events_unsafe(
event.project_id,
[event.event_id],
old_primary_hash=old_primary_hash,
)
def normalize_mechanism_meta(mechanism, sdk_info=None):
meta = get_path(mechanism, 'meta')
if not meta:
return
sdk_name = sdk_info['sdk_name'].lower() if sdk_info else ''
if sdk_name in ('ios', 'watchos', 'tvos', 'macos'):
sdk = 'darwin'
elif sdk_name in ('linux', 'android'):
sdk = 'linux'
elif sdk_name in ('windows', ):
sdk = 'windows'
else:
sdk = None
if 'errno' in meta:
normalize_mechanism_errno(meta['errno'], sdk)
if 'signal' in meta:
normalize_mechanism_signal(meta['signal'], sdk)
if 'mach_exception' in meta:
normalize_mechanism_mach_exception(meta['mach_exception'])
def is_valid_error_message(project_config, message):
"""
Verify that an error message is not being filtered
for the given project.
"""
filtered_errors = get_path(project_config.config, "filter_settings",
FilterTypes.ERROR_MESSAGES, "patterns")
if not filtered_errors:
return True
message = force_text(message).lower()
for error in filtered_errors:
try:
if fnmatch.fnmatch(message, error.lower()):
return False
except Exception:
# fnmatch raises a string when the pattern is bad.
# Patterns come from end users and can be full of mistakes.
pass
return True
def to_python(cls, data, rust_renormalized=RUST_RENORMALIZED_DEFAULT):
if not rust_renormalized:
if data and 'values' not in data and 'exc_omitted' not in data:
data = {"values": [data]}
values = get_path(data, 'values', default=[])
if not rust_renormalized:
if not isinstance(values, list):
raise InterfaceValidationError("Invalid value for 'values'")
kwargs = {
'values': [
v and SingleException.to_python(
v, slim_frames=False, rust_renormalized=rust_renormalized)
for v in values
],
}
if not rust_renormalized:
if data.get('exc_omitted'):
if len(data['exc_omitted']) != 2:
raise InterfaceValidationError(
"Invalid value for 'exc_omitted'")
kwargs['exc_omitted'] = data['exc_omitted']
else:
kwargs['exc_omitted'] = None
else:
kwargs.setdefault('exc_omitted', None)
instance = cls(**kwargs)
if not rust_renormalized:
# we want to wait to slim things til we've reconciled in_app
slim_exception_data(instance)
return instance
def normalize_stacktraces_for_grouping(data, grouping_config=None):
def _has_system_frames(frames):
system_frames = 0
for frame in frames:
if not frame.get('in_app'):
system_frames += 1
return bool(system_frames) and len(frames) != system_frames
stacktraces = []
for stacktrace_info in find_stacktraces_in_data(data, include_raw=True):
frames = get_path(stacktrace_info.stacktrace,
'frames',
filter=True,
default=())
if frames:
stacktraces.append(frames)
if not stacktraces:
return
# If a grouping config is available, run grouping enhancers
if grouping_config is not None:
platform = data.get('platform')
for frames in stacktraces:
grouping_config.enhancements.apply_modifications_to_frame(
frames, platform)
# normalize in-app
for frames in stacktraces:
has_system_frames = _has_system_frames(frames)
for frame in frames:
if not has_system_frames:
frame['in_app'] = False
elif frame.get('in_app') is None:
frame['in_app'] = False
def normalize_stacktraces_for_grouping(data, grouping_config=None):
"""
Applies grouping enhancement rules and ensure in_app is set on all frames.
"""
stacktraces = []
for stacktrace_info in find_stacktraces_in_data(data, include_raw=True):
frames = get_path(stacktrace_info.stacktrace, 'frames', filter=True, default=())
if frames:
stacktraces.append(frames)
if not stacktraces:
return
# If a grouping config is available, run grouping enhancers
platform = data.get('platform')
if grouping_config is not None:
for frames in stacktraces:
grouping_config.enhancements.apply_modifications_to_frame(frames, platform)
# normalize in-app
for stacktrace in stacktraces:
_normalize_in_app(stacktrace, platform=platform)
def get_multiple_related(self, data: JSONData,
relation: str) -> Optional[List[JSONData]]:
"""Returns a list of all the related objects of the named relation type.
This is like :meth:`get_related` but is for relation types which have a list of
related objects instead of exactly one. An example of this is a ``build`` can have
multiple ``buildBundles`` related to it.
Having this as a separate method makes it easier to handle the type checking.
"""
rel_ptr_data = safe.get_path(data, "relationships", relation, "data")
if rel_ptr_data is None:
# Because the related information was requested in the query does not mean a
# relation of that type did exist.
return None
assert isinstance(rel_ptr_data, list)
all_related = []
for relationship in rel_ptr_data:
rel_type = _RelType(relationship["type"])
rel_id = _RelId(relationship["id"])
related_item = self._items[(rel_type, rel_id)]
if related_item:
all_related.append(related_item)
return all_related
def insert(self, group, event, is_new, is_sample, is_regression,
is_new_group_environment, primary_hash, skip_consume=False):
project = event.project
retention_days = quotas.get_event_retention(
organization=project.organization,
)
event_data = event.get_raw_data()
unexpected_tags = set([
k for (k, v) in (get_path(event_data, 'tags', filter=True) or [])
if k in self.UNEXPECTED_TAG_KEYS
])
if unexpected_tags:
logger.error('%r received unexpected tags: %r', self, unexpected_tags)
self._send(project.id, 'insert', extra_data=({
'group_id': event.group_id,
'event_id': event.event_id,
'organization_id': project.organization_id,
'project_id': event.project_id,
# TODO(mitsuhiko): We do not want to send this incorrect
# message but this is what snuba needs at the moment.
'message': event.message,
'platform': event.platform,
'datetime': event.datetime,
'data': event_data,
'primary_hash': primary_hash,
'retention_days': retention_days,
}, {
'is_new': is_new,
'is_sample': is_sample,
'is_regression': is_regression,
'is_new_group_environment': is_new_group_environment,
'skip_consume': skip_consume,
},))
def get_pre_release_version_info(session: Session,
credentials: AppConnectCredentials,
app_id: str):
"""
Get all prerelease builds version information for an application
The release build version information has the following structure:
platform: str - the platform for the build (e.g. IOS, MAC_OS ...)
short_version: str - the short version build info ( e.g. '1.0.1'), also called "train" in starship documentation
id: str - the IID of the version
versions: vec - a vector with builds
version: str - the version of the build (e.g. '101'), looks like the build number
id: str - the IID of the build
NOTE: the pre release version information is identical to the release version information
:return: a list of prerelease builds version information (see above)
"""
url = f"v1/apps/{app_id}/preReleaseVersions"
data = _get_appstore_info_paged_data(session, credentials, url)
result = []
for d in data:
versions = []
v = {
"platform": safe.get_path(d, "attributes", "platform"),
"short_version": safe.get_path(d, "attributes", "version"),
"id": safe.get_path(d, "id"),
"versions": versions,
}
builds_url = safe.get_path(d, "relationships", "builds", "links",
"related")
for build in _get_appstore_info_paged_data(session, credentials,
builds_url):
b = {
"version": safe.get_path(build, "attributes", "version"),
"id": safe.get_path(build, "id"),
}
versions.append(b)
result.append(v)
return result
def _get_exception_info(self):
rv = []
# We only have one exception at a time
exception = get_path(self.exceptions, 0)
if not exception:
return ''
mechanism = upgrade_legacy_mechanism(exception.get('mechanism')) or {}
mechanism_meta = get_path(mechanism, 'meta', default={})
signal = get_path(mechanism_meta, 'signal', 'name')
name = get_path(mechanism_meta, 'mach_exception', 'name')
if name or signal:
rv.append('Exception Type: %s%s' % (
name or 'Unknown',
signal and (' (%s)' % signal) or '',
))
exc_name = get_path(mechanism_meta, 'signal', 'code_name')
exc_addr = get_path(mechanism, 'data', 'relevant_address')
if exc_name:
rv.append('Exception Codes: %s%s' % (
exc_name,
exc_addr is not None and (' at %s' % exc_addr) or '',
))
if exception.get('thread_id') is not None:
rv.append('Crashed Thread: %s' % exception['thread_id'])
if exception.get('value'):
rv.append('\nApplication Specific Information:\n%s' %
exception['value'])
return '\n'.join(rv)
def _get_exception_info(self):
rv = []
# We only have one exception at a time
exception = get_path(self.exceptions, 0)
if not exception:
return ""
mechanism = upgrade_legacy_mechanism(exception.get("mechanism")) or {}
mechanism_meta = get_path(mechanism, "meta", default={})
signal = get_path(mechanism_meta, "signal", "name")
name = get_path(mechanism_meta, "mach_exception", "name")
if name or signal:
rv.append(
"Exception Type: {}{}".format(
name or "Unknown", signal and (" (%s)" % signal) or ""
)
)
exc_name = get_path(mechanism_meta, "signal", "code_name")
exc_addr = get_path(mechanism, "data", "relevant_address")
if exc_name:
rv.append(
"Exception Codes: %s%s"
% (exc_name, exc_addr is not None and (" at %s" % exc_addr) or "")
)
if exception.get("thread_id") is not None:
rv.append("Crashed Thread: %s" % exception["thread_id"])
if exception.get("value"):
rv.append("\nApplication Specific Information:\n%s" % exception["value"])
return "\n".join(rv)
def _localhost_filter(project_config, data):
ip_address = get_path(data, "user", "ip_address") or ""
url = get_path(data, "request", "url") or ""
domain = urlparse(url).hostname
return ip_address in _LOCAL_IPS or domain in _LOCAL_DOMAINS
def serialize(self, obj, attrs, user):
errors = [
EventError(error).get_api_context()
for error in get_path(obj.data, "errors", filter=True, default=())
# TODO(ja): Temporary workaround to hide certain normalization errors.
# Remove this and the test in tests/sentry/api/serializers/test_event.py
if self.should_display_error(error)
]
(message, message_meta) = self._get_legacy_message_with_meta(obj)
(tags, tags_meta) = get_tags_with_meta(obj)
(context, context_meta) = self._get_attr_with_meta(obj, "extra", {})
(packages,
packages_meta) = self._get_attr_with_meta(obj, "modules", {})
received = obj.data.get("received")
if received:
# Sentry at one point attempted to record invalid types here.
# Remove after June 2 2016
try:
received = datetime.utcfromtimestamp(received).replace(
tzinfo=timezone.utc)
except TypeError:
received = None
d = {
"id": obj.event_id,
"groupID": str(obj.group_id) if obj.group_id else None,
"eventID": obj.event_id,
"projectID": str(obj.project_id),
"size": obj.size,
"entries": attrs["entries"],
"dist": obj.dist,
# See GH-3248
"message": message,
"title": obj.title,
"location": obj.location,
"user": attrs["user"],
"contexts": attrs["contexts"],
"sdk": attrs["sdk"],
# TODO(dcramer): move into contexts['extra']
"context": context,
"packages": packages,
"type": obj.get_event_type(),
"metadata": obj.get_event_metadata(),
"tags": tags,
"platform": obj.platform,
"dateReceived": received,
"errors": errors,
"_meta": {
"entries": attrs["_meta"]["entries"],
"message": message_meta,
"user": attrs["_meta"]["user"],
"contexts": attrs["_meta"]["contexts"],
"sdk": attrs["_meta"]["sdk"],
"context": context_meta,
"packages": packages_meta,
"tags": tags_meta,
},
}
# Serialize attributes that are specific to different types of events.
if obj.get_event_type() == "transaction":
d.update(self.__serialize_transaction_attrs(attrs, obj))
else:
d.update(self.__serialize_error_attrs(attrs, obj))
return d
def _get_attr_with_meta(self, event, attr, default=None):
value = event.data.get(attr, default)
meta = get_path(event.data, "_meta", attr)
return (value, meta_with_chunks(value, meta))
def has_metadata(self, data):
exception = get_path(data, 'exception', 'values', -1)
return exception and any(v is not None for v in six.itervalues(exception))
def exceptions(self):
return get_path(self.values, filter=True)
def normalize(self, request_env=None):
request_env = request_env or {}
data = self.data
errors = data['errors'] = []
# Before validating with a schema, attempt to cast values to their desired types
# so that the schema doesn't have to take every type variation into account.
text = six.text_type
fp_types = six.string_types + six.integer_types + (float, )
def to_values(v):
return {'values': v} if v and isinstance(v, (tuple, list)) else v
def convert_fingerprint(values):
rv = values[:]
bad_float = False
for idx, item in enumerate(rv):
if isinstance(item, float) and \
(abs(item) >= (1 << 53) or int(item) != item):
bad_float = True
rv[idx] = text(item)
if bad_float:
metrics.incr(
'events.bad_float_fingerprint',
skip_internal=True,
tags={
'project_id': data.get('project'),
},
)
return rv
casts = {
'environment':
lambda v: text(v) if v is not None else v,
'fingerprint':
lambda v: convert_fingerprint(v)
if isinstance(v, list) and all(isinstance(f, fp_types)
for f in v) else v,
'release':
lambda v: text(v) if v is not None else v,
'dist':
lambda v: text(v).strip() if v is not None else v,
'time_spent':
lambda v: int(v) if v is not None else v,
'tags':
lambda v: [(text(v_k).replace(' ', '-').strip(), text(v_v).strip())
for (v_k, v_v) in dict(v).items()],
'timestamp':
lambda v: process_timestamp(v),
'platform':
lambda v: v if v in VALID_PLATFORMS else 'other',
'sentry.interfaces.Message':
lambda v: v if isinstance(v, dict) else {
'message': v
},
# These can be sent as lists and need to be converted to {'values': [...]}
'exception':
to_values,
'sentry.interfaces.Exception':
to_values,
'breadcrumbs':
to_values,
'sentry.interfaces.Breadcrumbs':
to_values,
'threads':
to_values,
'sentry.interfaces.Threads':
to_values,
}
for c in casts:
if c in data:
try:
data[c] = casts[c](data[c])
except InvalidTimestamp as it:
errors.append({
'type': it.args[0],
'name': c,
'value': data[c]
})
del data[c]
except Exception as e:
errors.append({
'type': EventError.INVALID_DATA,
'name': c,
'value': data[c]
})
del data[c]
# raw 'message' is coerced to the Message interface, as its used for pure index of
# searchable strings. If both a raw 'message' and a Message interface exist, try and
# add the former as the 'formatted' attribute of the latter.
# See GH-3248
msg_str = data.pop('message', None)
if msg_str:
msg_if = data.setdefault('sentry.interfaces.Message',
{'message': msg_str})
if msg_if.get('message') != msg_str:
msg_if.setdefault('formatted', msg_str)
# Fill in ip addresses marked as {{auto}}
client_ip = request_env.get('client_ip')
if client_ip:
if get_path(data, ['sentry.interfaces.Http', 'env', 'REMOTE_ADDR'
]) == '{{auto}}':
data['sentry.interfaces.Http']['env'][
'REMOTE_ADDR'] = client_ip
if get_path(data, ['request', 'env', 'REMOTE_ADDR']) == '{{auto}}':
data['request']['env']['REMOTE_ADDR'] = client_ip
if get_path(
data,
['sentry.interfaces.User', 'ip_address']) == '{{auto}}':
data['sentry.interfaces.User']['ip_address'] = client_ip
if get_path(data, ['user', 'ip_address']) == '{{auto}}':
data['user']['ip_address'] = client_ip
# Validate main event body and tags against schema
is_valid, event_errors = validate_and_default_interface(data, 'event')
errors.extend(event_errors)
if 'tags' in data:
is_valid, tag_errors = validate_and_default_interface(data['tags'],
'tags',
name='tags')
errors.extend(tag_errors)
# Validate interfaces
for k in list(iter(data)):
if k in CLIENT_RESERVED_ATTRS:
continue
value = data.pop(k)
if not value:
self.logger.debug('Ignored empty interface value: %s', k)
continue
try:
interface = get_interface(k)
except ValueError:
self.logger.debug('Ignored unknown attribute: %s', k)
errors.append({
'type': EventError.INVALID_ATTRIBUTE,
'name': k
})
continue
try:
inst = interface.to_python(value)
data[inst.get_path()] = inst.to_json()
except Exception as e:
log = self.logger.debug if isinstance(
e, InterfaceValidationError) else self.logger.error
log('Discarded invalid value for interface: %s (%r)',
k,
value,
exc_info=True)
errors.append({
'type': EventError.INVALID_DATA,
'name': k,
'value': value
})
# Additional data coercion and defaulting
level = data.get('level') or DEFAULT_LOG_LEVEL
if isinstance(level, int) or (isinstance(level, six.string_types)
and level.isdigit()):
level = LOG_LEVELS.get(int(level), DEFAULT_LOG_LEVEL)
data['level'] = LOG_LEVELS_MAP.get(level,
LOG_LEVELS_MAP[DEFAULT_LOG_LEVEL])
if data.get('dist') and not data.get('release'):
data['dist'] = None
timestamp = data.get('timestamp')
if not timestamp:
timestamp = timezone.now()
# TODO (alex) can this all be replaced by utcnow?
# it looks like the only time that this would even be hit is when timestamp
# is not defined, as the earlier process_timestamp already converts existing
# timestamps to floats.
if isinstance(timestamp, datetime):
# We must convert date to local time so Django doesn't mess it up
# based on TIME_ZONE
if settings.TIME_ZONE:
if not timezone.is_aware(timestamp):
timestamp = timestamp.replace(tzinfo=timezone.utc)
elif timezone.is_aware(timestamp):
timestamp = timestamp.replace(tzinfo=None)
timestamp = float(timestamp.strftime('%s'))
data['timestamp'] = timestamp
data['received'] = float(timezone.now().strftime('%s'))
data.setdefault('checksum', None)
data.setdefault('culprit', None)
data.setdefault('dist', None)
data.setdefault('environment', None)
data.setdefault('extra', {})
data.setdefault('fingerprint', None)
data.setdefault('logger', DEFAULT_LOGGER_NAME)
data.setdefault('platform', None)
data.setdefault('server_name', None)
data.setdefault('site', None)
data.setdefault('tags', [])
data.setdefault('transaction', None)
# Fix case where legacy apps pass 'environment' as a tag
# instead of a top level key.
# TODO (alex) save() just reinserts the environment into the tags
if not data.get('environment'):
tagsdict = dict(data['tags'])
if 'environment' in tagsdict:
data['environment'] = tagsdict['environment']
del tagsdict['environment']
data['tags'] = tagsdict.items()
# the SDKs currently do not describe event types, and we must infer
# them from available attributes
data['type'] = eventtypes.infer(data).key
data['version'] = self.version
exception = data.get('sentry.interfaces.Exception')
stacktrace = data.get('sentry.interfaces.Stacktrace')
if exception and len(exception['values']) == 1 and stacktrace:
exception['values'][0]['stacktrace'] = stacktrace
del data['sentry.interfaces.Stacktrace']
# Exception mechanism needs SDK information to resolve proper names in
# exception meta (such as signal names). "SDK Information" really means
# the operating system version the event was generated on. Some
# normalization still works without sdk_info, such as mach_exception
# names (they can only occur on macOS).
if exception:
sdk_info = get_sdk_from_event(data)
for ex in exception['values']:
if 'mechanism' in ex:
normalize_mechanism_meta(ex['mechanism'], sdk_info)
# If there is no User ip_addres, update it either from the Http interface
# or the client_ip of the request.
auth = request_env.get('auth')
is_public = auth and auth.is_public
add_ip_platforms = ('javascript', 'cocoa', 'objc')
http_ip = data.get('sentry.interfaces.Http',
{}).get('env', {}).get('REMOTE_ADDR')
if http_ip:
data.setdefault('sentry.interfaces.User',
{}).setdefault('ip_address', http_ip)
elif client_ip and (is_public
or data.get('platform') in add_ip_platforms):
data.setdefault('sentry.interfaces.User',
{}).setdefault('ip_address', client_ip)
# Trim values
data['logger'] = trim(data['logger'].strip(), 64)
trim_dict(data['extra'],
max_size=settings.SENTRY_MAX_EXTRA_VARIABLE_SIZE)
if data['culprit']:
data['culprit'] = trim(data['culprit'], MAX_CULPRIT_LENGTH)
if data['transaction']:
data['transaction'] = trim(data['transaction'], MAX_CULPRIT_LENGTH)
return data
def test_trusted_external_relays_should_receive_minimal_configs(
relay, add_org_key, call_endpoint, default_project,
default_projectkey):
relay.is_internal = False
relay.save()
result, status_code = call_endpoint(full_config=False)
assert status_code < 400
cfg = safe.get_path(result, "configs", six.text_type(default_project.id))
assert safe.get_path(cfg, "disabled") is False
(public_key, ) = cfg["publicKeys"]
assert public_key["publicKey"] == default_projectkey.public_key
assert public_key["numericId"] == default_projectkey.id
assert public_key["isEnabled"]
assert "quotas" not in public_key
assert safe.get_path(cfg, "slug") == default_project.slug
last_change = safe.get_path(cfg, "lastChange")
assert _date_regex.match(last_change) is not None
last_fetch = safe.get_path(cfg, "lastFetch")
assert _date_regex.match(last_fetch) is not None
assert safe.get_path(cfg,
"organizationId") == default_project.organization.id
assert safe.get_path(cfg, "projectId") == default_project.id
assert safe.get_path(cfg, "slug") == default_project.slug
assert safe.get_path(cfg, "rev") is not None
assert safe.get_path(cfg, "config", "trustedRelays") == [relay.public_key]
assert safe.get_path(cfg, "config", "filterSettings") is None
assert safe.get_path(cfg, "config", "groupingConfig") is None
assert safe.get_path(cfg, "config", "datascrubbingSettings",
"scrubData") is not None
assert safe.get_path(cfg, "config", "datascrubbingSettings",
"scrubIpAddresses") is not None
assert safe.get_path(cfg, "config", "piiConfig", "rules") is None
assert safe.get_path(cfg, "config", "piiConfig", "applications") is None
assert safe.get_path(cfg, "config", "quotas") is None
def get_tag(data, key):
for k, v in get_path(data, "tags", filter=True):
if k == key:
return v
def serialize(self, obj, attrs, user):
errors = [
EventError(error).get_api_context()
for error in get_path(obj.data, 'errors', filter=True, default=())
# TODO(ja): Temporary workaround to hide certain normalization errors.
# Remove this and the test in tests/sentry/api/serializers/test_event.py
if self.should_display_error(error)
]
(message, message_meta) = self._get_legacy_message_with_meta(obj)
(tags, tags_meta) = self._get_tags_with_meta(obj)
(context, context_meta) = self._get_attr_with_meta(obj, 'extra', {})
(packages,
packages_meta) = self._get_attr_with_meta(obj, 'modules', {})
received = obj.data.get('received')
if received:
# Sentry at one point attempted to record invalid types here.
# Remove after June 2 2016
try:
received = datetime.utcfromtimestamp(received).replace(
tzinfo=timezone.utc, )
except TypeError:
received = None
d = {
'id': six.text_type(obj.id),
'groupID': six.text_type(obj.group_id),
'eventID': six.text_type(obj.event_id),
'size': obj.size,
'entries': attrs['entries'],
'dist': obj.dist,
# See GH-3248
'message': message,
'title': obj.title,
'location': obj.location,
'culprit': obj.culprit,
'user': attrs['user'],
'contexts': attrs['contexts'],
'crashFile': attrs['crash_file'],
'sdk': attrs['sdk'],
# TODO(dcramer): move into contexts['extra']
'context': context,
'packages': packages,
'type': obj.get_event_type(),
'metadata': obj.get_event_metadata(),
'tags': tags,
'platform': obj.platform,
'dateCreated': obj.datetime,
'dateReceived': received,
'errors': errors,
'fingerprints': obj.get_hashes(),
'_meta': {
'entries': attrs['_meta']['entries'],
'message': message_meta,
'user': attrs['_meta']['user'],
'contexts': attrs['_meta']['contexts'],
'sdk': attrs['_meta']['sdk'],
'context': context_meta,
'packages': packages_meta,
'tags': tags_meta,
},
}
return d
def message(self):
return (get_path(self.data, "logentry", "formatted")
or get_path(self.data, "logentry", "message") or "")
def real_message(self):
# XXX(mitsuhiko): this is a transitional attribute that should be
# removed. `message` will be renamed to `search_message` and this
# will become `message`.
return (get_path(self.data, "logentry", "formatted")
or get_path(self.data, "logentry", "message") or "")
def native_images_from_data(data):
return get_path(data,
"debug_meta",
"images",
default=(),
filter=is_native_image)
def _localhost_filter(project_config, data):
ip_address = get_path(data, 'user', 'ip_address') or ''
url = get_path(data, 'request', 'url') or ''
domain = urlparse(url).hostname
return ip_address in _LOCAL_IPS or domain in _LOCAL_DOMAINS
def _get_next_page(response_json) -> str:
"""
Gets the next page url from a app store connect paged response
"""
return safe.get_path(response_json, "links", "next")
def test_internal_relays_should_receive_full_configs(call_endpoint,
default_project,
default_projectkey):
result, status_code = call_endpoint(full_config=True)
assert status_code < 400
# Sweeping assertion that we do not have any snake_case in that config.
# Might need refining.
assert not set(x for x in _get_all_keys(result) if "-" in x or "_" in x)
cfg = safe.get_path(result, "configs", six.text_type(default_project.id))
assert safe.get_path(cfg, "disabled") is False
(public_key, ) = cfg["publicKeys"]
assert public_key["publicKey"] == default_projectkey.public_key
assert public_key["isEnabled"]
assert "quotas" in public_key
assert safe.get_path(cfg, "slug") == default_project.slug
last_change = safe.get_path(cfg, "lastChange")
assert _date_regex.match(last_change) is not None
last_fetch = safe.get_path(cfg, "lastFetch")
assert _date_regex.match(last_fetch) is not None
assert safe.get_path(cfg,
"organizationId") == default_project.organization.id
assert safe.get_path(cfg, "projectId") == default_project.id
assert safe.get_path(cfg, "slug") == default_project.slug
assert safe.get_path(cfg, "rev") is not None
assert safe.get_path(cfg, "config", "trustedRelays") == []
assert safe.get_path(cfg, "config", "filterSettings") is not None
assert safe.get_path(cfg, "config", "groupingConfig",
"enhancements") is not None
assert safe.get_path(cfg, "config", "groupingConfig", "id") is not None
assert safe.get_path(cfg, "config", "piiConfig", "applications") is None
assert safe.get_path(cfg, "config", "piiConfig", "rules") is None
assert safe.get_path(cfg, "config", "datascrubbingSettings",
"scrubData") is True
assert safe.get_path(cfg, "config", "datascrubbingSettings",
"scrubDefaults") is True
assert safe.get_path(cfg, "config", "datascrubbingSettings",
"scrubIpAddresses") is True
assert safe.get_path(cfg, "config", "datascrubbingSettings",
"sensitiveFields") == []
assert safe.get_path(cfg, "config", "quotas") == []
# Event retention depends on settings, so assert the actual value. Likely
# `None` in dev, but must not be missing.
assert cfg["config"]["eventRetention"] == quotas.get_event_retention(
default_project.organization)
