class NativeStacktraceProcessor(StacktraceProcessor):
supported_platforms = ('cocoa', 'native')
supported_images = ('apple', 'symbolic')
def __init__(self, *args, **kwargs):
StacktraceProcessor.__init__(self, *args, **kwargs)
debug_meta = self.data.get('debug_meta')
self.arch = cpu_name_from_data(self.data)
self.sym = None
self.difs_referenced = set()
if debug_meta:
self.available = True
self.debug_meta = debug_meta
self.sdk_info = get_sdk_from_event(self.data)
self.object_lookup = ObjectLookup([
img for img in self.debug_meta['images']
if img['type'] in self.supported_images
])
else:
self.available = False
def close(self):
StacktraceProcessor.close(self)
if self.difs_referenced:
metrics.incr(
'dsyms.processed',
amount=len(self.difs_referenced),
skip_internal=True,
tags={
'project_id': self.project.id,
},
)
def find_best_instruction(self, processable_frame):
"""Given a frame, stacktrace info and frame index this returns the
interpolated instruction address we then use for symbolication later.
"""
if self.arch is None:
return parse_addr(processable_frame['instruction_addr'])
crashing_frame = False
signal = None
ip_reg = None
# We only need to provide meta information for frame zero
if processable_frame.idx == 0:
# The signal is useful information for symbolic in some situations
# to disambiugate the first frame. If we can get this information
# from the mechanism we want to pass it onwards.
signal = None
exc = self.data.get('sentry.interfaces.Exception')
if exc is not None:
mechanism = exc['values'][0].get('mechanism')
if mechanism and 'meta' in mechanism and \
'signal' in mechanism['meta'] and \
'number' in mechanism['meta']['signal']:
signal = int(mechanism['meta']['signal']['number'])
registers = processable_frame.stacktrace_info.stacktrace.get(
'registers')
if registers:
ip_reg_name = arch_get_ip_reg_name(self.arch)
if ip_reg_name:
ip_reg = registers.get(ip_reg_name)
crashing_frame = True
return find_best_instruction(processable_frame['instruction_addr'],
arch=self.arch,
crashing_frame=crashing_frame,
signal=signal,
ip_reg=ip_reg)
def handles_frame(self, frame, stacktrace_info):
platform = frame.get('platform') or self.data.get('platform')
return (platform in self.supported_platforms and self.available
and 'instruction_addr' in frame)
def preprocess_frame(self, processable_frame):
instr_addr = self.find_best_instruction(processable_frame)
obj = self.object_lookup.find_object(instr_addr)
processable_frame.data = {
'instruction_addr': instr_addr,
'obj': obj,
'debug_id': obj.id if obj is not None else None,
'symbolserver_match': None,
}
if obj is not None:
processable_frame.set_cache_key_from_values((
FRAME_CACHE_VERSION,
# Because the images can move around, we want to rebase
# the address for the cache key to be within the image
# the same way as we do it in the symbolizer.
rebase_addr(instr_addr, obj),
obj.id,
obj.arch,
obj.size,
))
def preprocess_step(self, processing_task):
if not self.available:
return False
referenced_images = set(
pf.data['debug_id']
for pf in processing_task.iter_processable_frames(self)
if pf.cache_value is None and pf.data['debug_id'] is not None)
app_info = version_build_from_data(self.data)
if app_info is not None:
def on_referenced(dif):
dsym_app = DSymApp.objects.create_or_update_app(
sync_id=None,
app_id=app_info.id,
project=self.project,
data={'name': app_info.name},
platform=DifPlatform.APPLE,
no_fetch=True)
try:
with transaction.atomic():
version_dsym_file, created = VersionDSymFile.objects.get_or_create(
dsym_file=dif,
version=app_info.version,
build=app_info.build,
defaults=dict(dsym_app=dsym_app),
)
except IntegrityError:
# XXX: this can currently happen because we only
# support one app per debug file. Since this can
# happen in some cases anyways we ignore it.
pass
else:
on_referenced = None
self.sym = Symbolizer(self.project,
self.object_lookup,
referenced_images=referenced_images,
on_dif_referenced=on_referenced)
if options.get('symbolserver.enabled'):
self.fetch_system_symbols(processing_task)
def fetch_system_symbols(self, processing_task):
to_lookup = []
pf_list = []
for pf in processing_task.iter_processable_frames(self):
obj = pf.data['obj']
if pf.cache_value is not None or obj is None or \
self.sym.is_image_from_app_bundle(obj):
continue
# We can only look up objects in the symbol server that have a
# uuid. If we encounter things with an age appended or
# similar we need to skip.
try:
uuid.UUID(obj.id)
except (ValueError, TypeError):
continue
to_lookup.append({
'object_uuid':
obj.id,
'object_name':
obj.name or '<unknown>',
'addr':
'0x%x' % rebase_addr(pf.data['instruction_addr'], obj)
})
pf_list.append(pf)
if not to_lookup:
return
rv = lookup_system_symbols(to_lookup, self.sdk_info, self.arch)
if rv is not None:
for symrv, pf in zip(rv, pf_list):
if symrv is None:
continue
pf.data['symbolserver_match'] = symrv
def process_frame(self, processable_frame, processing_task):
frame = processable_frame.frame
raw_frame = dict(frame)
errors = []
if processable_frame.cache_value is None:
# Construct a raw frame that is used by the symbolizer
# backend. We only assemble the bare minimum we need here.
instruction_addr = processable_frame.data['instruction_addr']
in_app = self.sym.is_in_app(instruction_addr,
sdk_info=self.sdk_info)
if in_app and raw_frame.get('function') is not None:
in_app = not self.sym.is_internal_function(
raw_frame['function'])
if raw_frame.get('in_app') is None:
raw_frame['in_app'] = in_app
debug_id = processable_frame.data['debug_id']
if debug_id is not None:
self.difs_referenced.add(debug_id)
try:
symbolicated_frames = self.sym.symbolize_frame(
instruction_addr,
self.sdk_info,
symbolserver_match=processable_frame.
data['symbolserver_match'])
if not symbolicated_frames:
return None, [raw_frame], []
except SymbolicationFailed as e:
# User fixable but fatal errors are reported as processing
# issues
if e.is_user_fixable and e.is_fatal:
report_processing_issue(self.data,
scope='native',
object='dsym:%s' % e.image_uuid,
type=e.type,
data=e.get_data())
# This in many ways currently does not really do anything.
# The reason is that once a processing issue is reported
# the event will only be stored as a raw event and no
# group will be generated. As a result it also means that
# we will not have any user facing event or error showing
# up at all. We want to keep this here though in case we
# do not want to report some processing issues (eg:
# optional difs)
errors = []
if e.is_user_fixable or e.is_sdk_failure:
errors.append(e.get_data())
else:
logger.debug('Failed to symbolicate with native backend',
exc_info=True)
return [raw_frame], [raw_frame], errors
processable_frame.set_cache_value([in_app, symbolicated_frames])
else: # processable_frame.cache_value is present
in_app, symbolicated_frames = processable_frame.cache_value
raw_frame['in_app'] = in_app
new_frames = []
for sfrm in symbolicated_frames:
new_frame = dict(frame)
new_frame['function'] = sfrm['function']
if sfrm.get('symbol'):
new_frame['symbol'] = sfrm['symbol']
new_frame['abs_path'] = sfrm['abs_path']
new_frame['filename'] = sfrm.get('filename') or \
(sfrm['abs_path'] and posixpath.basename(sfrm['abs_path'])) or \
None
if sfrm.get('lineno'):
new_frame['lineno'] = sfrm['lineno']
if sfrm.get('colno'):
new_frame['colno'] = sfrm['colno']
if sfrm.get(
'package') or processable_frame.data['obj'] is not None:
new_frame['package'] = sfrm.get(
'package', processable_frame.data['obj'].name)
if new_frame.get('in_app') is None:
new_frame['in_app'] = in_app and \
not self.sym.is_internal_function(new_frame['function'])
new_frames.append(new_frame)
return new_frames, [raw_frame], []
class NativeStacktraceProcessor(StacktraceProcessor):
def __init__(self, *args, **kwargs):
StacktraceProcessor.__init__(self, *args, **kwargs)
debug_meta = self.data.get('debug_meta')
self.sym = None
if debug_meta:
self.available = True
self.debug_meta = debug_meta
self.sdk_info = get_sdk_from_event(self.data)
else:
self.available = False
def close(self):
StacktraceProcessor.close(self)
if self.sym is not None:
self.sym.close()
self.sym = None
def preprocess_related_data(self):
if not self.available:
return False
is_debug_build = self.debug_meta.get('is_debug_build')
referenced_images = find_stacktrace_referenced_images(
self.debug_meta['images'], [
x.stacktrace for x in self.stacktrace_infos])
self.sym = Symbolizer(self.project, self.debug_meta['images'],
referenced_images=referenced_images,
is_debug_build=is_debug_build)
# The symbolizer gets a reference to the debug meta's images so
# when it resolves the missing vmaddrs it changes them in the data
# dict.
return self.sym.resolve_missing_vmaddrs()
def process_frame(self, frame):
# XXX: warn on missing availability?
# Only process frames here that are of supported platforms and
# have the mandatory requirements for
if not self.available or \
self.get_effective_platform(frame) != 'cocoa' or \
'image_addr' not in frame or \
'instruction_addr' not in frame or \
'symbol_addr' not in frame:
return None
errors = []
# Construct a raw frame that is used by the symbolizer
# backend.
sym_frame = {
'object_name': frame.get('package'),
'object_addr': frame['image_addr'],
'instruction_addr': frame['instruction_addr'],
'symbol_name': frame.get('function'),
'symbol_addr': frame['symbol_addr'],
}
new_frame = dict(frame)
raw_frame = dict(frame)
try:
sfrm = self.sym.symbolize_frame(sym_frame, self.sdk_info)
except SymbolicationFailed as e:
if e.is_user_fixable or e.is_sdk_failure:
errors.append({
'type': EventError.NATIVE_INTERNAL_FAILURE,
'image_uuid': e.image_uuid,
'image_path': e.image_path,
'image_arch': e.image_arch,
'message': e.message,
})
else:
logger.debug('Failed to symbolicate with native backend',
exc_info=True)
else:
symbol = sfrm.get('symbol_name') or \
new_frame.get('function') or '<unknown>'
function = demangle_symbol(symbol, simplified=True)
new_frame['function'] = function
# If we demangled something, store the original in the
# symbol portion of the frame
if function != symbol:
new_frame['symbol'] = symbol
new_frame['abs_path'] = sfrm.get('filename') or None
if new_frame['abs_path']:
new_frame['filename'] = posixpath.basename(
new_frame['abs_path'])
if sfrm.get('line') is not None:
new_frame['lineno'] = sfrm['line']
else:
new_frame['instruction_offset'] = \
parse_addr(sfrm['instruction_addr']) - \
parse_addr(sfrm['symbol_addr'])
if sfrm.get('column') is not None:
new_frame['colno'] = sfrm['column']
new_frame['package'] = sfrm['object_name'] \
or new_frame.get('package')
new_frame['symbol_addr'] = '0x%x' % \
parse_addr(sfrm['symbol_addr'])
new_frame['instruction_addr'] = '0x%x' % parse_addr(
sfrm['instruction_addr'])
in_app = self.sym.is_in_app(sym_frame)
new_frame['in_app'] = raw_frame['in_app'] = in_app
return [new_frame], [raw_frame], errors
class NativeStacktraceProcessor(StacktraceProcessor):
supported_platforms = ('cocoa', 'native')
supported_images = ('apple', 'symbolic')
def __init__(self, *args, **kwargs):
StacktraceProcessor.__init__(self, *args, **kwargs)
debug_meta = self.data.get('debug_meta')
self.arch = cpu_name_from_data(self.data)
self.sym = None
self.dsyms_referenced = set()
if debug_meta:
self.available = True
self.debug_meta = debug_meta
self.sdk_info = get_sdk_from_event(self.data)
self.object_lookup = ObjectLookup(
[img for img in self.debug_meta['images'] if img['type'] in self.supported_images]
)
else:
self.available = False
def close(self):
StacktraceProcessor.close(self)
if self.dsyms_referenced:
metrics.incr(
'dsyms.processed',
amount=len(self.dsyms_referenced),
skip_internal=True,
tags={
'project_id': self.project.id,
},
)
def find_best_instruction(self, processable_frame):
"""Given a frame, stacktrace info and frame index this returns the
interpolated instruction address we then use for symbolication later.
"""
if self.arch is None:
return parse_addr(processable_frame['instruction_addr'])
crashing_frame = False
signal = None
ip_reg = None
# We only need to provide meta information for frame zero
if processable_frame.idx == 0:
# The signal is useful information for symbolic in some situations
# to disambiugate the first frame. If we can get this information
# from the mechanism we want to pass it onwards.
signal = None
exc = self.data.get('sentry.interfaces.Exception')
if exc is not None:
mechanism = exc['values'][0].get('mechanism')
if mechanism and 'meta' in mechanism and \
'signal' in mechanism['meta'] and \
'number' in mechanism['meta']['signal']:
signal = int(mechanism['meta']['signal']['number'])
registers = processable_frame.stacktrace_info.stacktrace.get(
'registers')
if registers:
ip_reg_name = arch_get_ip_reg_name(self.arch)
if ip_reg_name:
ip_reg = registers.get(ip_reg_name)
crashing_frame = True
return find_best_instruction(
processable_frame['instruction_addr'],
arch=self.arch,
crashing_frame=crashing_frame,
signal=signal,
ip_reg=ip_reg
)
def handles_frame(self, frame, stacktrace_info):
platform = frame.get('platform') or self.data.get('platform')
return (platform in self.supported_platforms and self.available
and 'instruction_addr' in frame)
def preprocess_frame(self, processable_frame):
instr_addr = self.find_best_instruction(processable_frame)
obj = self.object_lookup.find_object(instr_addr)
processable_frame.data = {
'instruction_addr': instr_addr,
'obj': obj,
'obj_uuid': obj.id if obj is not None else None,
'symbolserver_match': None,
}
if obj is not None:
processable_frame.set_cache_key_from_values(
(
FRAME_CACHE_VERSION,
# Because the images can move around, we want to rebase
# the address for the cache key to be within the image
# the same way as we do it in the symbolizer.
rebase_addr(instr_addr, obj),
obj.id,
obj.arch,
obj.size,
)
)
def preprocess_step(self, processing_task):
if not self.available:
return False
referenced_images = set(
pf.data['obj_uuid'] for pf in processing_task.iter_processable_frames(self)
if pf.cache_value is None and pf.data['obj_uuid'] is not None
)
app_info = version_build_from_data(self.data)
if app_info is not None:
def on_referenced(dsym_file):
dsym_app = DSymApp.objects.create_or_update_app(
sync_id=None,
app_id=app_info.id,
project=self.project,
data={'name': app_info.name},
platform=DSymPlatform.APPLE,
no_fetch=True
)
try:
with transaction.atomic():
version_dsym_file, created = VersionDSymFile.objects.get_or_create(
dsym_file=dsym_file,
version=app_info.version,
build=app_info.build,
defaults=dict(dsym_app=dsym_app),
)
except IntegrityError:
# XXX: this can currently happen because we only
# support one app per dsym file. Since this can
# happen in some cases anyways we ignore it.
pass
else:
on_referenced = None
self.sym = Symbolizer(
self.project,
self.object_lookup,
referenced_images=referenced_images,
on_dsym_file_referenced=on_referenced
)
if options.get('symbolserver.enabled'):
self.fetch_system_symbols(processing_task)
def fetch_system_symbols(self, processing_task):
to_lookup = []
pf_list = []
for pf in processing_task.iter_processable_frames(self):
obj = pf.data['obj']
if pf.cache_value is not None or obj is None or \
self.sym.is_image_from_app_bundle(obj):
continue
# We can only look up objects in the symbol server that have a
# uuid. If we encounter things with an age appended or
# similar we need to skip.
try:
uuid.UUID(obj.id)
except (ValueError, TypeError):
continue
to_lookup.append(
{
'object_uuid': obj.id,
'object_name': obj.name or '<unknown>',
'addr': '0x%x' % rebase_addr(pf.data['instruction_addr'], obj)
}
)
pf_list.append(pf)
if not to_lookup:
return
rv = lookup_system_symbols(to_lookup, self.sdk_info, self.arch)
if rv is not None:
for symrv, pf in zip(rv, pf_list):
if symrv is None:
continue
pf.data['symbolserver_match'] = symrv
def process_frame(self, processable_frame, processing_task):
frame = processable_frame.frame
raw_frame = dict(frame)
errors = []
if processable_frame.cache_value is None:
# Construct a raw frame that is used by the symbolizer
# backend. We only assemble the bare minimum we need here.
instruction_addr = processable_frame.data['instruction_addr']
in_app = self.sym.is_in_app(
instruction_addr,
sdk_info=self.sdk_info
)
if in_app and raw_frame.get('function') is not None:
in_app = not self.sym.is_internal_function(
raw_frame['function'])
if raw_frame.get('in_app') is None:
raw_frame['in_app'] = in_app
obj_uuid = processable_frame.data['obj_uuid']
if obj_uuid is not None:
self.dsyms_referenced.add(obj_uuid)
try:
symbolicated_frames = self.sym.symbolize_frame(
instruction_addr,
self.sdk_info,
symbolserver_match=processable_frame.data['symbolserver_match']
)
if not symbolicated_frames:
return None, [raw_frame], []
except SymbolicationFailed as e:
# User fixable but fatal errors are reported as processing
# issues
if e.is_user_fixable and e.is_fatal:
report_processing_issue(
self.data,
scope='native',
object='dsym:%s' % e.image_uuid,
type=e.type,
data=e.get_data()
)
# This in many ways currently does not really do anything.
# The reason is that once a processing issue is reported
# the event will only be stored as a raw event and no
# group will be generated. As a result it also means that
# we will not have any user facing event or error showing
# up at all. We want to keep this here though in case we
# do not want to report some processing issues (eg:
# optional dsyms)
errors = []
if e.is_user_fixable or e.is_sdk_failure:
errors.append(e.get_data())
else:
logger.debug('Failed to symbolicate with native backend',
exc_info=True)
return [raw_frame], [raw_frame], errors
processable_frame.set_cache_value([in_app, symbolicated_frames])
else: # processable_frame.cache_value is present
in_app, symbolicated_frames = processable_frame.cache_value
raw_frame['in_app'] = in_app
new_frames = []
for sfrm in symbolicated_frames:
new_frame = dict(frame)
new_frame['function'] = sfrm['function']
if sfrm.get('symbol'):
new_frame['symbol'] = sfrm['symbol']
new_frame['abs_path'] = sfrm['abs_path']
new_frame['filename'] = sfrm.get('filename') or \
(sfrm['abs_path'] and posixpath.basename(sfrm['abs_path'])) or \
None
if sfrm.get('lineno'):
new_frame['lineno'] = sfrm['lineno']
if sfrm.get('colno'):
new_frame['colno'] = sfrm['colno']
if sfrm.get('package') or processable_frame.data['obj'] is not None:
new_frame['package'] = sfrm.get(
'package', processable_frame.data['obj'].name)
if new_frame.get('in_app') is None:
new_frame['in_app'] = in_app and \
not self.sym.is_internal_function(new_frame['function'])
new_frames.append(new_frame)
return new_frames, [raw_frame], []
class NativeStacktraceProcessor(StacktraceProcessor):
supported_platforms = ('cocoa', 'native')
# TODO(ja): Clean up all uses of image type "apple", "uuid", "id" and "name"
supported_images = ('apple', 'symbolic', 'elf', 'macho', 'pe')
def __init__(self, *args, **kwargs):
StacktraceProcessor.__init__(self, *args, **kwargs)
# If true, the project has been opted into using the symbolicator
# service for native symbolication, which also means symbolic is not
# used at all anymore.
# The (iOS) symbolserver is still used regardless of this value.
self.use_symbolicator = _is_symbolicator_enabled(
self.project, self.data)
metrics.incr('native.use_symbolicator',
tags={'value': self.use_symbolicator})
self.arch = cpu_name_from_data(self.data)
self.signal = signal_from_data(self.data)
self.sym = None
self.difs_referenced = set()
images = get_path(self.data,
'debug_meta',
'images',
default=(),
filter=self._is_valid_image)
if images:
self.available = True
self.sdk_info = get_sdk_from_event(self.data)
self.object_lookup = ObjectLookup(images)
self.images = images
else:
self.available = False
def _is_valid_image(self, image):
# TODO(ja): Deprecate this. The symbolicator should take care of
# filtering valid images.
return bool(image) \
and image.get('type') in self.supported_images \
and image.get('image_addr') is not None \
and image.get('image_size') is not None \
and (image.get('debug_id') or image.get('id') or image.get('uuid')) is not None
def close(self):
StacktraceProcessor.close(self)
if self.difs_referenced:
metrics.incr(
'dsyms.processed',
amount=len(self.difs_referenced),
skip_internal=True,
tags={
'project_id': self.project.id,
},
)
def find_best_instruction(self, processable_frame):
"""Given a frame, stacktrace info and frame index this returns the
interpolated instruction address we then use for symbolication later.
"""
if self.arch is None:
return parse_addr(processable_frame['instruction_addr'])
crashing_frame = False
signal = None
ip_reg = None
# We only need to provide meta information for frame zero
if processable_frame.idx == 0:
# The signal is useful information for symbolic in some situations
# to disambiugate the first frame. If we can get this information
# from the mechanism we want to pass it onwards.
signal = self.signal
registers = processable_frame.stacktrace_info.stacktrace.get(
'registers')
if registers:
ip_reg_name = arch_get_ip_reg_name(self.arch)
if ip_reg_name:
ip_reg = registers.get(ip_reg_name)
crashing_frame = True
return find_best_instruction(processable_frame['instruction_addr'],
arch=self.arch,
crashing_frame=crashing_frame,
signal=signal,
ip_reg=ip_reg)
def handles_frame(self, frame, stacktrace_info):
if not self.available:
return False
platform = frame.get('platform') or self.data.get('platform')
if platform not in self.supported_platforms:
return False
if frame.get('data', {}).get('symbolicator_status'):
return False
if 'instruction_addr' not in frame:
return False
return True
def preprocess_frame(self, processable_frame):
instr_addr = self.find_best_instruction(processable_frame)
obj = self.object_lookup.find_object(instr_addr)
processable_frame.data = {
'instruction_addr': instr_addr,
'obj': obj,
'debug_id': obj.debug_id if obj is not None else None,
'symbolserver_match': None,
# `[]` is used to indicate to the symbolizer that the symbolicator
# deliberately discarded this frame, while `None` means the
# symbolicator didn't run (because `self.use_symbolicator` is
# false).
# If the symbolicator did run and was not able to symbolize the
# frame, this value will be a list with the raw frame as only item.
'symbolicator_match': [] if self.use_symbolicator else None,
}
if obj is not None:
processable_frame.set_cache_key_from_values((
FRAME_CACHE_VERSION,
# Because the images can move around, we want to rebase
# the address for the cache key to be within the image
# the same way as we do it in the symbolizer.
rebase_addr(instr_addr, obj),
obj.debug_id,
obj.arch,
obj.size,
))
def preprocess_step(self, processing_task):
if not self.available:
return False
referenced_images = set(
pf.data['debug_id']
for pf in processing_task.iter_processable_frames(self)
if pf.cache_value is None and pf.data['debug_id'] is not None)
self.sym = Symbolizer(self.project,
self.object_lookup,
referenced_images=referenced_images,
use_symbolicator=self.use_symbolicator)
if options.get('symbolserver.enabled'):
self.fetch_ios_system_symbols(processing_task)
if self.use_symbolicator:
self.run_symbolicator(processing_task)
def run_symbolicator(self, processing_task):
# TODO(markus): Make this work with minidumps. An unprocessed minidump
# event will not contain unsymbolicated frames, because the minidump
# upload already happened in store.
# It will also presumably not contain images, so `self.available` will
# already be `False`.
if not self.available:
return
request_id_cache_key = request_id_cache_key_for_event(self.data)
stacktraces = []
processable_stacktraces = []
has_frames = False
for stacktrace_info, pf_list in processing_task.iter_processable_stacktraces(
):
registers = stacktrace_info.stacktrace.get('registers') or {}
# The filtering condition of this list comprehension is copied
# from `iter_processable_frames`.
#
# We cannot reuse `iter_processable_frames` because the
# symbolicator currently expects a list of stacktraces, not
# flat frames.
#
# Right now we can't even filter out frames (e.g. using a frame
# cache locally). The stacktraces have to be as complete as
# possible because the symbolicator assumes the first frame of
# a stacktrace to be the crashing frame. This assumption is
# already violated because the SDK might chop off frames though
# (which is less likely to be the case though).
pf_list = [pf for pf in reversed(pf_list) if pf.processor == self]
frames = []
for pf in pf_list:
frame = {'instruction_addr': pf['instruction_addr']}
if pf.get('trust') is not None:
frame['trust'] = pf['trust']
frames.append(frame)
has_frames = True
stacktraces.append({'registers': registers, 'frames': frames})
processable_stacktraces.append(pf_list)
if not has_frames:
return
rv = run_symbolicator(project=self.project,
request_id_cache_key=request_id_cache_key,
stacktraces=stacktraces,
modules=self.images,
signal=self.signal)
if not rv:
handle_symbolication_failed(
SymbolicationFailed(
type=EventError.NATIVE_SYMBOLICATOR_FAILED),
data=self.data,
)
return
# TODO(markus): Set signal and os context from symbolicator response,
# for minidumps
assert len(self.images) == len(rv['modules']), (self.images, rv)
for image, complete_image in zip(self.images, rv['modules']):
merge_symbolicator_image(
image, complete_image, self.sdk_info,
lambda e: handle_symbolication_failed(e, data=self.data))
assert len(stacktraces) == len(rv['stacktraces'])
for pf_list, symbolicated_stacktrace in zip(processable_stacktraces,
rv['stacktraces']):
for symbolicated_frame in symbolicated_stacktrace.get(
'frames') or ():
pf = pf_list[symbolicated_frame['original_index']]
pf.data['symbolicator_match'].append(symbolicated_frame)
def fetch_ios_system_symbols(self, processing_task):
to_lookup = []
pf_list = []
for pf in processing_task.iter_processable_frames(self):
if pf.cache_value is not None:
continue
obj = pf.data['obj']
package = obj and obj.code_file
# TODO(ja): This should check for iOS specifically
if not package or not is_known_third_party(package):
continue
# We can only look up objects in the symbol server that have a
# uuid. If we encounter things with an age appended or
# similar we need to skip.
try:
uuid.UUID(obj.debug_id)
except (ValueError, TypeError):
continue
to_lookup.append({
'object_uuid':
obj.debug_id,
'object_name':
obj.code_file or '<unknown>',
'addr':
'0x%x' % rebase_addr(pf.data['instruction_addr'], obj)
})
pf_list.append(pf)
if not to_lookup:
return
rv = lookup_system_symbols(to_lookup, self.sdk_info, self.arch)
if rv is not None:
for symrv, pf in zip(rv, pf_list):
if symrv is None:
continue
pf.data['symbolserver_match'] = symrv
def process_frame(self, processable_frame, processing_task):
frame = processable_frame.frame
raw_frame = dict(frame)
errors = []
# Ensure that package is set in the raw frame, mapped from the
# debug_images array in the payload. Grouping and UI can use this path
# to infer in_app and exclude frames from grouping.
if raw_frame.get('package') is None:
obj = processable_frame.data['obj']
raw_frame['package'] = obj and obj.code_file or None
if processable_frame.cache_value is None:
# Construct a raw frame that is used by the symbolizer
# backend. We only assemble the bare minimum we need here.
instruction_addr = processable_frame.data['instruction_addr']
debug_id = processable_frame.data['debug_id']
if debug_id is not None:
self.difs_referenced.add(debug_id)
try:
symbolicated_frames = self.sym.symbolize_frame(
instruction_addr,
self.sdk_info,
symbolserver_match=processable_frame.
data['symbolserver_match'],
symbolicator_match=processable_frame.data.get(
'symbolicator_match'),
trust=raw_frame.get('trust'),
)
if not symbolicated_frames:
if raw_frame.get('trust') == 'scan':
return [], [raw_frame], []
else:
return None, [raw_frame], []
except SymbolicationFailed as e:
errors = []
handle_symbolication_failed(e, data=self.data, errors=errors)
return [raw_frame], [raw_frame], errors
_ignored = None # Used to be in_app
processable_frame.set_cache_value([_ignored, symbolicated_frames])
else: # processable_frame.cache_value is present
_ignored, symbolicated_frames = processable_frame.cache_value
platform = raw_frame.get('platform') or self.data.get('platform')
new_frames = []
for sfrm in symbolicated_frames:
new_frame = dict(raw_frame)
merge_symbolicated_frame(new_frame, sfrm, platform=platform)
new_frames.append(new_frame)
return new_frames, [raw_frame], []
def resolve_frame_symbols(data):
debug_meta = data.get('debug_meta')
if not debug_meta:
return
debug_images = debug_meta['images']
sdk_info = get_sdk_from_event(data)
stacktraces = find_all_stacktraces(data)
if not stacktraces:
return
project = Project.objects.get_from_cache(
id=data['project'],
)
errors = []
referenced_images = find_stacktrace_referenced_images(
debug_images, stacktraces)
sym = Symbolizer(project, debug_images,
referenced_images=referenced_images)
frame = None
idx = -1
def report_error(e):
errors.append({
'type': EventError.NATIVE_INTERNAL_FAILURE,
'frame': frame,
'error': 'frame #%d: %s: %s' % (
idx,
e.__class__.__name__,
six.text_type(e),
)
})
longest_addr = 0
processed_frames = []
with sym:
for stacktrace in stacktraces:
for idx, frame in enumerate(stacktrace['frames']):
if 'image_addr' not in frame or \
'instruction_addr' not in frame or \
'symbol_addr' not in frame:
continue
try:
sfrm = sym.symbolize_frame({
'object_name': frame.get('package'),
'object_addr': frame['image_addr'],
'instruction_addr': frame['instruction_addr'],
'symbol_addr': frame['symbol_addr'],
}, sdk_info, report_error=report_error)
if not sfrm:
continue
# XXX: log here if symbol could not be found?
frame['function'] = sfrm.get('symbol_name') or \
frame.get('function') or '<unknown>'
frame['abs_path'] = sfrm.get('filename') or None
if frame['abs_path']:
frame['filename'] = posixpath.basename(frame['abs_path'])
if sfrm.get('line') is not None:
frame['lineno'] = sfrm['line']
else:
frame['instruction_offset'] = \
parse_addr(sfrm['instruction_addr']) - \
parse_addr(sfrm['symbol_addr'])
if sfrm.get('column') is not None:
frame['colno'] = sfrm['column']
frame['package'] = sfrm['object_name'] or frame.get('package')
frame['symbol_addr'] = '0x%x' % parse_addr(sfrm['symbol_addr'])
frame['instruction_addr'] = '0x%x' % parse_addr(
sfrm['instruction_addr'])
frame['in_app'] = is_in_app(frame)
longest_addr = max(longest_addr, len(frame['symbol_addr']),
len(frame['instruction_addr']))
processed_frames.append(frame)
except Exception:
logger.exception('Failed to symbolicate')
errors.append({
'type': EventError.NATIVE_INTERNAL_FAILURE,
'error': 'The symbolicator encountered an internal failure',
})
# Pad out addresses to be of the same length
for frame in processed_frames:
for key in 'symbol_addr', 'instruction_addr':
frame[key] = '0x' + frame[key][2:].rjust(longest_addr - 2, '0')
if errors:
data.setdefault('errors', []).extend(errors)
return data
class NativeStacktraceProcessor(StacktraceProcessor):
supported_platforms = ('cocoa', 'native')
# TODO(ja): Clean up all uses of image type "apple", "uuid", "id" and "name"
supported_images = ('apple', 'symbolic', 'elf', 'macho', 'pe')
def __init__(self, *args, **kwargs):
StacktraceProcessor.__init__(self, *args, **kwargs)
self.arch = cpu_name_from_data(self.data)
self.sym = None
self.difs_referenced = set()
images = get_path(self.data, 'debug_meta', 'images', default=(),
filter=self._is_valid_image)
if images:
self.available = True
self.sdk_info = get_sdk_from_event(self.data)
self.object_lookup = ObjectLookup(images)
else:
self.available = False
def _is_valid_image(self, image):
# TODO(ja): Deprecate this. The symbolicator should take care of
# filtering valid images.
return bool(image) \
and image.get('type') in self.supported_images \
and image.get('image_addr') is not None \
and image.get('image_size') is not None \
and (image.get('debug_id') or image.get('id') or image.get('uuid')) is not None
def close(self):
StacktraceProcessor.close(self)
if self.difs_referenced:
metrics.incr(
'dsyms.processed',
amount=len(self.difs_referenced),
skip_internal=True,
tags={
'project_id': self.project.id,
},
)
def find_best_instruction(self, processable_frame):
"""Given a frame, stacktrace info and frame index this returns the
interpolated instruction address we then use for symbolication later.
"""
if self.arch is None:
return parse_addr(processable_frame['instruction_addr'])
crashing_frame = False
signal = None
ip_reg = None
# We only need to provide meta information for frame zero
if processable_frame.idx == 0:
# The signal is useful information for symbolic in some situations
# to disambiugate the first frame. If we can get this information
# from the mechanism we want to pass it onwards.
exceptions = get_path(self.data, 'exception', 'values', filter=True)
signal = get_path(exceptions, 0, 'mechanism', 'meta', 'signal', 'number')
if signal is not None:
signal = int(signal)
registers = processable_frame.stacktrace_info.stacktrace.get('registers')
if registers:
ip_reg_name = arch_get_ip_reg_name(self.arch)
if ip_reg_name:
ip_reg = registers.get(ip_reg_name)
crashing_frame = True
return find_best_instruction(
processable_frame['instruction_addr'],
arch=self.arch,
crashing_frame=crashing_frame,
signal=signal,
ip_reg=ip_reg
)
def handles_frame(self, frame, stacktrace_info):
platform = frame.get('platform') or self.data.get('platform')
return (platform in self.supported_platforms and self.available
and 'instruction_addr' in frame)
def preprocess_frame(self, processable_frame):
instr_addr = self.find_best_instruction(processable_frame)
obj = self.object_lookup.find_object(instr_addr)
processable_frame.data = {
'instruction_addr': instr_addr,
'obj': obj,
'debug_id': obj.debug_id if obj is not None else None,
'symbolserver_match': None,
}
if obj is not None:
processable_frame.set_cache_key_from_values(
(
FRAME_CACHE_VERSION,
# Because the images can move around, we want to rebase
# the address for the cache key to be within the image
# the same way as we do it in the symbolizer.
rebase_addr(instr_addr, obj),
obj.debug_id,
obj.arch,
obj.size,
)
)
def preprocess_step(self, processing_task):
if not self.available:
return False
referenced_images = set(
pf.data['debug_id'] for pf in processing_task.iter_processable_frames(self)
if pf.cache_value is None and pf.data['debug_id'] is not None
)
self.sym = Symbolizer(
self.project,
self.object_lookup,
referenced_images=referenced_images,
)
if options.get('symbolserver.enabled'):
self.fetch_system_symbols(processing_task)
def fetch_system_symbols(self, processing_task):
to_lookup = []
pf_list = []
for pf in processing_task.iter_processable_frames(self):
obj = pf.data['obj']
if pf.cache_value is not None or obj is None or \
self.sym.is_image_from_app_bundle(obj):
continue
# We can only look up objects in the symbol server that have a
# uuid. If we encounter things with an age appended or
# similar we need to skip.
try:
uuid.UUID(obj.debug_id)
except (ValueError, TypeError):
continue
to_lookup.append(
{
'object_uuid': obj.debug_id,
'object_name': obj.name or '<unknown>',
'addr': '0x%x' % rebase_addr(pf.data['instruction_addr'], obj)
}
)
pf_list.append(pf)
if not to_lookup:
return
rv = lookup_system_symbols(to_lookup, self.sdk_info, self.arch)
if rv is not None:
for symrv, pf in zip(rv, pf_list):
if symrv is None:
continue
pf.data['symbolserver_match'] = symrv
def process_frame(self, processable_frame, processing_task):
frame = processable_frame.frame
raw_frame = dict(frame)
errors = []
if processable_frame.cache_value is None:
# Construct a raw frame that is used by the symbolizer
# backend. We only assemble the bare minimum we need here.
instruction_addr = processable_frame.data['instruction_addr']
in_app = self.sym.is_in_app(
instruction_addr,
sdk_info=self.sdk_info
)
if in_app and raw_frame.get('function') is not None:
in_app = not self.sym.is_internal_function(
raw_frame['function'])
if raw_frame.get('in_app') is None:
raw_frame['in_app'] = in_app
debug_id = processable_frame.data['debug_id']
if debug_id is not None:
self.difs_referenced.add(debug_id)
try:
symbolicated_frames = self.sym.symbolize_frame(
instruction_addr,
self.sdk_info,
symbolserver_match=processable_frame.data['symbolserver_match'],
trust=raw_frame.get('trust'),
)
if not symbolicated_frames:
if raw_frame.get('trust') == 'scan':
return [], [raw_frame], []
else:
return None, [raw_frame], []
except SymbolicationFailed as e:
# User fixable but fatal errors are reported as processing
# issues
if e.is_user_fixable and e.is_fatal:
report_processing_issue(
self.data,
scope='native',
object='dsym:%s' % e.image_uuid,
type=e.type,
data=e.get_data()
)
# This in many ways currently does not really do anything.
# The reason is that once a processing issue is reported
# the event will only be stored as a raw event and no
# group will be generated. As a result it also means that
# we will not have any user facing event or error showing
# up at all. We want to keep this here though in case we
# do not want to report some processing issues (eg:
# optional difs)
errors = []
if e.is_user_fixable or e.is_sdk_failure:
errors.append(e.get_data())
else:
logger.debug('Failed to symbolicate with native backend',
exc_info=True)
return [raw_frame], [raw_frame], errors
processable_frame.set_cache_value([in_app, symbolicated_frames])
else: # processable_frame.cache_value is present
in_app, symbolicated_frames = processable_frame.cache_value
raw_frame['in_app'] = in_app
new_frames = []
for sfrm in symbolicated_frames:
new_frame = dict(frame)
new_frame['function'] = sfrm['function']
if sfrm.get('symbol'):
new_frame['symbol'] = sfrm['symbol']
new_frame['abs_path'] = sfrm['abs_path']
new_frame['filename'] = sfrm.get('filename') or \
(sfrm['abs_path'] and posixpath.basename(sfrm['abs_path'])) or \
None
if sfrm.get('lineno'):
new_frame['lineno'] = sfrm['lineno']
if sfrm.get('colno'):
new_frame['colno'] = sfrm['colno']
if sfrm.get('package') or processable_frame.data['obj'] is not None:
new_frame['package'] = sfrm.get(
'package', processable_frame.data['obj'].name)
if new_frame.get('in_app') is None:
new_frame['in_app'] = in_app and \
not self.sym.is_internal_function(new_frame['function'])
new_frames.append(new_frame)
return new_frames, [raw_frame], []
class NativeStacktraceProcessor(StacktraceProcessor):
def __init__(self, *args, **kwargs):
StacktraceProcessor.__init__(self, *args, **kwargs)
debug_meta = self.data.get('debug_meta')
self.sym = None
if debug_meta:
self.available = True
self.debug_meta = debug_meta
self.sdk_info = get_sdk_from_event(self.data)
else:
self.available = False
def close(self):
StacktraceProcessor.close(self)
if self.sym is not None:
self.sym.close()
self.sym = None
def preprocess_related_data(self):
if not self.available:
return False
is_debug_build = self.debug_meta.get('is_debug_build')
referenced_images = find_stacktrace_referenced_images(
self.debug_meta['images'],
[x.stacktrace for x in self.stacktrace_infos])
self.sym = Symbolizer(self.project,
self.debug_meta['images'],
referenced_images=referenced_images,
is_debug_build=is_debug_build)
# The symbolizer gets a reference to the debug meta's images so
# when it resolves the missing vmaddrs it changes them in the data
# dict.
return self.sym.resolve_missing_vmaddrs()
def find_best_instruction(self, frame, stacktrace_info, idx):
"""Given a frame, stacktrace info and frame index this returns the
interpolated instruction address we then use for symbolication later.
"""
meta = None
# We only need to provide meta information for frame zero
if idx == 0:
# The signal is useful information for symsynd in some situations
# to disambiugate the first frame. If we can get this information
# from the mechanism we want to pass it onwards.
signal = None
exc = self.data.get('sentry.interfaces.Exception')
if exc is not None:
mechanism = exc['values'][0].get('mechanism')
if mechanism and 'posix_signal' in mechanism and \
'signal' in mechanism['posix_signal']:
signal = mechanism['posix_signal']['signal']
meta = {
'frame_number': 0,
'registers': stacktrace_info.stacktrace.get('registers'),
'signal': signal,
}
return self.sym.find_best_instruction(frame, meta=meta)
def process_frame(self, frame, stacktrace_info, idx):
# XXX: warn on missing availability?
# Only process frames here that are of supported platforms and
# have the mandatory requirements for
if not self.available or \
self.get_effective_platform(frame) != 'cocoa' or \
'instruction_addr' not in frame:
return None
errors = []
# Construct a raw frame that is used by the symbolizer
# backend. We only assemble the bare minimum we need here.
sym_input_frame = {
'object_name':
frame.get('package'),
'instruction_addr':
self.find_best_instruction(frame, stacktrace_info, idx),
'symbol_name':
frame.get('function'),
'symbol_addr':
frame.get('symbol_addr'),
}
in_app = self.sym.is_in_app(sym_input_frame)
new_frames = []
raw_frame = dict(frame)
raw_frame['in_app'] = in_app
try:
symbolicated_frames = self.sym.symbolize_frame(
sym_input_frame, self.sdk_info, symbolize_inlined=True)
if not symbolicated_frames:
return None, [raw_frame], []
except SymbolicationFailed as e:
errors = []
if e.is_user_fixable or e.is_sdk_failure:
errors.append({
'type': e.type,
'image_uuid': e.image_uuid,
'image_path': e.image_path,
'image_arch': e.image_arch,
'message': e.message,
})
else:
logger.debug('Failed to symbolicate with native backend',
exc_info=True)
return None, [raw_frame], errors
for sfrm in symbolicated_frames:
symbol = sfrm.get('symbol_name') or \
frame.get('function') or '<unknown>'
function = demangle_symbol(symbol, simplified=True)
new_frame = dict(frame)
new_frame['function'] = function
# If we demangled something, store the original in the
# symbol portion of the frame
if function != symbol:
new_frame['symbol'] = symbol
new_frame['abs_path'] = sfrm.get('filename') or None
if new_frame['abs_path']:
new_frame['filename'] = posixpath.basename(
new_frame['abs_path'])
if sfrm.get('line') is not None:
new_frame['lineno'] = sfrm['line']
if sfrm.get('column') is not None:
new_frame['colno'] = sfrm['column']
new_frame['package'] = sfrm['object_name'] \
or new_frame.get('package')
new_frame['in_app'] = in_app
new_frames.append(new_frame)
return new_frames, [raw_frame], []
class NativeStacktraceProcessor(StacktraceProcessor):
def __init__(self, *args, **kwargs):
StacktraceProcessor.__init__(self, *args, **kwargs)
debug_meta = self.data.get('debug_meta')
self.cpu_name = cpu_name_from_data(self.data)
self.sym = None
if debug_meta:
self.available = True
self.debug_meta = debug_meta
self.sdk_info = get_sdk_from_event(self.data)
self.image_lookup = ImageLookup(self.debug_meta['images'])
else:
self.available = False
def close(self):
StacktraceProcessor.close(self)
if self.sym is not None:
self.sym.close()
self.sym = None
def find_best_instruction(self, processable_frame):
"""Given a frame, stacktrace info and frame index this returns the
interpolated instruction address we then use for symbolication later.
"""
if self.cpu_name is None:
return parse_addr(processable_frame['instruction_addr'])
meta = None
# We only need to provide meta information for frame zero
if processable_frame.idx == 0:
# The signal is useful information for symsynd in some situations
# to disambiugate the first frame. If we can get this information
# from the mechanism we want to pass it onwards.
signal = None
exc = self.data.get('sentry.interfaces.Exception')
if exc is not None:
mechanism = exc['values'][0].get('mechanism')
if mechanism and 'posix_signal' in mechanism and \
'signal' in mechanism['posix_signal']:
signal = mechanism['posix_signal']['signal']
meta = {
'frame_number': 0,
'registers': processable_frame.stacktrace_info.stacktrace.get('registers'),
'signal': signal,
}
return find_best_instruction(
processable_frame['instruction_addr'],
self.cpu_name, meta=meta)
def handles_frame(self, frame, stacktrace_info):
platform = frame.get('platform') or self.data.get('platform')
return (
platform == 'cocoa' and
self.available and
'instruction_addr' in frame
)
def preprocess_frame(self, processable_frame):
instr_addr = self.find_best_instruction(processable_frame)
img = self.image_lookup.find_image(instr_addr)
processable_frame.data = {
'instruction_addr': instr_addr,
'image': img,
'image_uuid': img['uuid'] if img is not None else None,
'symbolserver_match': None,
}
if img is not None:
processable_frame.set_cache_key_from_values((
FRAME_CACHE_VERSION,
# Because the images can move around, we want to rebase
# the address for the cache key to be within the image
# the same way as we do it in the symbolizer.
rebase_addr(instr_addr, img),
img['uuid'].lower(),
img['cpu_type'],
img['cpu_subtype'],
img['image_size'],
))
def preprocess_step(self, processing_task):
if not self.available:
return False
referenced_images = set(
pf.data['image_uuid']
for pf in processing_task.iter_processable_frames(self)
if pf.cache_value is None and pf.data['image_uuid'] is not None)
def on_referenced(dsym_file):
app_info = version_build_from_data(self.data)
if app_info is not None:
dsym_app = DSymApp.objects.create_or_update_app(
sync_id=None,
app_id=app_info.id,
project=self.project,
data={'name': app_info.name},
platform=DSymPlatform.APPLE,
)
version_dsym_file, created = VersionDSymFile.objects.get_or_create(
dsym_file=dsym_file,
dsym_app=dsym_app,
version=app_info.version,
build=app_info.build,
)
self.sym = Symbolizer(self.project, self.image_lookup,
cpu_name=self.cpu_name,
referenced_images=referenced_images,
on_dsym_file_referenced=on_referenced)
# The symbolizer gets a reference to the debug meta's images so
# when it resolves the missing vmaddrs it changes them in the data
# dict.
data = self.sym.resolve_missing_vmaddrs()
if options.get('symbolserver.enabled'):
self.fetch_system_symbols(processing_task)
return data
def fetch_system_symbols(self, processing_task):
to_lookup = []
pf_list = []
for pf in processing_task.iter_processable_frames(self):
img = pf.data['image']
if pf.cache_value is not None or img is None or \
self.sym.is_frame_from_app_bundle(pf.frame, img):
continue
to_lookup.append({
'object_uuid': img['uuid'],
'object_name': img['name'],
'addr': '0x%x' % rebase_addr(pf.data['instruction_addr'], img)
})
pf_list.append(pf)
if not to_lookup:
return
rv = lookup_system_symbols(to_lookup, self.sdk_info,
self.sym.cpu_name)
if rv is not None:
for symrv, pf in zip(rv, pf_list):
if symrv is None:
continue
pf.data['symbolserver_match'] = symrv
def process_frame(self, processable_frame, processing_task):
frame = processable_frame.frame
errors = []
new_frames = []
raw_frame = dict(frame)
if processable_frame.cache_value is None:
# Construct a raw frame that is used by the symbolizer
# backend. We only assemble the bare minimum we need here.
sym_input_frame = {
'object_name': frame.get('package'),
'instruction_addr': processable_frame.data['instruction_addr'],
'symbol_name': frame.get('function'),
}
in_app = self.sym.is_in_app(sym_input_frame)
raw_frame['in_app'] = in_app
try:
symbolicated_frames = self.sym.symbolize_frame(
sym_input_frame, self.sdk_info,
symbolserver_match=processable_frame.data['symbolserver_match'],
symbolize_inlined=True)
if not symbolicated_frames:
return None, [raw_frame], []
except SymbolicationFailed as e:
reprocessing_active = False
if self.project:
reprocessing_active = bool(
self.project.get_option('sentry:reprocessing_active', False)
)
# User fixable but fatal errors are reported as processing
# issues but only if the feature is activated.
if reprocessing_active and e.is_user_fixable and e.is_fatal:
report_processing_issue(self.data,
scope='native',
object='dsym:%s' % e.image_uuid,
type=e.type,
data={
'image_path': e.image_path,
'image_uuid': e.image_uuid,
'image_arch': e.image_arch,
'message': e.message,
}
)
# This in many ways currently does not really do anything.
# The reason is that once a processing issue is reported
# the event will only be stored as a raw event and no
# group will be generated. As a result it also means that
# we will not have any user facing event or error showing
# up at all. We want to keep this here though in case we
# do not want to report some processing issues (eg:
# optional dsyms)
errors = []
if e.is_user_fixable or e.is_sdk_failure:
errors.append({
'type': e.type,
'image_uuid': e.image_uuid,
'image_path': e.image_path,
'image_arch': e.image_arch,
'message': e.message,
})
else:
logger.debug('Failed to symbolicate with native backend',
exc_info=True)
return [raw_frame], [raw_frame], errors
processable_frame.set_cache_value([in_app, symbolicated_frames])
else:
in_app, symbolicated_frames = processable_frame.cache_value
raw_frame['in_app'] = in_app
for sfrm in symbolicated_frames:
symbol = sfrm.get('symbol_name') or \
frame.get('function') or NATIVE_UNKNOWN_STRING
function = demangle_symbol(symbol, simplified=True)
new_frame = dict(frame)
new_frame['function'] = function
# If we demangled something, store the original in the
# symbol portion of the frame
if function != symbol:
new_frame['symbol'] = symbol
new_frame['abs_path'] = sfrm.get('filename') or None
if new_frame['abs_path']:
new_frame['filename'] = posixpath.basename(
new_frame['abs_path'])
if sfrm.get('line') is not None:
new_frame['lineno'] = sfrm['line']
if sfrm.get('column') is not None:
new_frame['colno'] = sfrm['column']
new_frame['package'] = sfrm['object_name'] \
or new_frame.get('package')
new_frame['in_app'] = in_app
new_frames.append(new_frame)
return new_frames, [raw_frame], []
def resolve_frame_symbols(data):
debug_meta = data['debug_meta']
debug_images = debug_meta['images']
sdk_info = get_sdk_from_event(data)
stacktraces = find_all_stacktraces(data)
if not stacktraces:
return
project = Project.objects.get_from_cache(
id=data['project'],
)
errors = []
referenced_images = find_stacktrace_referenced_images(
debug_images, [x[0] for x in stacktraces])
sym = Symbolizer(project, debug_images,
referenced_images=referenced_images)
frame = None
idx = -1
def report_error(e):
errors.append({
'type': EventError.NATIVE_INTERNAL_FAILURE,
'frame': frame,
'error': 'frame #%d: %s: %s' % (
idx,
e.__class__.__name__,
six.text_type(e),
)
})
with sym:
for stacktrace, container in stacktraces:
store_raw = False
new_frames = list(stacktrace['frames'])
for idx, frame in enumerate(stacktrace['frames']):
if 'image_addr' not in frame or \
'instruction_addr' not in frame or \
'symbol_addr' not in frame:
continue
try:
sfrm = sym.symbolize_frame({
'object_name': frame.get('package'),
'object_addr': frame['image_addr'],
'instruction_addr': frame['instruction_addr'],
'symbol_addr': frame['symbol_addr'],
}, sdk_info, report_error=report_error)
if not sfrm:
continue
new_frame = dict(frame)
# XXX: log here if symbol could not be found?
symbol = sfrm.get('symbol_name') or \
new_frame.get('function') or '<unknown>'
function = demangle_symbol(symbol, simplified=True)
new_frame['function'] = function
# If we demangled something, store the original in the
# symbol portion of the frame
if function != symbol:
new_frame['symbol'] = symbol
new_frame['abs_path'] = sfrm.get('filename') or None
if new_frame['abs_path']:
new_frame['filename'] = posixpath.basename(new_frame['abs_path'])
if sfrm.get('line') is not None:
new_frame['lineno'] = sfrm['line']
else:
new_frame['instruction_offset'] = \
parse_addr(sfrm['instruction_addr']) - \
parse_addr(sfrm['symbol_addr'])
if sfrm.get('column') is not None:
new_frame['colno'] = sfrm['column']
new_frame['package'] = sfrm['object_name'] or new_frame.get('package')
new_frame['symbol_addr'] = '0x%x' % parse_addr(sfrm['symbol_addr'])
new_frame['instruction_addr'] = '0x%x' % parse_addr(
sfrm['instruction_addr'])
new_frame['in_app'] = is_in_app(new_frame)
if new_frame != frame:
new_frames[idx] = new_frame
store_raw = True
except Exception:
logger.exception('Failed to symbolicate')
errors.append({
'type': EventError.NATIVE_INTERNAL_FAILURE,
'error': 'The symbolicator encountered an internal failure',
})
# Remember the raw stacktrace.
if store_raw and container is not None:
container['raw_stacktrace'] = {
'frames': stacktrace['frames'],
}
# Put the new frames in
stacktrace['frames'] = new_frames
if errors:
data.setdefault('errors', []).extend(errors)
return data
class NativeStacktraceProcessor(StacktraceProcessor):
def __init__(self, *args, **kwargs):
StacktraceProcessor.__init__(self, *args, **kwargs)
debug_meta = self.data.get('debug_meta')
self.cpu_name = cpu_name_from_data(self.data)
self.sym = None
self.dsyms_referenced = set()
if debug_meta:
self.available = True
self.debug_meta = debug_meta
self.sdk_info = get_sdk_from_event(self.data)
self.image_lookup = ImageLookup(self.debug_meta['images'])
else:
self.available = False
def close(self):
StacktraceProcessor.close(self)
if self.dsyms_referenced:
metrics.incr('dsyms.processed',
amount=len(self.dsyms_referenced),
instance=self.project.id)
if self.sym is not None:
self.sym.close()
self.sym = None
def find_best_instruction(self, processable_frame):
"""Given a frame, stacktrace info and frame index this returns the
interpolated instruction address we then use for symbolication later.
"""
if self.cpu_name is None:
return parse_addr(processable_frame['instruction_addr'])
meta = None
# We only need to provide meta information for frame zero
if processable_frame.idx == 0:
# The signal is useful information for symsynd in some situations
# to disambiugate the first frame. If we can get this information
# from the mechanism we want to pass it onwards.
signal = None
exc = self.data.get('sentry.interfaces.Exception')
if exc is not None:
mechanism = exc['values'][0].get('mechanism')
if mechanism and 'posix_signal' in mechanism and \
'signal' in mechanism['posix_signal']:
signal = mechanism['posix_signal']['signal']
meta = {
'frame_number':
0,
'registers':
processable_frame.stacktrace_info.stacktrace.get('registers'),
'signal':
signal,
}
return find_best_instruction(processable_frame['instruction_addr'],
self.cpu_name,
meta=meta)
def handles_frame(self, frame, stacktrace_info):
platform = frame.get('platform') or self.data.get('platform')
return (platform == 'cocoa' and self.available
and 'instruction_addr' in frame)
def preprocess_frame(self, processable_frame):
instr_addr = self.find_best_instruction(processable_frame)
img = self.image_lookup.find_image(instr_addr)
processable_frame.data = {
'instruction_addr': instr_addr,
'image': img,
'image_uuid': img['uuid'] if img is not None else None,
'symbolserver_match': None,
}
if img is not None:
processable_frame.set_cache_key_from_values((
FRAME_CACHE_VERSION,
# Because the images can move around, we want to rebase
# the address for the cache key to be within the image
# the same way as we do it in the symbolizer.
rebase_addr(instr_addr, img),
img['uuid'].lower(),
img['cpu_type'],
img['cpu_subtype'],
img['image_size'],
))
def preprocess_step(self, processing_task):
if not self.available:
return False
referenced_images = set(
pf.data['image_uuid']
for pf in processing_task.iter_processable_frames(self)
if pf.cache_value is None and pf.data['image_uuid'] is not None)
def on_referenced(dsym_file):
app_info = version_build_from_data(self.data)
if app_info is not None:
dsym_app = DSymApp.objects.create_or_update_app(
sync_id=None,
app_id=app_info.id,
project=self.project,
data={'name': app_info.name},
platform=DSymPlatform.APPLE,
)
try:
with transaction.atomic():
version_dsym_file, created = VersionDSymFile.objects.get_or_create(
dsym_file=dsym_file,
dsym_app=dsym_app,
version=app_info.version,
build=app_info.build,
)
except IntegrityError:
# XXX: this can currently happen because we only
# support one app per dsym file. Since this can
# happen in some cases anyways we ignore it.
pass
self.sym = Symbolizer(self.project,
self.image_lookup,
cpu_name=self.cpu_name,
referenced_images=referenced_images,
on_dsym_file_referenced=on_referenced)
# The symbolizer gets a reference to the debug meta's images so
# when it resolves the missing vmaddrs it changes them in the data
# dict.
data = self.sym.resolve_missing_vmaddrs()
if options.get('symbolserver.enabled'):
self.fetch_system_symbols(processing_task)
return data
def fetch_system_symbols(self, processing_task):
to_lookup = []
pf_list = []
for pf in processing_task.iter_processable_frames(self):
img = pf.data['image']
if pf.cache_value is not None or img is None or \
self.sym.is_frame_from_app_bundle(pf.frame, img):
continue
to_lookup.append({
'object_uuid':
img['uuid'],
'object_name':
img['name'],
'addr':
'0x%x' % rebase_addr(pf.data['instruction_addr'], img)
})
pf_list.append(pf)
if not to_lookup:
return
rv = lookup_system_symbols(to_lookup, self.sdk_info, self.sym.cpu_name)
if rv is not None:
for symrv, pf in zip(rv, pf_list):
if symrv is None:
continue
pf.data['symbolserver_match'] = symrv
def process_frame(self, processable_frame, processing_task):
frame = processable_frame.frame
errors = []
new_frames = []
raw_frame = dict(frame)
if processable_frame.cache_value is None:
# Construct a raw frame that is used by the symbolizer
# backend. We only assemble the bare minimum we need here.
sym_input_frame = {
'object_name': frame.get('package'),
'instruction_addr': processable_frame.data['instruction_addr'],
'symbol_name': frame.get('function'),
}
in_app = self.sym.is_in_app(sym_input_frame)
raw_frame['in_app'] = in_app
img_uuid = processable_frame.data['image_uuid']
if img_uuid is not None:
self.dsyms_referenced.add(img_uuid)
try:
symbolicated_frames = self.sym.symbolize_frame(
sym_input_frame,
self.sdk_info,
symbolserver_match=processable_frame.
data['symbolserver_match'],
symbolize_inlined=True)
if not symbolicated_frames:
return None, [raw_frame], []
except SymbolicationFailed as e:
reprocessing_active = False
if self.project:
reprocessing_active = bool(
self.project.get_option('sentry:reprocessing_active',
False))
# User fixable but fatal errors are reported as processing
# issues but only if the feature is activated.
if reprocessing_active and e.is_user_fixable and e.is_fatal:
report_processing_issue(self.data,
scope='native',
object='dsym:%s' % e.image_uuid,
type=e.type,
data={
'image_path': e.image_path,
'image_uuid': e.image_uuid,
'image_arch': e.image_arch,
'message': e.message,
})
# This in many ways currently does not really do anything.
# The reason is that once a processing issue is reported
# the event will only be stored as a raw event and no
# group will be generated. As a result it also means that
# we will not have any user facing event or error showing
# up at all. We want to keep this here though in case we
# do not want to report some processing issues (eg:
# optional dsyms)
errors = []
if e.is_user_fixable or e.is_sdk_failure:
errors.append({
'type': e.type,
'image_uuid': e.image_uuid,
'image_path': e.image_path,
'image_arch': e.image_arch,
'message': e.message,
})
else:
logger.debug('Failed to symbolicate with native backend',
exc_info=True)
return [raw_frame], [raw_frame], errors
processable_frame.set_cache_value([in_app, symbolicated_frames])
else:
in_app, symbolicated_frames = processable_frame.cache_value
raw_frame['in_app'] = in_app
for sfrm in symbolicated_frames:
symbol = sfrm.get('symbol_name') or \
frame.get('function') or NATIVE_UNKNOWN_STRING
function = demangle_symbol(symbol, simplified=True)
new_frame = dict(frame)
new_frame['function'] = function
# If we demangled something, store the original in the
# symbol portion of the frame
if function != symbol:
new_frame['symbol'] = symbol
new_frame['abs_path'] = sfrm.get('filename') or None
if new_frame['abs_path']:
new_frame['filename'] = posixpath.basename(
new_frame['abs_path'])
if sfrm.get('line') is not None:
new_frame['lineno'] = sfrm['line']
if sfrm.get('column') is not None:
new_frame['colno'] = sfrm['column']
new_frame['package'] = sfrm['object_name'] \
or new_frame.get('package')
new_frame['in_app'] = in_app
new_frames.append(new_frame)
return new_frames, [raw_frame], []
class NativeStacktraceProcessor(StacktraceProcessor):
supported_platforms = ('cocoa', 'native')
# TODO(ja): Clean up all uses of image type "apple", "uuid", "id" and "name"
supported_images = ('apple', 'symbolic', 'elf', 'macho', 'pe')
def __init__(self, *args, **kwargs):
StacktraceProcessor.__init__(self, *args, **kwargs)
# If true, the project has been opted into using the symbolicator
# service for native symbolication, which also means symbolic is not
# used at all anymore.
# The (iOS) symbolserver is still used regardless of this value.
self.use_symbolicator = _is_symbolicator_enabled(self.project, self.data)
metrics.incr('native.use_symbolicator', tags={'value': self.use_symbolicator})
self.arch = cpu_name_from_data(self.data)
self.signal = signal_from_data(self.data)
self.sym = None
self.difs_referenced = set()
images = get_path(self.data, 'debug_meta', 'images', default=(),
filter=self._is_valid_image)
if images:
self.available = True
self.sdk_info = get_sdk_from_event(self.data)
self.object_lookup = ObjectLookup(images)
self.images = images
else:
self.available = False
def _is_valid_image(self, image):
# TODO(ja): Deprecate this. The symbolicator should take care of
# filtering valid images.
return bool(image) \
and image.get('type') in self.supported_images \
and image.get('image_addr') is not None \
and image.get('image_size') is not None \
and (image.get('debug_id') or image.get('id') or image.get('uuid')) is not None
def close(self):
StacktraceProcessor.close(self)
if self.difs_referenced:
metrics.incr(
'dsyms.processed',
amount=len(self.difs_referenced),
skip_internal=True,
tags={
'project_id': self.project.id,
},
)
def find_best_instruction(self, processable_frame):
"""Given a frame, stacktrace info and frame index this returns the
interpolated instruction address we then use for symbolication later.
"""
if self.arch is None:
return parse_addr(processable_frame['instruction_addr'])
crashing_frame = False
signal = None
ip_reg = None
# We only need to provide meta information for frame zero
if processable_frame.idx == 0:
# The signal is useful information for symbolic in some situations
# to disambiugate the first frame. If we can get this information
# from the mechanism we want to pass it onwards.
signal = self.signal
registers = processable_frame.stacktrace_info.stacktrace.get('registers')
if registers:
ip_reg_name = arch_get_ip_reg_name(self.arch)
if ip_reg_name:
ip_reg = registers.get(ip_reg_name)
crashing_frame = True
return find_best_instruction(
processable_frame['instruction_addr'],
arch=self.arch,
crashing_frame=crashing_frame,
signal=signal,
ip_reg=ip_reg
)
def handles_frame(self, frame, stacktrace_info):
if not self.available:
return False
platform = frame.get('platform') or self.data.get('platform')
if platform not in self.supported_platforms:
return False
if frame.get('data', {}).get('symbolicator_status'):
return False
if 'instruction_addr' not in frame:
return False
return True
def preprocess_frame(self, processable_frame):
instr_addr = self.find_best_instruction(processable_frame)
obj = self.object_lookup.find_object(instr_addr)
processable_frame.data = {
'instruction_addr': instr_addr,
'obj': obj,
'debug_id': obj.debug_id if obj is not None else None,
'symbolserver_match': None,
# `[]` is used to indicate to the symbolizer that the symbolicator
# deliberately discarded this frame, while `None` means the
# symbolicator didn't run (because `self.use_symbolicator` is
# false).
# If the symbolicator did run and was not able to symbolize the
# frame, this value will be a list with the raw frame as only item.
'symbolicator_match': [] if self.use_symbolicator else None,
}
if obj is not None:
processable_frame.set_cache_key_from_values(
(
FRAME_CACHE_VERSION,
# Because the images can move around, we want to rebase
# the address for the cache key to be within the image
# the same way as we do it in the symbolizer.
rebase_addr(instr_addr, obj),
obj.debug_id,
obj.arch,
obj.size,
)
)
def preprocess_step(self, processing_task):
if not self.available:
return False
referenced_images = set(
pf.data['debug_id'] for pf in processing_task.iter_processable_frames(self)
if pf.cache_value is None and pf.data['debug_id'] is not None
)
self.sym = Symbolizer(
self.project,
self.object_lookup,
referenced_images=referenced_images,
use_symbolicator=self.use_symbolicator
)
if options.get('symbolserver.enabled'):
self.fetch_ios_system_symbols(processing_task)
if self.use_symbolicator:
self.run_symbolicator(processing_task)
def run_symbolicator(self, processing_task):
# TODO(markus): Make this work with minidumps. An unprocessed minidump
# event will not contain unsymbolicated frames, because the minidump
# upload already happened in store.
# It will also presumably not contain images, so `self.available` will
# already be `False`.
if not self.available:
return
request_id_cache_key = request_id_cache_key_for_event(self.data)
stacktraces = []
processable_stacktraces = []
has_frames = False
for stacktrace_info, pf_list in processing_task.iter_processable_stacktraces():
registers = stacktrace_info.stacktrace.get('registers') or {}
# The filtering condition of this list comprehension is copied
# from `iter_processable_frames`.
#
# We cannot reuse `iter_processable_frames` because the
# symbolicator currently expects a list of stacktraces, not
# flat frames.
#
# Right now we can't even filter out frames (e.g. using a frame
# cache locally). The stacktraces have to be as complete as
# possible because the symbolicator assumes the first frame of
# a stacktrace to be the crashing frame. This assumption is
# already violated because the SDK might chop off frames though
# (which is less likely to be the case though).
pf_list = [
pf for pf in reversed(pf_list)
if pf.processor == self
]
frames = []
for pf in pf_list:
frame = {'instruction_addr': pf['instruction_addr']}
if pf.get('trust') is not None:
frame['trust'] = pf['trust']
frames.append(frame)
has_frames = True
stacktraces.append({
'registers': registers,
'frames': frames
})
processable_stacktraces.append(pf_list)
if not has_frames:
return
rv = run_symbolicator(
project=self.project,
request_id_cache_key=request_id_cache_key,
stacktraces=stacktraces, modules=self.images,
signal=self.signal
)
if not handle_symbolicator_response_status(self.data, rv):
return
# TODO(markus): Set signal and os context from symbolicator response,
# for minidumps
assert len(self.images) == len(rv['modules']), (self.images, rv)
for image, complete_image in zip(self.images, rv['modules']):
merge_symbolicator_image(
image, complete_image, self.sdk_info,
lambda e: handle_symbolication_failed(e, data=self.data)
)
assert len(stacktraces) == len(rv['stacktraces'])
for pf_list, symbolicated_stacktrace in zip(
processable_stacktraces,
rv['stacktraces']
):
for symbolicated_frame in symbolicated_stacktrace.get('frames') or ():
pf = pf_list[symbolicated_frame['original_index']]
pf.data['symbolicator_match'].append(symbolicated_frame)
def fetch_ios_system_symbols(self, processing_task):
to_lookup = []
pf_list = []
for pf in processing_task.iter_processable_frames(self):
if pf.cache_value is not None:
continue
obj = pf.data['obj']
package = obj and obj.code_file
# TODO(ja): This should check for iOS specifically
if not package or not is_known_third_party(package):
continue
# We can only look up objects in the symbol server that have a
# uuid. If we encounter things with an age appended or
# similar we need to skip.
try:
uuid.UUID(obj.debug_id)
except (ValueError, TypeError):
continue
to_lookup.append(
{
'object_uuid': obj.debug_id,
'object_name': obj.code_file or '<unknown>',
'addr': '0x%x' % rebase_addr(pf.data['instruction_addr'], obj)
}
)
pf_list.append(pf)
if not to_lookup:
return
rv = lookup_system_symbols(to_lookup, self.sdk_info, self.arch)
if rv is not None:
for symrv, pf in zip(rv, pf_list):
if symrv is None:
continue
pf.data['symbolserver_match'] = symrv
def process_frame(self, processable_frame, processing_task):
frame = processable_frame.frame
raw_frame = dict(frame)
errors = []
# Ensure that package is set in the raw frame, mapped from the
# debug_images array in the payload. Grouping and UI can use this path
# to infer in_app and exclude frames from grouping.
if raw_frame.get('package') is None:
obj = processable_frame.data['obj']
raw_frame['package'] = obj and obj.code_file or None
if processable_frame.cache_value is None:
# Construct a raw frame that is used by the symbolizer
# backend. We only assemble the bare minimum we need here.
instruction_addr = processable_frame.data['instruction_addr']
debug_id = processable_frame.data['debug_id']
if debug_id is not None:
self.difs_referenced.add(debug_id)
try:
symbolicated_frames = self.sym.symbolize_frame(
instruction_addr,
self.sdk_info,
symbolserver_match=processable_frame.data['symbolserver_match'],
symbolicator_match=processable_frame.data.get('symbolicator_match'),
trust=raw_frame.get('trust'),
)
if not symbolicated_frames:
if raw_frame.get('trust') == 'scan':
return [], [raw_frame], []
else:
return None, [raw_frame], []
except SymbolicationFailed as e:
errors = []
handle_symbolication_failed(e, data=self.data, errors=errors)
return [raw_frame], [raw_frame], errors
_ignored = None # Used to be in_app
processable_frame.set_cache_value([_ignored, symbolicated_frames])
else: # processable_frame.cache_value is present
_ignored, symbolicated_frames = processable_frame.cache_value
platform = raw_frame.get('platform') or self.data.get('platform')
new_frames = []
for sfrm in symbolicated_frames:
new_frame = dict(raw_frame)
merge_symbolicated_frame(new_frame, sfrm, platform=platform)
new_frames.append(new_frame)
return new_frames, [raw_frame], []
def resolve_frame_symbols(data):
debug_meta = data['debug_meta']
debug_images = debug_meta['images']
sdk_info = get_sdk_from_event(data)
stacktraces = find_all_stacktraces(data)
if not stacktraces:
return
project = Project.objects.get_from_cache(
id=data['project'],
)
errors = []
referenced_images = find_stacktrace_referenced_images(
debug_images, [x[0] for x in stacktraces])
sym = Symbolizer(project, debug_images,
referenced_images=referenced_images)
frame = None
idx = -1
def report_error(exc_type, exc_value, tb):
if exc_value.is_user_fixable or exc_value.is_sdk_failure:
errors.append({
'type': EventError.NATIVE_INTERNAL_FAILURE,
'frame': frame,
'error': u'frame #%d: %s' % (idx, exc_value)
})
if not exc_value.is_user_fixable:
logger.debug('Failed to symbolicate',
exc_info=(exc_type, exc_value, tb))
with sym:
for stacktrace, container in stacktraces:
store_raw = False
new_frames = list(stacktrace['frames'])
for idx, frame in enumerate(stacktrace['frames']):
if 'image_addr' not in frame or \
'instruction_addr' not in frame or \
'symbol_addr' not in frame:
continue
try:
# Construct a raw frame that is used by the symbolizer
# backend.
raw_frame = {
'object_name': frame.get('package'),
'object_addr': frame['image_addr'],
'instruction_addr': frame['instruction_addr'],
'symbol_addr': frame['symbol_addr'],
}
new_frame = dict(frame)
try:
sfrm = sym.symbolize_frame(raw_frame, sdk_info)
except SymbolicationFailed:
report_error(*sys.exc_info())
else:
symbol = sfrm.get('symbol_name') or \
new_frame.get('function') or '<unknown>'
function = demangle_symbol(symbol, simplified=True)
new_frame['function'] = function
# If we demangled something, store the original in the
# symbol portion of the frame
if function != symbol:
new_frame['symbol'] = symbol
new_frame['abs_path'] = sfrm.get('filename') or None
if new_frame['abs_path']:
new_frame['filename'] = posixpath.basename(
new_frame['abs_path'])
if sfrm.get('line') is not None:
new_frame['lineno'] = sfrm['line']
else:
new_frame['instruction_offset'] = \
parse_addr(sfrm['instruction_addr']) - \
parse_addr(sfrm['symbol_addr'])
if sfrm.get('column') is not None:
new_frame['colno'] = sfrm['column']
new_frame['package'] = sfrm['object_name'] \
or new_frame.get('package')
new_frame['symbol_addr'] = '0x%x' % \
parse_addr(sfrm['symbol_addr'])
new_frame['instruction_addr'] = '0x%x' % parse_addr(
sfrm['instruction_addr'])
new_frame['in_app'] = sym.is_in_app(raw_frame)
if new_frame != frame:
new_frames[idx] = new_frame
store_raw = True
except Exception:
logger.exception('Failed to symbolicate')
errors.append({
'type': EventError.NATIVE_INTERNAL_FAILURE,
'error': 'The symbolicator encountered an internal failure',
})
# Remember the raw stacktrace.
if store_raw and container is not None:
container['raw_stacktrace'] = {
'frames': stacktrace['frames'],
}
# Put the new frames in
stacktrace['frames'] = new_frames
if errors:
data.setdefault('errors', []).extend(errors)
return data
class NativeStacktraceProcessor(StacktraceProcessor):
supported_platforms = ('cocoa', 'native')
# TODO(ja): Clean up all uses of image type "apple", "uuid", "id" and "name"
supported_images = ('apple', 'symbolic', 'elf', 'macho', 'pe')
def __init__(self, *args, **kwargs):
StacktraceProcessor.__init__(self, *args, **kwargs)
# If true, the project has been opted into using the symbolicator
# service for native symbolication, which also means symbolic is not
# used at all anymore.
# The (iOS) symbolserver is still used regardless of this value.
self.use_symbolicator = _is_symbolicator_enabled(self.project)
self.arch = cpu_name_from_data(self.data)
self.signal = signal_from_data(self.data)
self.sym = None
self.difs_referenced = set()
images = get_path(self.data,
'debug_meta',
'images',
default=(),
filter=self._is_valid_image)
if images:
self.available = True
self.sdk_info = get_sdk_from_event(self.data)
self.object_lookup = ObjectLookup(images)
self.images = images
else:
self.available = False
def _is_valid_image(self, image):
# TODO(ja): Deprecate this. The symbolicator should take care of
# filtering valid images.
return bool(image) \
and image.get('type') in self.supported_images \
and image.get('image_addr') is not None \
and image.get('image_size') is not None \
and (image.get('debug_id') or image.get('id') or image.get('uuid')) is not None
def close(self):
StacktraceProcessor.close(self)
if self.difs_referenced:
metrics.incr(
'dsyms.processed',
amount=len(self.difs_referenced),
skip_internal=True,
tags={
'project_id': self.project.id,
},
)
def find_best_instruction(self, processable_frame):
"""Given a frame, stacktrace info and frame index this returns the
interpolated instruction address we then use for symbolication later.
"""
if self.arch is None:
return parse_addr(processable_frame['instruction_addr'])
crashing_frame = False
signal = None
ip_reg = None
# We only need to provide meta information for frame zero
if processable_frame.idx == 0:
# The signal is useful information for symbolic in some situations
# to disambiugate the first frame. If we can get this information
# from the mechanism we want to pass it onwards.
signal = self.signal
registers = processable_frame.stacktrace_info.stacktrace.get(
'registers')
if registers:
ip_reg_name = arch_get_ip_reg_name(self.arch)
if ip_reg_name:
ip_reg = registers.get(ip_reg_name)
crashing_frame = True
return find_best_instruction(processable_frame['instruction_addr'],
arch=self.arch,
crashing_frame=crashing_frame,
signal=signal,
ip_reg=ip_reg)
def handles_frame(self, frame, stacktrace_info):
platform = frame.get('platform') or self.data.get('platform')
return (platform in self.supported_platforms and self.available
and 'instruction_addr' in frame)
def preprocess_frame(self, processable_frame):
instr_addr = self.find_best_instruction(processable_frame)
obj = self.object_lookup.find_object(instr_addr)
processable_frame.data = {
'instruction_addr': instr_addr,
'obj': obj,
'debug_id': obj.debug_id if obj is not None else None,
'symbolserver_match': None,
# `[]` is used to indicate to the symbolizer that the symbolicator
# deliberately discarded this frame, while `None` means the
# symbolicator didn't run (because `self.use_symbolicator` is
# false).
# If the symbolicator did run and was not able to symbolize the
# frame, this value will be a list with the raw frame as only item.
'symbolicator_match': [] if self.use_symbolicator else None,
}
if obj is not None:
processable_frame.set_cache_key_from_values((
FRAME_CACHE_VERSION,
# Because the images can move around, we want to rebase
# the address for the cache key to be within the image
# the same way as we do it in the symbolizer.
rebase_addr(instr_addr, obj),
obj.debug_id,
obj.arch,
obj.size,
))
def preprocess_step(self, processing_task):
if not self.available:
return False
referenced_images = set(
pf.data['debug_id']
for pf in processing_task.iter_processable_frames(self)
if pf.cache_value is None and pf.data['debug_id'] is not None)
self.sym = Symbolizer(
self.project,
self.object_lookup,
referenced_images=referenced_images,
)
if options.get('symbolserver.enabled'):
self.fetch_system_symbols(processing_task)
if self.use_symbolicator:
self.run_symbolicator(processing_task)
def run_symbolicator(self, processing_task):
# TODO(markus): Make this work with minidumps. An unprocessed minidump
# event will not contain unsymbolicated frames, because the minidump
# upload already happened in store.
# It will also presumably not contain images, so `self.available` will
# already be `False`.
if not self.available:
return
request_id_cache_key = request_id_cache_key_for_event(self.data)
stacktraces = []
processable_stacktraces = []
for stacktrace_info, pf_list in processing_task.iter_processable_stacktraces(
):
registers = stacktrace_info.stacktrace.get('registers') or {}
# The filtering condition of this list comprehension is copied
# from `iter_processable_frames`.
#
# We cannot reuse `iter_processable_frames` because the
# symbolicator currently expects a list of stacktraces, not
# flat frames.
#
# Right now we can't even filter out frames (e.g. using a frame
# cache locally). The stacktraces have to be as complete as
# possible because the symbolicator assumes the first frame of
# a stacktrace to be the crashing frame. This assumption is
# already violated because the SDK might chop off frames though
# (which is less likely to be the case though).
pf_list = [pf for pf in reversed(pf_list) if pf.processor == self]
frames = []
for pf in pf_list:
frame = {'instruction_addr': pf['instruction_addr']}
if pf.get('trust') is not None:
frame['trust'] = pf['trust']
frames.append(frame)
stacktraces.append({'registers': registers, 'frames': frames})
processable_stacktraces.append(pf_list)
rv = run_symbolicator(stacktraces=stacktraces,
modules=self.images,
project=self.project,
arch=self.arch,
signal=self.signal,
request_id_cache_key=request_id_cache_key)
if not rv:
self.data \
.setdefault('errors', []) \
.extend(self._handle_symbolication_failed(
SymbolicationFailed(type=EventError.NATIVE_SYMBOLICATOR_FAILED)
))
return
# TODO(markus): Set signal and os context from symbolicator response,
# for minidumps
assert len(self.images) == len(rv['modules']), (self.images, rv)
for image, fetched_debug_file in zip(self.images, rv['modules']):
status = fetched_debug_file.pop('status')
# Set image data from symbolicator as symbolicator might know more
# than the SDK, especially for minidumps
if fetched_debug_file.get('arch') == 'unknown':
fetched_debug_file.pop('arch')
image.update(fetched_debug_file)
if status in ('found', 'unused'):
continue
elif status == 'missing_debug_file':
error = SymbolicationFailed(
type=EventError.NATIVE_MISSING_DSYM)
elif status == 'malformed_debug_file':
error = SymbolicationFailed(type=EventError.NATIVE_BAD_DSYM)
elif status == 'too_large':
error = SymbolicationFailed(type=EventError.FETCH_TOO_LARGE)
elif status == 'fetching_failed':
error = SymbolicationFailed(
type=EventError.FETCH_GENERIC_ERROR)
elif status == 'other':
error = SymbolicationFailed(type=EventError.UNKNOWN_ERROR)
else:
logger.error("Unknown status: %s", status)
continue
error.image_arch = image['arch']
error.image_path = image['code_file']
error.image_name = image_name(image['code_file'])
error.image_uuid = image['debug_id']
self.data.setdefault('errors', []) \
.extend(self._handle_symbolication_failed(error))
assert len(stacktraces) == len(rv['stacktraces'])
for pf_list, symbolicated_stacktrace in zip(processable_stacktraces,
rv['stacktraces']):
for symbolicated_frame in symbolicated_stacktrace.get(
'frames') or ():
pf = pf_list[symbolicated_frame['original_index']]
pf.data['symbolicator_match'].append(symbolicated_frame)
def fetch_system_symbols(self, processing_task):
to_lookup = []
pf_list = []
for pf in processing_task.iter_processable_frames(self):
obj = pf.data['obj']
if pf.cache_value is not None or obj is None or \
self.sym.is_image_from_app_bundle(obj):
continue
# We can only look up objects in the symbol server that have a
# uuid. If we encounter things with an age appended or
# similar we need to skip.
try:
uuid.UUID(obj.debug_id)
except (ValueError, TypeError):
continue
to_lookup.append({
'object_uuid':
obj.debug_id,
'object_name':
obj.name or '<unknown>',
'addr':
'0x%x' % rebase_addr(pf.data['instruction_addr'], obj)
})
pf_list.append(pf)
if not to_lookup:
return
rv = lookup_system_symbols(to_lookup, self.sdk_info, self.arch)
if rv is not None:
for symrv, pf in zip(rv, pf_list):
if symrv is None:
continue
pf.data['symbolserver_match'] = symrv
def _handle_symbolication_failed(self, e):
# User fixable but fatal errors are reported as processing
# issues
if e.is_user_fixable and e.is_fatal:
report_processing_issue(self.data,
scope='native',
object='dsym:%s' % e.image_uuid,
type=e.type,
data=e.get_data())
# This in many ways currently does not really do anything.
# The reason is that once a processing issue is reported
# the event will only be stored as a raw event and no
# group will be generated. As a result it also means that
# we will not have any user facing event or error showing
# up at all. We want to keep this here though in case we
# do not want to report some processing issues (eg:
# optional difs)
errors = []
if e.is_user_fixable or e.is_sdk_failure:
errors.append(e.get_data())
else:
logger.debug('Failed to symbolicate with native backend',
exc_info=True)
return errors
def process_frame(self, processable_frame, processing_task):
frame = processable_frame.frame
raw_frame = dict(frame)
errors = []
if processable_frame.cache_value is None:
# Construct a raw frame that is used by the symbolizer
# backend. We only assemble the bare minimum we need here.
instruction_addr = processable_frame.data['instruction_addr']
in_app = self.sym.is_in_app(instruction_addr,
sdk_info=self.sdk_info)
if in_app and raw_frame.get('function') is not None:
in_app = not self.sym.is_internal_function(
raw_frame['function'])
if raw_frame.get('in_app') is None:
raw_frame['in_app'] = in_app
debug_id = processable_frame.data['debug_id']
if debug_id is not None:
self.difs_referenced.add(debug_id)
try:
symbolicated_frames = self.sym.symbolize_frame(
instruction_addr,
self.sdk_info,
symbolserver_match=processable_frame.
data['symbolserver_match'],
symbolicator_match=processable_frame.data.get(
'symbolicator_match'),
trust=raw_frame.get('trust'),
)
if not symbolicated_frames:
if raw_frame.get('trust') == 'scan':
return [], [raw_frame], []
else:
return None, [raw_frame], []
except SymbolicationFailed as e:
errors = self._handle_symbolication_failed(e)
return [raw_frame], [raw_frame], errors
processable_frame.set_cache_value([in_app, symbolicated_frames])
else: # processable_frame.cache_value is present
in_app, symbolicated_frames = processable_frame.cache_value
raw_frame['in_app'] = in_app
new_frames = []
for sfrm in symbolicated_frames:
new_frame = dict(frame)
new_frame['function'] = sfrm['function']
if sfrm.get('symbol'):
new_frame['symbol'] = sfrm['symbol']
new_frame['abs_path'] = sfrm['abs_path']
new_frame['filename'] = sfrm.get('filename') or \
(sfrm['abs_path'] and posixpath.basename(sfrm['abs_path'])) or \
None
if sfrm.get('lineno'):
new_frame['lineno'] = sfrm['lineno']
if sfrm.get('colno'):
new_frame['colno'] = sfrm['colno']
if sfrm.get(
'package') or processable_frame.data['obj'] is not None:
new_frame['package'] = sfrm.get(
'package', processable_frame.data['obj'].name)
if new_frame.get('in_app') is None:
new_frame['in_app'] = in_app and \
not self.sym.is_internal_function(new_frame['function'])
new_frames.append(new_frame)
return new_frames, [raw_frame], []
class NativeStacktraceProcessor(StacktraceProcessor):
def __init__(self, *args, **kwargs):
StacktraceProcessor.__init__(self, *args, **kwargs)
debug_meta = self.data.get('debug_meta')
self.cpu_name = cpu_name_from_data(self.data)
self.sym = None
if debug_meta:
self.available = True
self.debug_meta = debug_meta
self.sdk_info = get_sdk_from_event(self.data)
self.image_lookup = ImageLookup(self.debug_meta['images'])
else:
self.available = False
def close(self):
StacktraceProcessor.close(self)
if self.sym is not None:
self.sym.close()
self.sym = None
def find_best_instruction(self, processable_frame):
"""Given a frame, stacktrace info and frame index this returns the
interpolated instruction address we then use for symbolication later.
"""
if self.cpu_name is None:
return parse_addr(processable_frame['instruction_addr'])
meta = None
# We only need to provide meta information for frame zero
if processable_frame.idx == 0:
# The signal is useful information for symsynd in some situations
# to disambiugate the first frame. If we can get this information
# from the mechanism we want to pass it onwards.
signal = None
exc = self.data.get('sentry.interfaces.Exception')
if exc is not None:
mechanism = exc['values'][0].get('mechanism')
if mechanism and 'posix_signal' in mechanism and \
'signal' in mechanism['posix_signal']:
signal = mechanism['posix_signal']['signal']
meta = {
'frame_number':
0,
'registers':
processable_frame.stacktrace_info.stacktrace.get('registers'),
'signal':
signal,
}
return find_best_instruction(processable_frame['instruction_addr'],
self.cpu_name,
meta=meta)
def handles_frame(self, frame, stacktrace_info):
platform = frame.get('platform') or self.data.get('platform')
return (platform == 'cocoa' and self.available
and 'instruction_addr' in frame)
def preprocess_frame(self, processable_frame):
instr_addr = self.find_best_instruction(processable_frame)
img = self.image_lookup.find_image(instr_addr)
processable_frame.data = {
'instruction_addr': instr_addr,
'image_uuid': img['uuid'] if img is not None else None,
}
if img is not None:
processable_frame.set_cache_key_from_values((
FRAME_CACHE_VERSION,
# Because the images can move around, we want to rebase
# the address for the cache key to be within the image
# the same way as we do it in the symbolizer.
(parse_addr(img['image_vmaddr']) + instr_addr -
parse_addr(img['image_addr'])),
img['uuid'].lower(),
img['cpu_type'],
img['cpu_subtype'],
img['image_size'],
))
def preprocess_step(self, processing_task):
if not self.available:
return False
referenced_images = set(
pf.data['image_uuid']
for pf in processing_task.iter_processable_frames(self)
if pf.cache_value is None and pf.data['image_uuid'] is not None)
self.sym = Symbolizer(self.project,
self.image_lookup,
cpu_name=self.cpu_name,
referenced_images=referenced_images)
# The symbolizer gets a reference to the debug meta's images so
# when it resolves the missing vmaddrs it changes them in the data
# dict.
return self.sym.resolve_missing_vmaddrs()
def process_frame(self, processable_frame, processing_task):
frame = processable_frame.frame
errors = []
new_frames = []
raw_frame = dict(frame)
if processable_frame.cache_value is None:
# Construct a raw frame that is used by the symbolizer
# backend. We only assemble the bare minimum we need here.
sym_input_frame = {
'object_name': frame.get('package'),
'instruction_addr': processable_frame.data['instruction_addr'],
'symbol_name': frame.get('function'),
}
in_app = self.sym.is_in_app(sym_input_frame)
raw_frame['in_app'] = in_app
try:
symbolicated_frames = self.sym.symbolize_frame(
sym_input_frame, self.sdk_info, symbolize_inlined=True)
if not symbolicated_frames:
return None, [raw_frame], []
except SymbolicationFailed as e:
errors = []
if e.is_user_fixable or e.is_sdk_failure:
errors.append({
'type': e.type,
'image_uuid': e.image_uuid,
'image_path': e.image_path,
'image_arch': e.image_arch,
'message': e.message,
})
else:
logger.debug('Failed to symbolicate with native backend',
exc_info=True)
return None, [raw_frame], errors
processable_frame.set_cache_value([in_app, symbolicated_frames])
else:
in_app, symbolicated_frames = processable_frame.cache_value
raw_frame['in_app'] = in_app
for sfrm in symbolicated_frames:
symbol = sfrm.get('symbol_name') or \
frame.get('function') or NATIVE_UNKNOWN_STRING
function = demangle_symbol(symbol, simplified=True)
new_frame = dict(frame)
new_frame['function'] = function
# If we demangled something, store the original in the
# symbol portion of the frame
if function != symbol:
new_frame['symbol'] = symbol
new_frame['abs_path'] = sfrm.get('filename') or None
if new_frame['abs_path']:
new_frame['filename'] = posixpath.basename(
new_frame['abs_path'])
if sfrm.get('line') is not None:
new_frame['lineno'] = sfrm['line']
if sfrm.get('column') is not None:
new_frame['colno'] = sfrm['column']
new_frame['package'] = sfrm['object_name'] \
or new_frame.get('package')
new_frame['in_app'] = in_app
new_frames.append(new_frame)
return new_frames, [raw_frame], []