def process_native_image(image):
# NOTE that this is dead code as soon as Rust renormalization is fully
# enabled. After that, this code should be deleted. There is a difference
# TODO(untitaker): Remove with other normalization code.
try:
native_image = {
'code_file':
image.get('code_file') or image.get('name'),
'debug_id':
normalize_debug_id(
image.get('debug_id') or image.get('id') or image.get('uuid')),
'image_addr':
_addr(image.get('image_addr')),
'image_size':
_addr(image.get('image_size')),
'image_vmaddr':
_addr(image.get('image_vmaddr')),
}
if image.get('arch') is not None:
native_image['arch'] = image.get('arch')
if image.get('code_id') is not None:
native_image['code_id'] = image.get('code_id')
if image.get('debug_file') is not None:
native_image['debug_file'] = image.get('debug_file')
return native_image
except KeyError as e:
raise InterfaceValidationError('Missing value for symbolic image: %s' %
e.args[0])
def from_object(cls, obj, path, name=None, debug_id=None):
if debug_id is not None:
try:
debug_id = normalize_debug_id(debug_id)
except SymbolicError:
debug_id = None
# Only allow overrides in the debug_id's age if the rest of the debug id
# matches with what we determine from the object file. We generally
# trust the server more than the client.
obj_id = obj.debug_id
if obj_id and debug_id and obj_id[:36] == debug_id[:36]:
obj_id = debug_id
return cls(
file_format=obj.file_format,
arch=obj.arch,
debug_id=obj_id,
code_id=obj.code_id,
path=path,
# TODO: Extract the object name from the object
name=name,
data={
"type": obj.kind,
"features": list(obj.features)
},
)
def get(self, request, project):
"""
List a Project's Debug Information Files
````````````````````````````````````````
Retrieve a list of debug information files for a given project.
:pparam string organization_slug: the slug of the organization the
file belongs to.
:pparam string project_slug: the slug of the project to list the
DIFs of.
:qparam string query: If set, this parameter is used to locate DIFs with.
:qparam string id: If set, the specified DIF will be sent in the response.
:auth: required
"""
query = request.GET.get('query')
queryset = ProjectDebugFile.objects.filter(
project=project,
).select_related('file')
if query:
if len(query) <= 45:
# If this query contains a debug identifier, normalize it to
# allow for more lenient queries (e.g. supporting Breakpad ids).
try:
query = normalize_debug_id(query.strip())
except SymbolicError:
pass
q = Q(object_name__icontains=query) \
| Q(debug_id__icontains=query) \
| Q(code_id__icontains=query) \
| Q(cpu_name__icontains=query) \
| Q(file__headers__icontains=query)
KNOWN_DIF_FORMATS_REVERSE = dict((v, k) for (k, v) in six.iteritems(KNOWN_DIF_FORMATS))
file_format = KNOWN_DIF_FORMATS_REVERSE.get(query)
if file_format:
q |= Q(file__headers__icontains=file_format)
queryset = queryset.filter(q)
download_requested = request.GET.get('id') is not None
if download_requested and (request.access.has_scope('project:write')):
return self.download(request.GET.get('id'), project)
return self.paginate(
request=request,
queryset=queryset,
order_by='-id',
paginator_cls=OffsetPaginator,
default_per_page=20,
on_results=lambda x: serialize(x, request.user),
)
def process_symbolic_image(image):
try:
symbolic_image = {
'id': normalize_debug_id(image.get('id')),
'image_addr': _addr(image.get('image_addr')),
'image_size': parse_addr(image['image_size']),
'image_vmaddr': _addr(image.get('image_vmaddr') or 0),
'name': image.get('name'),
}
if image.get('arch') is not None:
symbolic_image['arch'] = image.get('arch')
return symbolic_image
except KeyError as e:
raise InterfaceValidationError('Missing value for symbolic image: %s' % e.args[0])
def process_symbolic_image(image):
try:
symbolic_image = {
'id': normalize_debug_id(image.get('id')),
'image_addr': _addr(image.get('image_addr')),
'image_size': parse_addr(image['image_size']),
'image_vmaddr': _addr(image.get('image_vmaddr') or 0),
'name': image.get('name'),
}
if image.get('arch') is not None:
symbolic_image['arch'] = image.get('arch')
return symbolic_image
except KeyError as e:
raise InterfaceValidationError('Missing value for symbolic image: %s' %
e.args[0])
def get_frames_for_symbolication(frames, data, modules):
modules_by_debug_id = None
rv = []
for frame in reversed(frames):
if not _handles_frame(data, frame):
continue
s_frame = dict(frame)
# validate and expand addressing modes. If we can't validate and
# expand it, we keep None which is absolute. That's not great but
# at least won't do damage.
addr_mode = s_frame.pop("addr_mode", None)
sanitized_addr_mode = None
# None and abs mean absolute addressing explicitly.
if addr_mode in (None, "abs"):
pass
# this is relative addressing to module by index or debug id.
elif addr_mode.startswith("rel:"):
arg = addr_mode[4:]
idx = None
if modules_by_debug_id is None:
modules_by_debug_id = {
x.get("debug_id"): idx
for idx, x in enumerate(modules)
}
try:
idx = modules_by_debug_id.get(normalize_debug_id(arg))
except ParseDebugIdError:
pass
if idx is None and arg.isdigit():
idx = int(arg)
if idx is not None:
sanitized_addr_mode = "rel:%d" % idx
if sanitized_addr_mode is not None:
s_frame["addr_mode"] = sanitized_addr_mode
rv.append(s_frame)
return rv
def process_native_image(image):
# NOTE that this is dead code as soon as Rust renormalization is fully
# enabled. After that, this code should be deleted. There is a difference
# TODO(untitaker): Remove with other normalization code.
try:
native_image = {
'code_file': image.get('code_file') or image.get('name'),
'debug_id': normalize_debug_id(
image.get('debug_id') or image.get('id') or image.get('uuid')),
'image_addr': _addr(image.get('image_addr')),
'image_size': _addr(image.get('image_size')),
'image_vmaddr': _addr(image.get('image_vmaddr')),
}
if image.get('arch') is not None:
native_image['arch'] = image.get('arch')
if image.get('code_id') is not None:
native_image['code_id'] = image.get('code_id')
if image.get('debug_file') is not None:
native_image['debug_file'] = image.get('debug_file')
return native_image
except KeyError as e:
raise InterfaceValidationError('Missing value for symbolic image: %s' % e.args[0])
def get(self, request, project):
"""
List a Project's Debug Information Files
````````````````````````````````````````
Retrieve a list of debug information files for a given project.
:pparam string organization_slug: the slug of the organization the
file belongs to.
:pparam string project_slug: the slug of the project to list the
DIFs of.
:qparam string query: If set, this parameter is used to locate DIFs with.
:qparam string id: If set, the specified DIF will be sent in the response.
:qparam string file_formats: If set, only DIFs with these formats will be returned.
:auth: required
"""
download_requested = request.GET.get("id") is not None
if download_requested and (request.access.has_scope("project:write")):
return self.download(request.GET.get("id"), project)
code_id = request.GET.get("code_id")
debug_id = request.GET.get("debug_id")
query = request.GET.get("query")
file_formats = request.GET.getlist("file_formats")
# If this query contains a debug identifier, normalize it to allow for
# more lenient queries (e.g. supporting Breakpad ids). Use the index to
# speed up such queries.
if query and len(query) <= 45 and not debug_id:
try:
debug_id = normalize_debug_id(query.strip())
except SymbolicError:
pass
if debug_id:
# If a debug ID is specified, do not consider the stored code
# identifier and strictly filter by debug identifier. Often there
# are mismatches in the code identifier in PEs.
q = Q(debug_id__exact=debug_id)
elif code_id:
q = Q(code_id__exact=code_id)
elif query:
q = (Q(object_name__icontains=query)
| Q(debug_id__icontains=query)
| Q(code_id__icontains=query)
| Q(cpu_name__icontains=query)
| Q(file__headers__icontains=query))
known_file_format = DIF_MIMETYPES.get(query)
if known_file_format:
q |= Q(file__headers__icontains=known_file_format)
else:
q = Q()
file_format_q = Q()
for file_format in file_formats:
known_file_format = DIF_MIMETYPES.get(file_format)
if known_file_format:
file_format_q |= Q(file__headers__icontains=known_file_format)
q &= file_format_q
queryset = ProjectDebugFile.objects.filter(
q, project=project).select_related("file")
return self.paginate(
request=request,
queryset=queryset,
order_by="-id",
paginator_cls=OffsetPaginator,
default_per_page=20,
on_results=lambda x: serialize(x, request.user),
)
def get(self, request, project):
"""
List a Project's Debug Information Files
````````````````````````````````````````
Retrieve a list of debug information files for a given project.
:pparam string organization_slug: the slug of the organization the
file belongs to.
:pparam string project_slug: the slug of the project to list the
DIFs of.
:qparam string query: If set, this parameter is used to locate DIFs with.
:qparam string id: If set, the specified DIF will be sent in the response.
:auth: required
"""
download_requested = request.GET.get('id') is not None
if download_requested and (request.access.has_scope('project:write')):
return self.download(request.GET.get('id'), project)
code_id = request.GET.get('code_id')
debug_id = request.GET.get('debug_id')
query = request.GET.get('query')
if code_id:
# If a code identifier is provided, try to find an exact match and
# only consider the debug identifier if the DIF does not have a
# primary code identifier.
q = Q(code_id__exact=code_id)
if debug_id:
q |= Q(code_id__exact=None, debug_id__exact=debug_id)
elif debug_id:
# If only a debug ID is specified, do not consider the stored code
# identifier and strictly filter by debug identifier.
q = Q(debug_id__exact=debug_id)
elif query:
if len(query) <= 45:
# If this query contains a debug identifier, normalize it to
# allow for more lenient queries (e.g. supporting Breakpad ids).
try:
query = normalize_debug_id(query.strip())
except SymbolicError:
pass
q = Q(object_name__icontains=query) \
| Q(debug_id__icontains=query) \
| Q(code_id__icontains=query) \
| Q(cpu_name__icontains=query) \
| Q(file__headers__icontains=query)
KNOWN_DIF_FORMATS_REVERSE = dict((v, k) for (k, v) in six.iteritems(KNOWN_DIF_FORMATS))
file_format = KNOWN_DIF_FORMATS_REVERSE.get(query)
if file_format:
q |= Q(file__headers__icontains=file_format)
else:
q = Q()
queryset = ProjectDebugFile.objects \
.filter(q, project=project) \
.select_related('file')
return self.paginate(
request=request,
queryset=queryset,
order_by='-id',
paginator_cls=OffsetPaginator,
default_per_page=20,
on_results=lambda x: serialize(x, request.user),
)
def detect_dif_from_path(path, name=None, debug_id=None, accept_unknown=False):
"""Detects which kind of Debug Information File (DIF) the file at `path` is.
:param accept_unknown: If this is ``False`` an exception will be logged with the error
when a file which is not a known DIF is found. This is useful for when ingesting ZIP
files directly from Apple App Store Connect which you know will also contain files
which are not DIFs.
:returns: an array since an Archive can contain more than one Object.
:raises BadDif: If the file is not a valid DIF.
"""
# proguard files (proguard/UUID.txt) or
# (proguard/mapping-UUID.txt).
proguard_id = _analyze_progard_filename(path)
if proguard_id is not None:
data = {"features": ["mapping"]}
return [
DifMeta(
file_format="proguard",
arch="any",
debug_id=proguard_id,
code_id=None,
path=path,
name=name,
data=data,
)
]
dif_kind = determine_dif_kind(path)
if dif_kind == DifKind.BcSymbolMap:
if debug_id is None:
# In theory we could also parse debug_id from the filename here. However we
# would need to validate that it is a valid debug_id ourselves as symbolic does
# not expose this yet.
raise BadDif("Missing debug_id for BCSymbolMap")
try:
BcSymbolMap.open(path)
except SymbolicError as e:
logger.debug("File failed to load as BCSymbolmap: %s", path)
raise BadDif("Invalid BCSymbolMap: %s" % e)
else:
logger.debug("File loaded as BCSymbolMap: %s", path)
return [
DifMeta(file_format="bcsymbolmap",
arch="any",
debug_id=debug_id,
name=name,
path=path)
]
elif dif_kind == DifKind.UuidMap:
if debug_id is None:
# Assume the basename is the debug_id, if it wasn't symbolic will fail. This is
# required for when we get called for files extracted from a zipfile.
basename = os.path.basename(path)
try:
debug_id = normalize_debug_id(os.path.splitext(basename)[0])
except SymbolicError as e:
logger.debug("Filename does not look like a debug ID: %s",
path)
raise BadDif("Invalid UuidMap: %s" % e)
try:
UuidMapping.from_plist(debug_id, path)
except SymbolicError as e:
logger.debug("File failed to load as UUIDMap: %s", path)
raise BadDif("Invalid UuidMap: %s" % e)
else:
logger.debug("File loaded as UUIDMap: %s", path)
return [
DifMeta(file_format="uuidmap",
arch="any",
debug_id=debug_id,
name=name,
path=path)
]
else:
# native debug information files (MachO, ELF or Breakpad)
try:
archive = Archive.open(path)
except ObjectErrorUnsupportedObject as e:
raise BadDif("Unsupported debug information file: %s" % e)
except SymbolicError as e:
if accept_unknown:
level = logging.DEBUG
else:
level = logging.WARNING
logger.log(level, "dsymfile.bad-fat-object", exc_info=True)
raise BadDif("Invalid debug information file: %s" % e)
else:
objs = []
for obj in archive.iter_objects():
objs.append(
DifMeta.from_object(obj,
path,
name=name,
debug_id=debug_id))
logger.debug("File is Archive with %s objects: %s", len(objs),
path)
return objs
