def process_native_image(image): # NOTE that this is dead code as soon as Rust renormalization is fully # enabled. After that, this code should be deleted. There is a difference # TODO(untitaker): Remove with other normalization code. try: native_image = { 'code_file': image.get('code_file') or image.get('name'), 'debug_id': normalize_debug_id( image.get('debug_id') or image.get('id') or image.get('uuid')), 'image_addr': _addr(image.get('image_addr')), 'image_size': _addr(image.get('image_size')), 'image_vmaddr': _addr(image.get('image_vmaddr')), } if image.get('arch') is not None: native_image['arch'] = image.get('arch') if image.get('code_id') is not None: native_image['code_id'] = image.get('code_id') if image.get('debug_file') is not None: native_image['debug_file'] = image.get('debug_file') return native_image except KeyError as e: raise InterfaceValidationError('Missing value for symbolic image: %s' % e.args[0])
def from_object(cls, obj, path, name=None, debug_id=None): if debug_id is not None: try: debug_id = normalize_debug_id(debug_id) except SymbolicError: debug_id = None # Only allow overrides in the debug_id's age if the rest of the debug id # matches with what we determine from the object file. We generally # trust the server more than the client. obj_id = obj.debug_id if obj_id and debug_id and obj_id[:36] == debug_id[:36]: obj_id = debug_id return cls( file_format=obj.file_format, arch=obj.arch, debug_id=obj_id, code_id=obj.code_id, path=path, # TODO: Extract the object name from the object name=name, data={ "type": obj.kind, "features": list(obj.features) }, )
def get(self, request, project): """ List a Project's Debug Information Files ```````````````````````````````````````` Retrieve a list of debug information files for a given project. :pparam string organization_slug: the slug of the organization the file belongs to. :pparam string project_slug: the slug of the project to list the DIFs of. :qparam string query: If set, this parameter is used to locate DIFs with. :qparam string id: If set, the specified DIF will be sent in the response. :auth: required """ query = request.GET.get('query') queryset = ProjectDebugFile.objects.filter( project=project, ).select_related('file') if query: if len(query) <= 45: # If this query contains a debug identifier, normalize it to # allow for more lenient queries (e.g. supporting Breakpad ids). try: query = normalize_debug_id(query.strip()) except SymbolicError: pass q = Q(object_name__icontains=query) \ | Q(debug_id__icontains=query) \ | Q(code_id__icontains=query) \ | Q(cpu_name__icontains=query) \ | Q(file__headers__icontains=query) KNOWN_DIF_FORMATS_REVERSE = dict((v, k) for (k, v) in six.iteritems(KNOWN_DIF_FORMATS)) file_format = KNOWN_DIF_FORMATS_REVERSE.get(query) if file_format: q |= Q(file__headers__icontains=file_format) queryset = queryset.filter(q) download_requested = request.GET.get('id') is not None if download_requested and (request.access.has_scope('project:write')): return self.download(request.GET.get('id'), project) return self.paginate( request=request, queryset=queryset, order_by='-id', paginator_cls=OffsetPaginator, default_per_page=20, on_results=lambda x: serialize(x, request.user), )
def process_symbolic_image(image): try: symbolic_image = { 'id': normalize_debug_id(image.get('id')), 'image_addr': _addr(image.get('image_addr')), 'image_size': parse_addr(image['image_size']), 'image_vmaddr': _addr(image.get('image_vmaddr') or 0), 'name': image.get('name'), } if image.get('arch') is not None: symbolic_image['arch'] = image.get('arch') return symbolic_image except KeyError as e: raise InterfaceValidationError('Missing value for symbolic image: %s' % e.args[0])
def get_frames_for_symbolication(frames, data, modules): modules_by_debug_id = None rv = [] for frame in reversed(frames): if not _handles_frame(data, frame): continue s_frame = dict(frame) # validate and expand addressing modes. If we can't validate and # expand it, we keep None which is absolute. That's not great but # at least won't do damage. addr_mode = s_frame.pop("addr_mode", None) sanitized_addr_mode = None # None and abs mean absolute addressing explicitly. if addr_mode in (None, "abs"): pass # this is relative addressing to module by index or debug id. elif addr_mode.startswith("rel:"): arg = addr_mode[4:] idx = None if modules_by_debug_id is None: modules_by_debug_id = { x.get("debug_id"): idx for idx, x in enumerate(modules) } try: idx = modules_by_debug_id.get(normalize_debug_id(arg)) except ParseDebugIdError: pass if idx is None and arg.isdigit(): idx = int(arg) if idx is not None: sanitized_addr_mode = "rel:%d" % idx if sanitized_addr_mode is not None: s_frame["addr_mode"] = sanitized_addr_mode rv.append(s_frame) return rv
def get(self, request, project): """ List a Project's Debug Information Files ```````````````````````````````````````` Retrieve a list of debug information files for a given project. :pparam string organization_slug: the slug of the organization the file belongs to. :pparam string project_slug: the slug of the project to list the DIFs of. :qparam string query: If set, this parameter is used to locate DIFs with. :qparam string id: If set, the specified DIF will be sent in the response. :qparam string file_formats: If set, only DIFs with these formats will be returned. :auth: required """ download_requested = request.GET.get("id") is not None if download_requested and (request.access.has_scope("project:write")): return self.download(request.GET.get("id"), project) code_id = request.GET.get("code_id") debug_id = request.GET.get("debug_id") query = request.GET.get("query") file_formats = request.GET.getlist("file_formats") # If this query contains a debug identifier, normalize it to allow for # more lenient queries (e.g. supporting Breakpad ids). Use the index to # speed up such queries. if query and len(query) <= 45 and not debug_id: try: debug_id = normalize_debug_id(query.strip()) except SymbolicError: pass if debug_id: # If a debug ID is specified, do not consider the stored code # identifier and strictly filter by debug identifier. Often there # are mismatches in the code identifier in PEs. q = Q(debug_id__exact=debug_id) elif code_id: q = Q(code_id__exact=code_id) elif query: q = (Q(object_name__icontains=query) | Q(debug_id__icontains=query) | Q(code_id__icontains=query) | Q(cpu_name__icontains=query) | Q(file__headers__icontains=query)) known_file_format = DIF_MIMETYPES.get(query) if known_file_format: q |= Q(file__headers__icontains=known_file_format) else: q = Q() file_format_q = Q() for file_format in file_formats: known_file_format = DIF_MIMETYPES.get(file_format) if known_file_format: file_format_q |= Q(file__headers__icontains=known_file_format) q &= file_format_q queryset = ProjectDebugFile.objects.filter( q, project=project).select_related("file") return self.paginate( request=request, queryset=queryset, order_by="-id", paginator_cls=OffsetPaginator, default_per_page=20, on_results=lambda x: serialize(x, request.user), )
def get(self, request, project): """ List a Project's Debug Information Files ```````````````````````````````````````` Retrieve a list of debug information files for a given project. :pparam string organization_slug: the slug of the organization the file belongs to. :pparam string project_slug: the slug of the project to list the DIFs of. :qparam string query: If set, this parameter is used to locate DIFs with. :qparam string id: If set, the specified DIF will be sent in the response. :auth: required """ download_requested = request.GET.get('id') is not None if download_requested and (request.access.has_scope('project:write')): return self.download(request.GET.get('id'), project) code_id = request.GET.get('code_id') debug_id = request.GET.get('debug_id') query = request.GET.get('query') if code_id: # If a code identifier is provided, try to find an exact match and # only consider the debug identifier if the DIF does not have a # primary code identifier. q = Q(code_id__exact=code_id) if debug_id: q |= Q(code_id__exact=None, debug_id__exact=debug_id) elif debug_id: # If only a debug ID is specified, do not consider the stored code # identifier and strictly filter by debug identifier. q = Q(debug_id__exact=debug_id) elif query: if len(query) <= 45: # If this query contains a debug identifier, normalize it to # allow for more lenient queries (e.g. supporting Breakpad ids). try: query = normalize_debug_id(query.strip()) except SymbolicError: pass q = Q(object_name__icontains=query) \ | Q(debug_id__icontains=query) \ | Q(code_id__icontains=query) \ | Q(cpu_name__icontains=query) \ | Q(file__headers__icontains=query) KNOWN_DIF_FORMATS_REVERSE = dict((v, k) for (k, v) in six.iteritems(KNOWN_DIF_FORMATS)) file_format = KNOWN_DIF_FORMATS_REVERSE.get(query) if file_format: q |= Q(file__headers__icontains=file_format) else: q = Q() queryset = ProjectDebugFile.objects \ .filter(q, project=project) \ .select_related('file') return self.paginate( request=request, queryset=queryset, order_by='-id', paginator_cls=OffsetPaginator, default_per_page=20, on_results=lambda x: serialize(x, request.user), )
def detect_dif_from_path(path, name=None, debug_id=None, accept_unknown=False): """Detects which kind of Debug Information File (DIF) the file at `path` is. :param accept_unknown: If this is ``False`` an exception will be logged with the error when a file which is not a known DIF is found. This is useful for when ingesting ZIP files directly from Apple App Store Connect which you know will also contain files which are not DIFs. :returns: an array since an Archive can contain more than one Object. :raises BadDif: If the file is not a valid DIF. """ # proguard files (proguard/UUID.txt) or # (proguard/mapping-UUID.txt). proguard_id = _analyze_progard_filename(path) if proguard_id is not None: data = {"features": ["mapping"]} return [ DifMeta( file_format="proguard", arch="any", debug_id=proguard_id, code_id=None, path=path, name=name, data=data, ) ] dif_kind = determine_dif_kind(path) if dif_kind == DifKind.BcSymbolMap: if debug_id is None: # In theory we could also parse debug_id from the filename here. However we # would need to validate that it is a valid debug_id ourselves as symbolic does # not expose this yet. raise BadDif("Missing debug_id for BCSymbolMap") try: BcSymbolMap.open(path) except SymbolicError as e: logger.debug("File failed to load as BCSymbolmap: %s", path) raise BadDif("Invalid BCSymbolMap: %s" % e) else: logger.debug("File loaded as BCSymbolMap: %s", path) return [ DifMeta(file_format="bcsymbolmap", arch="any", debug_id=debug_id, name=name, path=path) ] elif dif_kind == DifKind.UuidMap: if debug_id is None: # Assume the basename is the debug_id, if it wasn't symbolic will fail. This is # required for when we get called for files extracted from a zipfile. basename = os.path.basename(path) try: debug_id = normalize_debug_id(os.path.splitext(basename)[0]) except SymbolicError as e: logger.debug("Filename does not look like a debug ID: %s", path) raise BadDif("Invalid UuidMap: %s" % e) try: UuidMapping.from_plist(debug_id, path) except SymbolicError as e: logger.debug("File failed to load as UUIDMap: %s", path) raise BadDif("Invalid UuidMap: %s" % e) else: logger.debug("File loaded as UUIDMap: %s", path) return [ DifMeta(file_format="uuidmap", arch="any", debug_id=debug_id, name=name, path=path) ] else: # native debug information files (MachO, ELF or Breakpad) try: archive = Archive.open(path) except ObjectErrorUnsupportedObject as e: raise BadDif("Unsupported debug information file: %s" % e) except SymbolicError as e: if accept_unknown: level = logging.DEBUG else: level = logging.WARNING logger.log(level, "dsymfile.bad-fat-object", exc_info=True) raise BadDif("Invalid debug information file: %s" % e) else: objs = [] for obj in archive.iter_objects(): objs.append( DifMeta.from_object(obj, path, name=name, debug_id=debug_id)) logger.debug("File is Archive with %s objects: %s", len(objs), path) return objs