def detect_dif_from_path(path, name=None, debug_id=None):
"""This detects which kind of dif(Debug Information File) the path
provided is. It returns an array since an Archive can contain more than
one Object.
"""
# proguard files (proguard/UUID.txt) or
# (proguard/mapping-UUID.txt).
proguard_id = _analyze_progard_filename(path)
if proguard_id is not None:
data = {"features": ["mapping"]}
return [
DifMeta(
file_format="proguard",
arch="any",
debug_id=proguard_id,
code_id=None,
path=path,
name=name,
data=data,
)
]
# native debug information files (MachO, ELF or Breakpad)
try:
archive = Archive.open(path)
except ObjectErrorUnsupportedObject as e:
raise BadDif("Unsupported debug information file: %s" % e)
except SymbolicError as e:
logger.warning("dsymfile.bad-fat-object", exc_info=True)
raise BadDif("Invalid debug information file: %s" % e)
else:
objs = []
for obj in archive.iter_objects():
objs.append(DifMeta.from_object(obj, path, name=name, debug_id=debug_id))
return objs
def detect_dif_from_path(path, name=None):
"""This detects which kind of dif(Debug Information File) the path
provided is. It returns an array since an Archive can contain more than
one Object.
"""
# proguard files (proguard/UUID.txt) or
# (proguard/mapping-UUID.txt).
proguard_id = _analyze_progard_filename(path)
if proguard_id is not None:
data = {'features': ['mapping']}
return [DifMeta(
file_format='proguard',
arch='any',
debug_id=proguard_id,
code_id=None,
path=path,
name=name,
data=data,
)]
# native debug information files (MachO, ELF or Breakpad)
try:
archive = Archive.open(path)
except ObjectErrorUnsupportedObject as e:
raise BadDif("Unsupported debug information file: %s" % e)
except SymbolicError as e:
logger.warning('dsymfile.bad-fat-object', exc_info=True)
raise BadDif("Invalid debug information file: %s" % e)
else:
objs = []
for obj in archive.iter_objects():
objs.append(DifMeta.from_object(obj, path, name=name))
return objs
def _update_cachefile(self, debug_file, path, cls):
debug_id = debug_file.debug_id
# Skip silently if this cache cannot be computed from the given DIF
if not cls.computes_from(debug_file):
return None, None, None
# Locate the object inside the Archive. Since we have keyed debug
# files by debug_id, we expect a corresponding object. Otherwise, we
# fail silently, just like with missing symbols.
try:
archive = Archive.open(path)
obj = archive.get_object(debug_id=debug_id)
if obj is None:
return None, None, None
# Check features from the actual object file, if this is a legacy
# DIF where features have not been extracted yet.
if (debug_file.data or {}).get('features') is None:
if not set(cls.required_features) <= obj.features:
return None, None, None
cache = cls.cache_cls.from_object(obj)
except SymbolicError as e:
if not isinstance(e, cls.ignored_errors):
logger.error('dsymfile.%s-build-error' % cls.cache_name,
exc_info=True,
extra=dict(debug_id=debug_id))
metrics.incr('%s.failed' % cls.cache_name,
tags={
'error': e.__class__.__name__,
},
skip_internal=False)
return None, None, e.message
file = File.objects.create(name=debug_id,
type='project.%s' % cls.cache_name)
file.putfile(cache.open_stream())
# Try to insert the new Cache into the database. This only fail if
# (1) another process has concurrently added the same sym cache, or if
# (2) the debug symbol was deleted, either due to a newer upload or via
# the API.
try:
with transaction.atomic():
return cls.objects.create(
project=debug_file.project,
cache_file=file,
debug_file=debug_file,
checksum=debug_file.file.checksum,
version=cache.version,
), cache, None
except IntegrityError:
file.delete()
# Check for a concurrently inserted cache and use that instead. This
# could have happened (1) due to a concurrent insert, or (2) a new
# upload that has already succeeded to compute a cache. The latter
# case is extremely unlikely.
cache_file = cls.objects \
.filter(project=debug_file.project, debug_file__debug_id=debug_id) \
.select_related('cache_file') \
.order_by('-id') \
.first()
if cache_file is not None:
return cache_file, None, None
# There was no new cache, indicating that the debug file has been
# replaced with a newer version. Another job will create the
# corresponding cache eventually. To prevent querying the database
# another time, simply use the in-memory cache for now:
return None, cache, None
def _update_cachefile(self, debug_file, path, cls):
debug_id = debug_file.debug_id
# Skip silently if this cache cannot be computed from the given DIF
if not cls.computes_from(debug_file):
return None, None, None
# Locate the object inside the Archive. Since we have keyed debug
# files by debug_id, we expect a corresponding object. Otherwise, we
# fail silently, just like with missing symbols.
try:
archive = Archive.open(path)
obj = archive.get_object(debug_id=debug_id)
if obj is None:
return None, None, None
# Check features from the actual object file, if this is a legacy
# DIF where features have not been extracted yet.
if (debug_file.data or {}).get('features') is None:
if not set(cls.required_features) <= obj.features:
return None, None, None
cache = cls.cache_cls.from_object(obj)
except SymbolicError as e:
if not isinstance(e, cls.ignored_errors):
logger.error('dsymfile.%s-build-error' % cls.cache_name,
exc_info=True, extra=dict(debug_id=debug_id))
metrics.incr('%s.failed' % cls.cache_name, tags={
'error': e.__class__.__name__,
}, skip_internal=False)
return None, None, e.message
file = File.objects.create(name=debug_id, type='project.%s' % cls.cache_name)
file.putfile(cache.open_stream())
# Try to insert the new Cache into the database. This only fail if
# (1) another process has concurrently added the same sym cache, or if
# (2) the debug symbol was deleted, either due to a newer upload or via
# the API.
try:
with transaction.atomic():
return cls.objects.create(
project=debug_file.project,
cache_file=file,
debug_file=debug_file,
checksum=debug_file.file.checksum,
version=cache.version,
), cache, None
except IntegrityError:
file.delete()
# Check for a concurrently inserted cache and use that instead. This
# could have happened (1) due to a concurrent insert, or (2) a new
# upload that has already succeeded to compute a cache. The latter
# case is extremely unlikely.
cache_file = cls.objects \
.filter(project=debug_file.project, debug_file__debug_id=debug_id) \
.select_related('cache_file') \
.order_by('-id') \
.first()
if cache_file is not None:
return cache_file, None, None
# There was no new cache, indicating that the debug file has been
# replaced with a newer version. Another job will create the
# corresponding cache eventually. To prevent querying the database
# another time, simply use the in-memory cache for now:
return None, cache, None
def detect_dif_from_path(path, name=None, debug_id=None, accept_unknown=False):
"""Detects which kind of Debug Information File (DIF) the file at `path` is.
:param accept_unknown: If this is ``False`` an exception will be logged with the error
when a file which is not a known DIF is found. This is useful for when ingesting ZIP
files directly from Apple App Store Connect which you know will also contain files
which are not DIFs.
:returns: an array since an Archive can contain more than one Object.
:raises BadDif: If the file is not a valid DIF.
"""
# proguard files (proguard/UUID.txt) or
# (proguard/mapping-UUID.txt).
proguard_id = _analyze_progard_filename(path)
if proguard_id is not None:
data = {"features": ["mapping"]}
return [
DifMeta(
file_format="proguard",
arch="any",
debug_id=proguard_id,
code_id=None,
path=path,
name=name,
data=data,
)
]
dif_kind = determine_dif_kind(path)
if dif_kind == DifKind.BcSymbolMap:
if debug_id is None:
# In theory we could also parse debug_id from the filename here. However we
# would need to validate that it is a valid debug_id ourselves as symbolic does
# not expose this yet.
raise BadDif("Missing debug_id for BCSymbolMap")
try:
BcSymbolMap.open(path)
except SymbolicError as e:
logger.debug("File failed to load as BCSymbolmap: %s", path)
raise BadDif("Invalid BCSymbolMap: %s" % e)
else:
logger.debug("File loaded as BCSymbolMap: %s", path)
return [
DifMeta(file_format="bcsymbolmap",
arch="any",
debug_id=debug_id,
name=name,
path=path)
]
elif dif_kind == DifKind.UuidMap:
if debug_id is None:
# Assume the basename is the debug_id, if it wasn't symbolic will fail. This is
# required for when we get called for files extracted from a zipfile.
basename = os.path.basename(path)
try:
debug_id = normalize_debug_id(os.path.splitext(basename)[0])
except SymbolicError as e:
logger.debug("Filename does not look like a debug ID: %s",
path)
raise BadDif("Invalid UuidMap: %s" % e)
try:
UuidMapping.from_plist(debug_id, path)
except SymbolicError as e:
logger.debug("File failed to load as UUIDMap: %s", path)
raise BadDif("Invalid UuidMap: %s" % e)
else:
logger.debug("File loaded as UUIDMap: %s", path)
return [
DifMeta(file_format="uuidmap",
arch="any",
debug_id=debug_id,
name=name,
path=path)
]
else:
# native debug information files (MachO, ELF or Breakpad)
try:
archive = Archive.open(path)
except ObjectErrorUnsupportedObject as e:
raise BadDif("Unsupported debug information file: %s" % e)
except SymbolicError as e:
if accept_unknown:
level = logging.DEBUG
else:
level = logging.WARNING
logger.log(level, "dsymfile.bad-fat-object", exc_info=True)
raise BadDif("Invalid debug information file: %s" % e)
else:
objs = []
for obj in archive.iter_objects():
objs.append(
DifMeta.from_object(obj,
path,
name=name,
debug_id=debug_id))
logger.debug("File is Archive with %s objects: %s", len(objs),
path)
return objs