def create_files_from_macho_zip(fileobj, project=None):
"""Creates all missing dsym files from the given zip file. This
returns a list of all files created.
"""
if not have_symsynd:
raise RuntimeError('symsynd is unavailable. Install sentry with '
'the dsym feature flag.')
scratchpad = tempfile.mkdtemp()
try:
safe_extract_zip(fileobj, scratchpad)
to_create = []
for dirpath, dirnames, filenames in os.walk(scratchpad):
for fn in filenames:
fn = os.path.join(dirpath, fn)
try:
uuids = get_macho_uuids(fn)
except (IOError, ValueError):
# Whatever was contained there, was probably not a
# macho file.
continue
for cpu, uuid in uuids:
to_create.append((cpu, uuid, fn))
rv = []
for cpu, uuid, filename in to_create:
with open(filename, 'rb') as f:
rv.append((_create_macho_dsym_from_uuid(
project, cpu, uuid, f, os.path.basename(filename))))
return rv
finally:
shutil.rmtree(scratchpad)
def create_files_from_macho_zip(fileobj, project=None):
"""Creates all missing dsym files from the given zip file. This
returns a list of all files created.
"""
if not have_symsynd:
raise RuntimeError('symsynd is unavailable. Install sentry with '
'the dsym feature flag.')
scratchpad = tempfile.mkdtemp()
try:
safe_extract_zip(fileobj, scratchpad)
to_create = []
for dirpath, dirnames, filenames in os.walk(scratchpad):
for fn in filenames:
fn = os.path.join(dirpath, fn)
try:
uuids = get_macho_uuids(fn)
except (IOError, ValueError):
# Whatever was contained there, was probably not a
# macho file.
continue
for cpu, uuid in uuids:
to_create.append((cpu, uuid, fn))
rv = []
for cpu, uuid, filename in to_create:
with open(filename, 'rb') as f:
rv.append(
(_create_macho_dsym_from_uuid(project, cpu, uuid, f,
os.path.basename(filename))))
return rv
finally:
shutil.rmtree(scratchpad)
def create_files_from_dif_zip(fileobj, project):
"""Creates all missing debug files from the given zip file. This
returns a list of all files created.
"""
scratchpad = tempfile.mkdtemp()
try:
safe_extract_zip(fileobj, scratchpad, strip_toplevel=False)
to_create = []
for dirpath, dirnames, filenames in os.walk(scratchpad):
for fn in filenames:
fn = os.path.join(dirpath, fn)
try:
difs = detect_dif_from_path(fn)
except BadDif:
difs = None
if difs is None:
difs = []
to_create = to_create + difs
rv = create_debug_file_from_dif(to_create, project)
# Uploading new dsysm changes the reprocessing revision
bump_reprocessing_revision(project)
return rv
finally:
shutil.rmtree(scratchpad)
def create_files_from_macho_zip(fileobj, project=None):
"""Creates all missing dsym files from the given zip file. This
returns a list of all files created.
"""
scratchpad = tempfile.mkdtemp()
try:
safe_extract_zip(fileobj, scratchpad)
to_create = []
for dirpath, dirnames, filenames in os.walk(scratchpad):
for fn in filenames:
fn = os.path.join(dirpath, fn)
try:
di = DebugInfo.open_path(fn)
except DebugInfoError:
# Whatever was contained there, was probably not a
# macho file.
continue
for variant in di.get_variants():
to_create.append((
variant.cpu_name,
str(variant.uuid), # noqa: B308
fn,
))
rv = []
for cpu, uuid, filename in to_create:
with open(filename, 'rb') as f:
rv.append(
(_create_macho_dsym_from_uuid(project, cpu, uuid, f,
os.path.basename(filename))))
return rv
finally:
shutil.rmtree(scratchpad)
def create_files_from_dsym_zip(fileobj, project,
update_symcaches=True):
"""Creates all missing dsym files from the given zip file. This
returns a list of all files created.
"""
scratchpad = tempfile.mkdtemp()
try:
safe_extract_zip(fileobj, scratchpad, strip_toplevel=False)
to_create = []
for dirpath, dirnames, filenames in os.walk(scratchpad):
for fn in filenames:
fn = os.path.join(dirpath, fn)
# proguard files (proguard/UUID.txt) or
# (proguard/mapping-UUID.txt).
proguard_uuid = _analyze_progard_filename(fn)
if proguard_uuid is not None:
to_create.append(('proguard', 'any', six.text_type(proguard_uuid), fn, ))
continue
# macho style debug symbols
try:
fo = FatObject.from_path(fn)
except UnsupportedObjectFile:
pass
except SymbolicError:
# Whatever was contained there, was probably not a
# macho file.
# XXX: log?
logger.warning('dsymfile.bad-fat-object', exc_info=True)
else:
for obj in fo.iter_objects():
to_create.append((obj.kind, obj.arch,
six.text_type(obj.uuid), fn))
continue
rv = []
for dsym_type, cpu, file_uuid, filename in to_create:
with open(filename, 'rb') as f:
dsym, created = _create_dsym_from_uuid(
project, dsym_type, cpu, file_uuid, f, os.path.basename(filename)
)
if created:
rv.append(dsym)
# By default we trigger the symcache generation on upload to avoid
# some obvious dogpiling.
if update_symcaches:
from sentry.tasks.symcache_update import symcache_update
uuids_to_update = [six.text_type(x.uuid) for x in rv
if x.supports_symcache]
if uuids_to_update:
symcache_update.delay(project_id=project.id,
uuids=uuids_to_update)
return rv
finally:
shutil.rmtree(scratchpad)
def extract(self) -> TemporaryDirectory:
"""Extract contents to a temporary directory.
The caller is responsible for cleanup of the temporary files.
"""
temp_dir = TemporaryDirectory()
safe_extract_zip(self._fileobj, temp_dir.name, strip_toplevel=False)
return temp_dir
def create_files_from_dsym_zip(fileobj, project=None):
"""Creates all missing dsym files from the given zip file. This
returns a list of all files created.
"""
scratchpad = tempfile.mkdtemp()
try:
safe_extract_zip(fileobj, scratchpad, strip_toplevel=False)
to_create = []
for dirpath, dirnames, filenames in os.walk(scratchpad):
for fn in filenames:
fn = os.path.join(dirpath, fn)
# proguard files (proguard/UUID.txt) or
# (proguard/mapping-UUID.txt).
proguard_uuid = _analyze_progard_filename(fn)
if proguard_uuid is not None:
to_create.append((
'proguard',
'any',
six.text_type(proguard_uuid),
fn,
))
# macho style debug symbols
try:
di = DebugInfo.open_path(fn)
except DebugInfoError:
# Whatever was contained there, was probably not a
# macho file.
pass
else:
for variant in di.get_variants():
to_create.append((
'macho',
variant.cpu_name,
six.text_type(variant.uuid),
fn,
))
continue
rv = []
for dsym_type, cpu, file_uuid, filename in to_create:
with open(filename, 'rb') as f:
dsym, created = _create_dsym_from_uuid(
project, dsym_type, cpu, file_uuid, f,
os.path.basename(filename))
if created:
rv.append(dsym)
return rv
finally:
shutil.rmtree(scratchpad)
def create_files_from_dsym_zip(fileobj, project=None):
"""Creates all missing dsym files from the given zip file. This
returns a list of all files created.
"""
scratchpad = tempfile.mkdtemp()
try:
safe_extract_zip(fileobj, scratchpad, strip_toplevel=False)
to_create = []
for dirpath, dirnames, filenames in os.walk(scratchpad):
for fn in filenames:
fn = os.path.join(dirpath, fn)
# proguard files (proguard/UUID.txt) or
# (proguard/mapping-UUID.txt).
proguard_uuid = _analyze_progard_filename(fn)
if proguard_uuid is not None:
to_create.append((
'proguard',
'any',
six.text_type(proguard_uuid),
fn,
))
# macho style debug symbols
try:
di = DebugInfo.open_path(fn)
except DebugInfoError:
# Whatever was contained there, was probably not a
# macho file.
pass
else:
for variant in di.get_variants():
to_create.append((
'macho',
variant.cpu_name,
six.text_type(variant.uuid),
fn,
))
continue
rv = []
for dsym_type, cpu, file_uuid, filename in to_create:
with open(filename, 'rb') as f:
dsym, created = _create_dsym_from_uuid(
project, dsym_type, cpu, file_uuid, f,
os.path.basename(filename))
if created:
rv.append(dsym)
return rv
finally:
shutil.rmtree(scratchpad)
def create_files_from_dif_zip(fileobj, project, update_caches=True):
"""Creates all missing debug files from the given zip file. This
returns a list of all files created.
"""
scratchpad = tempfile.mkdtemp()
try:
safe_extract_zip(fileobj, scratchpad, strip_toplevel=False)
to_create = []
for dirpath, dirnames, filenames in os.walk(scratchpad):
for fn in filenames:
fn = os.path.join(dirpath, fn)
try:
difs = detect_dif_from_path(fn)
except BadDif:
difs = None
if difs is None:
difs = []
to_create = to_create + difs
rv = create_debug_file_from_dif(to_create, project)
# Trigger generation of symcaches and cficaches to avoid dogpiling when
# events start coming in.
if update_caches:
from sentry.tasks.symcache_update import symcache_update
ids_to_update = [
six.text_type(dif.debug_id) for dif in rv
if dif.supports_caches
]
if ids_to_update:
symcache_update.delay(project_id=project.id,
debug_ids=ids_to_update)
# Uploading new dsysm changes the reprocessing revision
bump_reprocessing_revision(project)
return rv
finally:
shutil.rmtree(scratchpad)
def create_files_from_dsym_zip(fileobj, project,
update_symcaches=True):
"""Creates all missing dsym files from the given zip file. This
returns a list of all files created.
"""
scratchpad = tempfile.mkdtemp()
try:
safe_extract_zip(fileobj, scratchpad, strip_toplevel=False)
to_create = []
for dirpath, dirnames, filenames in os.walk(scratchpad):
for fn in filenames:
fn = os.path.join(dirpath, fn)
try:
difs = detect_dif_from_path(fn)
except BadDif:
difs = None
if difs is None:
difs = []
to_create = to_create + difs
rv = create_dsym_from_dif(to_create, project)
# By default we trigger the symcache generation on upload to avoid
# some obvious dogpiling.
if update_symcaches:
from sentry.tasks.symcache_update import symcache_update
ids_to_update = [six.text_type(dif.debug_id) for dif in rv
if dif.supports_symcache]
if ids_to_update:
symcache_update.delay(project_id=project.id,
debug_ids=ids_to_update)
# Uploading new dsysm changes the reprocessing revision
bump_reprocessing_revision(project)
return rv
finally:
shutil.rmtree(scratchpad)
def create_files_from_dsym_zip(fileobj, project, update_symcaches=True):
"""Creates all missing dsym files from the given zip file. This
returns a list of all files created.
"""
scratchpad = tempfile.mkdtemp()
try:
safe_extract_zip(fileobj, scratchpad, strip_toplevel=False)
to_create = []
for dirpath, dirnames, filenames in os.walk(scratchpad):
for fn in filenames:
fn = os.path.join(dirpath, fn)
try:
difs = detect_dif_from_path(fn)
except BadDif:
difs = None
if difs is None:
difs = []
to_create = to_create + difs
rv = create_dsym_from_dif(to_create, project)
# By default we trigger the symcache generation on upload to avoid
# some obvious dogpiling.
if update_symcaches:
from sentry.tasks.symcache_update import symcache_update
uuids_to_update = [
six.text_type(x.uuid) for x in rv if x.supports_symcache
]
if uuids_to_update:
symcache_update.delay(project_id=project.id,
uuids=uuids_to_update)
# Uploading new dsysm changes the reprocessing revision
bump_reprocessing_revision(project)
return rv
finally:
shutil.rmtree(scratchpad)
def assemble_artifacts(org_id, version, checksum, chunks, **kwargs):
"""
Creates release files from an uploaded artifact bundle.
"""
import shutil
import tempfile
from sentry.utils.zip import safe_extract_zip
from sentry.models import File, Organization, Release, ReleaseFile
organization = Organization.objects.get_from_cache(pk=org_id)
bind_organization_context(organization)
set_assemble_status(AssembleTask.ARTIFACTS, org_id, checksum,
ChunkFileState.ASSEMBLING)
# Assemble the chunks into a temporary file
rv = assemble_file(
AssembleTask.ARTIFACTS,
organization,
"release-artifacts.zip",
checksum,
chunks,
file_type="release.bundle",
)
# If not file has been created this means that the file failed to
# assemble because of bad input data. Return.
if rv is None:
return
bundle, temp_file = rv
scratchpad = tempfile.mkdtemp()
# Initially, always delete the bundle file. Later on, we can start to store
# the artifact bundle as a release file.
delete_bundle = True
try:
try:
safe_extract_zip(temp_file, scratchpad, strip_toplevel=False)
except BaseException:
raise AssembleArtifactsError("failed to extract bundle")
try:
manifest_path = path.join(scratchpad, "manifest.json")
with open(manifest_path, "rb") as manifest:
manifest = json.loads(manifest.read())
except BaseException:
raise AssembleArtifactsError("failed to open release manifest")
org_slug = manifest.get("org")
if organization.slug != org_slug:
raise AssembleArtifactsError(
"organization does not match uploaded bundle")
release_name = manifest.get("release")
if release_name != version:
raise AssembleArtifactsError(
"release does not match uploaded bundle")
try:
release = Release.objects.get(organization_id=organization.id,
version=release_name)
except Release.DoesNotExist:
raise AssembleArtifactsError("release does not exist")
dist_name = manifest.get("dist")
dist = None
if dist_name:
dist = release.add_dist(dist_name)
artifacts = manifest.get("files", {})
for rel_path, artifact in six.iteritems(artifacts):
artifact_url = artifact.get("url", rel_path)
artifact_basename = artifact_url.rsplit("/", 1)[-1]
file = File.objects.create(name=artifact_basename,
type="release.file",
headers=artifact.get("headers", {}))
full_path = path.join(scratchpad, rel_path)
with open(full_path, "rb") as fp:
file.putfile(fp, logger=logger)
kwargs = {
"organization_id": organization.id,
"release": release,
"name": artifact_url,
"dist": dist,
}
# Release files must have unique names within their release
# and dist. If a matching file already exists, replace its
# file with the new one; otherwise create it.
try:
release_file = ReleaseFile.objects.get(**kwargs)
except ReleaseFile.DoesNotExist:
try:
with transaction.atomic():
ReleaseFile.objects.create(file=file, **kwargs)
except IntegrityError:
# NB: This indicates a race, where another assemble task or
# file upload job has just created a conflicting file. Since
# we're upserting here anyway, yield to the faster actor and
# do not try again.
file.delete()
else:
old_file = release_file.file
release_file.update(file=file)
old_file.delete()
except AssembleArtifactsError as e:
set_assemble_status(AssembleTask.ARTIFACTS,
org_id,
checksum,
ChunkFileState.ERROR,
detail=six.text_type(e))
except BaseException:
logger.error("failed to assemble release bundle", exc_info=True)
set_assemble_status(
AssembleTask.ARTIFACTS,
org_id,
checksum,
ChunkFileState.ERROR,
detail="internal server error",
)
else:
set_assemble_status(AssembleTask.ARTIFACTS, org_id, checksum,
ChunkFileState.OK)
finally:
shutil.rmtree(scratchpad)
if delete_bundle:
bundle.delete()