def _merge_archives(release_file: ReleaseFile, new_file: File,
new_archive: ReleaseArchive):
max_attempts = RELEASE_ARCHIVE_MAX_MERGE_ATTEMPTS
success = False
for attempt in range(max_attempts):
old_file = release_file.file
with ReleaseArchive(old_file.getfile().file) as old_archive:
buffer = BytesIO()
merge_release_archives(old_archive, new_archive, buffer)
replacement = File.objects.create(name=old_file.name,
type=old_file.type)
buffer.seek(0)
replacement.putfile(buffer)
with transaction.atomic():
release_file.refresh_from_db()
if release_file.file == old_file:
# Nothing has changed. It is safe to update
release_file.update(file=replacement)
success = True
break
else:
metrics.incr("tasks.assemble.merge_archives_retry",
instance=str(attempt))
else:
logger.error("Failed to merge archive in %s attempts, giving up.",
max_attempts)
if success:
old_file.delete()
new_file.delete()
def _merge_archives(release_file: ReleaseFile, new_file: File,
new_archive: ReleaseArchive):
lock_key = f"assemble:merge_archives:{release_file.id}"
lock = app.locks.get(lock_key, duration=60)
try:
with lock.blocking_acquire(RELEASE_ARCHIVE_MERGE_INITIAL_DELAY,
RELEASE_ARCHIVE_MERGE_TIMEOUT):
old_file = release_file.file
old_file_contents = ReleaseFile.cache.getfile(release_file)
buffer = BytesIO()
with metrics.timer("tasks.assemble.merge_archives_pure"):
did_merge = merge_release_archives(old_file_contents,
new_archive, buffer)
if did_merge:
replacement = File.objects.create(name=old_file.name,
type=old_file.type)
buffer.seek(0)
replacement.putfile(buffer)
release_file.update(file=replacement)
old_file.delete()
except UnableToAcquireLock as error:
logger.error("merge_archives.fail", extra={"error": error})
new_file.delete()
def _simple_update(release_file: ReleaseFile, new_file: File,
new_archive: ReleaseArchive,
additional_fields: dict) -> bool:
"""Update function used in _upsert_release_file"""
old_file = release_file.file
release_file.update(file=new_file, **additional_fields)
old_file.delete()
return True
def _simple_update(release_file: ReleaseFile, new_file: File,
new_archive: ReleaseArchive):
""" Update function used in _upsert_release_file """
old_file = release_file.file
release_file.update(file=new_file)
old_file.delete()
