def _merge_archives(release_file: ReleaseFile, new_file: File, new_archive: ReleaseArchive): max_attempts = RELEASE_ARCHIVE_MAX_MERGE_ATTEMPTS success = False for attempt in range(max_attempts): old_file = release_file.file with ReleaseArchive(old_file.getfile().file) as old_archive: buffer = BytesIO() merge_release_archives(old_archive, new_archive, buffer) replacement = File.objects.create(name=old_file.name, type=old_file.type) buffer.seek(0) replacement.putfile(buffer) with transaction.atomic(): release_file.refresh_from_db() if release_file.file == old_file: # Nothing has changed. It is safe to update release_file.update(file=replacement) success = True break else: metrics.incr("tasks.assemble.merge_archives_retry", instance=str(attempt)) else: logger.error("Failed to merge archive in %s attempts, giving up.", max_attempts) if success: old_file.delete() new_file.delete()
def _upsert_release_file(file: File, archive: ReleaseArchive, update_fn, key_fields, additional_fields) -> bool: success = False release_file = None # Release files must have unique names within their release # and dist. If a matching file already exists, replace its # file with the new one; otherwise create it. try: release_file = ReleaseFile.objects.get(**key_fields) except ReleaseFile.DoesNotExist: try: with atomic_transaction(using=router.db_for_write(ReleaseFile)): release_file = ReleaseFile.objects.create( file=file, **dict(key_fields, **additional_fields)) except IntegrityError: # NB: This indicates a race, where another assemble task or # file upload job has just created a conflicting file. Since # we're upserting here anyway, yield to the faster actor and # do not try again. file.delete() else: success = True else: success = update_fn(release_file, file, archive, additional_fields) return success
def _merge_archives(release_file: ReleaseFile, new_file: File, new_archive: ReleaseArchive): lock_key = f"assemble:merge_archives:{release_file.id}" lock = app.locks.get(lock_key, duration=60) try: with lock.blocking_acquire(RELEASE_ARCHIVE_MERGE_INITIAL_DELAY, RELEASE_ARCHIVE_MERGE_TIMEOUT): old_file = release_file.file old_file_contents = ReleaseFile.cache.getfile(release_file) buffer = BytesIO() with metrics.timer("tasks.assemble.merge_archives_pure"): did_merge = merge_release_archives(old_file_contents, new_archive, buffer) if did_merge: replacement = File.objects.create(name=old_file.name, type=old_file.type) buffer.seek(0) replacement.putfile(buffer) release_file.update(file=replacement) old_file.delete() except UnableToAcquireLock as error: logger.error("merge_archives.fail", extra={"error": error}) new_file.delete()
def post(self, request, project, version): """ Upload a new file Upload a new file for the given release. {method} {path} name=http%3A%2F%2Fexample.com%2Fapplication.js # ... Unlike other API requests, files must be uploaded using the traditional multipart/form-data content-type. The optional 'name' attribute should reflect the absolute path that this file will be referenced as. For example, in the case of JavaScript you might specify the full web URI. """ release = Release.objects.get( project=project, version=version, ) if 'file' not in request.FILES: return Response(status=400) fileobj = request.FILES['file'] full_name = request.DATA.get('name', fileobj.name) name = full_name.rsplit('/', 1)[-1] # TODO(dcramer): File's are unique on (name, checksum) so we need to # ensure that this file does not already exist for other purposes file = File( name=name, type='release.file', headers={ 'Content-Type': fileobj.content_type, } ) file.putfile(fileobj) try: with transaction.atomic(): releasefile = ReleaseFile.objects.create( project=release.project, release=release, file=file, name=full_name, ) except IntegrityError: file.delete() return Response(status=409) return Response(serialize(releasefile, request.user), status=201)
def post(self, request, project, version): """ Upload a new file Upload a new file for the given release. {method} {path} name=http%3A%2F%2Fexample.com%2Fapplication.js # ... Unlike other API requests, files must be uploaded using the traditional multipart/form-data content-type. The optional 'name' attribute should reflect the absolute path that this file will be referenced as. For example, in the case of JavaScript you might specify the full web URI. """ release = Release.objects.get( project=project, version=version, ) if 'file' not in request.FILES: return Response(status=400) fileobj = request.FILES['file'] full_name = request.DATA.get('name', fileobj.name) name = full_name.rsplit('/', 1)[-1] # TODO(dcramer): File's are unique on (name, checksum) so we need to # ensure that this file does not already exist for other purposes file = File(name=name, type='release.file', headers={ 'Content-Type': fileobj.content_type, }) file.putfile(fileobj) try: with transaction.atomic(): releasefile = ReleaseFile.objects.create( project=release.project, release=release, file=file, name=full_name, ) except IntegrityError: file.delete() return Response(status=409) return Response(serialize(releasefile, request.user), status=201)
def _upsert_release_file(file: File, archive: ReleaseArchive, update_fn, **kwargs): release_file = None # Release files must have unique names within their release # and dist. If a matching file already exists, replace its # file with the new one; otherwise create it. try: release_file = ReleaseFile.objects.get(**kwargs) except ReleaseFile.DoesNotExist: try: with transaction.atomic(): release_file = ReleaseFile.objects.create(file=file, **kwargs) except IntegrityError: # NB: This indicates a race, where another assemble task or # file upload job has just created a conflicting file. Since # we're upserting here anyway, yield to the faster actor and # do not try again. file.delete() else: update_fn(release_file, file, archive)
def post(self, request, project, version): """ Upload a new file Upload a new file for the given release. {method} {path} name=http%3A%2F%2Fexample.com%2Fapplication.js &header=X-SourceMap%3A%20http%3A%2F%2Fexample.com%2Fapplication.js.map # ... Unlike other API requests, files must be uploaded using the traditional multipart/form-data content-type. The optional 'name' attribute should reflect the absolute path that this file will be referenced as. For example, in the case of JavaScript you might specify the full web URI. """ try: release = Release.objects.get( project=project, version=version, ) except Release.DoesNotExist: raise ResourceDoesNotExist if 'file' not in request.FILES: return Response({'detail': 'Missing uploaded file'}, status=400) fileobj = request.FILES['file'] full_name = request.DATA.get('name', fileobj.name) name = full_name.rsplit('/', 1)[-1] headers = { 'Content-Type': fileobj.content_type, } for headerval in request.DATA.getlist('header') or (): try: k, v = headerval.split(':', 1) except ValueError: return Response({'detail': 'header value was not formatted correctly'}, status=400) else: headers[k] = v.strip() # TODO(dcramer): File's are unique on (name, checksum) so we need to # ensure that this file does not already exist for other purposes file = File( name=name, type='release.file', headers=headers, ) file.putfile(fileobj) try: with transaction.atomic(): releasefile = ReleaseFile.objects.create( project=release.project, release=release, file=file, name=full_name, ) except IntegrityError: file.delete() return Response(status=409) return Response(serialize(releasefile, request.user), status=201)
def post(self, request, project, version): """ Upload a New File ````````````````` Upload a new file for the given release. Unlike other API requests, files must be uploaded using the traditional multipart/form-data content-type. The optional 'name' attribute should reflect the absolute path that this file will be referenced as. For example, in the case of JavaScript you might specify the full web URI. :pparam string organization_slug: the slug of the organization the release belongs to. :pparam string project_slug: the slug of the project to change the release of. :pparam string version: the version identifier of the release. :param string name: the name (full path) of the file. :param file file: the multipart encoded file. :param string header: this parameter can be supplied multiple times to attach headers to the file. Each header is a string in the format ``key:value``. For instance it can be used to define a content type. :auth: required """ try: release = Release.objects.get( project=project, version=version, ) except Release.DoesNotExist: raise ResourceDoesNotExist if 'file' not in request.FILES: return Response({'detail': 'Missing uploaded file'}, status=400) fileobj = request.FILES['file'] full_name = request.DATA.get('name', fileobj.name) name = full_name.rsplit('/', 1)[-1] headers = { 'Content-Type': fileobj.content_type, } for headerval in request.DATA.getlist('header') or (): try: k, v = headerval.split(':', 1) except ValueError: return Response({'detail': 'header value was not formatted correctly'}, status=400) else: headers[k] = v.strip() # TODO(dcramer): File's are unique on (name, checksum) so we need to # ensure that this file does not already exist for other purposes file = File( name=name, type='release.file', headers=headers, ) file.putfile(fileobj) try: with transaction.atomic(): releasefile = ReleaseFile.objects.create( project=release.project, release=release, file=file, name=full_name, ) except IntegrityError: file.delete() return Response(status=409) return Response(serialize(releasefile, request.user), status=201)
def post(self, request, project, version): """ Upload a New File ````````````````` Upload a new file for the given release. Unlike other API requests, files must be uploaded using the traditional multipart/form-data content-type. The optional 'name' attribute should reflect the absolute path that this file will be referenced as. For example, in the case of JavaScript you might specify the full web URI. :pparam string organization_slug: the slug of the organization the release belongs to. :pparam string project_slug: the slug of the project to change the release of. :pparam string version: the version identifier of the release. :param string name: the name (full path) of the file. :param file file: the multipart encoded file. :param string header: this parameter can be supplied multiple times to attach headers to the file. Each header is a string in the format ``key:value``. For instance it can be used to define a content type. :auth: required """ try: release = Release.objects.get( project=project, version=version, ) except Release.DoesNotExist: raise ResourceDoesNotExist if 'file' not in request.FILES: return Response({'detail': 'Missing uploaded file'}, status=400) fileobj = request.FILES['file'] full_name = request.DATA.get('name', fileobj.name) name = full_name.rsplit('/', 1)[-1] headers = { 'Content-Type': fileobj.content_type, } for headerval in request.DATA.getlist('header') or (): try: k, v = headerval.split(':', 1) except ValueError: return Response( {'detail': 'header value was not formatted correctly'}, status=400) else: headers[k] = v.strip() # TODO(dcramer): File's are unique on (name, checksum) so we need to # ensure that this file does not already exist for other purposes file = File( name=name, type='release.file', headers=headers, ) file.putfile(fileobj) try: with transaction.atomic(): releasefile = ReleaseFile.objects.create( project=release.project, release=release, file=file, name=full_name, ) except IntegrityError: file.delete() return Response(status=409) return Response(serialize(releasefile, request.user), status=201)