def test_multipart_manager_generate_presigned_part_url_content_length(
multipart_manager: MultipartManager, ):
# TODO: make this work for Minio
upload_url = multipart_manager._generate_presigned_part_url(
'new-object', 'fake-upload-id', 1, 100)
# Ensure Content-Length is a signed header
assert 'content-length' in upload_url
def upload_complete_view(request: Request, upload_id: str) -> HttpResponseBase:
"""
Complete a multipart upload.
After all data has been uploaded using the URLs provided by initialize, this endpoint must
be called to create the object in the object store. A presigned URL that performs the
completion is returned, as the completion might take several minutes for large files.
"""
request_serializer = UploadCompletionRequestSerializer(data=request.data)
request_serializer.is_valid(raise_exception=True)
parts: List[TransferredPart] = request_serializer.save()
upload = get_object_or_404(Upload, upload_id=upload_id)
completion = TransferredParts(
object_key=upload.blob.name,
upload_id=str(upload.multipart_upload_id),
parts=parts,
)
completed_upload = MultipartManager.from_storage(AssetBlob.blob.field.storage).complete_upload(
completion
)
response_serializer = UploadCompletionResponseSerializer(
{
'complete_url': completed_upload.complete_url,
'body': completed_upload.body,
}
)
return Response(response_serializer.data)
def test_multipart_manager_initialize_upload(
multipart_manager: MultipartManager):
initialization = multipart_manager.initialize_upload(
'new-object',
100,
)
assert initialization
def test_multipart_manager_generate_presigned_complete_url(
multipart_manager: MultipartManager):
upload_url = multipart_manager._generate_presigned_complete_url(
TransferredParts(object_key='new-object',
upload_id='fake-upload-id',
parts=[]))
assert isinstance(upload_url, str)
def test_multipart_manager_initialize_upload(
multipart_manager: MultipartManager, content_type):
initialization = multipart_manager.initialize_upload(
'new-object',
100,
content_type=content_type,
)
assert initialization
def test_multipart_manager_finalize_upload(multipart_manager: MultipartManager,
file_size: int):
initialization = multipart_manager.initialize_upload(
'new-object',
file_size,
)
finalization = UploadFinalization(object_key=initialization.object_key,
upload_id=initialization.upload_id,
parts=[])
for part in initialization.parts:
resp = requests.put(part.upload_url, data=b'a' * part.size)
resp.raise_for_status()
finalization.parts.append(
PartFinalization(part_number=part.part_number,
size=part.size,
etag=resp.headers['ETag']))
multipart_manager.finalize_upload(finalization)
def test_multipart_manager_iter_part_sizes(file_size, requested_part_size,
initial_part_size, final_part_size,
part_count):
part_nums, part_sizes = zip(
*MultipartManager._iter_part_sizes(file_size, requested_part_size))
# TOOD: zip(*) returns a tuple, but semantically this should be a list
assert part_nums == tuple(range(1, part_count + 1))
assert all(part_size == initial_part_size for part_size in part_sizes[:-1])
assert part_sizes[-1] == final_part_size
def test_multipart_manager_get_object_size(storage,
multipart_manager: MultipartManager,
file_size: int):
key = f'object-with-size-{file_size}'
# Storage.save may rename this, if the key is not unique
key = storage.save(name=key, content=BytesIO(b'X' * file_size))
size = multipart_manager.get_object_size(object_key=key, )
assert size == file_size
storage.delete(key)
def test_multipart_manager_complete_upload(multipart_manager: MultipartManager,
file_size: int):
initialization = multipart_manager.initialize_upload(
'new-object',
file_size,
)
transferred_parts = TransferredParts(object_key=initialization.object_key,
upload_id=initialization.upload_id,
parts=[])
for part in initialization.parts:
resp = requests.put(part.upload_url, data=b'a' * part.size)
resp.raise_for_status()
transferred_parts.parts.append(
TransferredPart(part_number=part.part_number,
size=part.size,
etag=resp.headers['ETag']))
completed_upload = multipart_manager.complete_upload(transferred_parts)
assert completed_upload
assert completed_upload.complete_url
assert completed_upload.body
def initialize_multipart_upload(cls, etag, size):
upload_id = uuid4()
object_key = cls.object_key(upload_id)
multipart_initialization = MultipartManager.from_storage(
AssetBlob.blob.field.storage
).initialize_upload(object_key, size)
upload = cls(
upload_id=upload_id,
blob=object_key,
etag=etag,
size=size,
multipart_upload_id=multipart_initialization.upload_id,
)
return upload, {'upload_id': upload.upload_id, 'parts': multipart_initialization.parts}
def test_multipart_manager_generate_presigned_complete_body(
multipart_manager: MultipartManager):
body = multipart_manager._generate_presigned_complete_body(
TransferredParts(
object_key='new-object',
upload_id='fake-upload-id',
parts=[
TransferredPart(part_number=1, size=1, etag='fake-etag-1'),
TransferredPart(part_number=2, size=2, etag='fake-etag-2'),
],
))
assert body == (
'<?xml version="1.0" encoding="UTF-8"?>'
'<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
'<Part><PartNumber>1</PartNumber><ETag>fake-etag-1</ETag></Part>'
'<Part><PartNumber>2</PartNumber><ETag>fake-etag-2</ETag></Part>'
'</CompleteMultipartUpload>')
def test_multipart_manager_generate_presigned_part_url(
multipart_manager: MultipartManager):
upload_url = multipart_manager._generate_presigned_part_url(
'new-object', 'fake-upload-id', 1, 100)
assert isinstance(upload_url, str)
def test_multipart_manager_create_upload_id(
multipart_manager: MultipartManager):
upload_id = multipart_manager._create_upload_id('new-object')
assert isinstance(upload_id, str)
def test_multipart_manager_test_upload(multipart_manager: MultipartManager):
multipart_manager.test_upload()
def test_multipart_manager_supported_storage(storage: Storage):
assert MultipartManager.supported_storage(storage)
def test_multipart_manager_get_object_size_not_found(
multipart_manager: MultipartManager):
with pytest.raises(ObjectNotFoundError):
multipart_manager.get_object_size(object_key='no-such-object', )
def multipart_manager(storage: Storage) -> MultipartManager:
return MultipartManager.from_storage(storage)
def test_multipart_manager_supported_storage_unsupported():
storage = FileSystemStorage()
assert not MultipartManager.supported_storage(storage)