async def _chunked_upload(self, path: WaterButlerPath,
stream: streams.BaseStream) -> dict:
"""Upload a large file to Box over multiple requests. This method will be used if the
file to upload is larger than ``NONCHUNKED_UPLOAD_LIMIT``. Checksum verification is built
into this process, so manual verification is not needed.
API Docs: https://developer.box.com/reference#chunked-upload
"""
# Step 1: Add a sha1 calculator. The final sha1 will be needed to complete the session
stream.add_writer('sha1', streams.HashStreamWriter(hashlib.sha1))
# Step 2: Create an upload session with Box and recieve session id.
session_data = await self._create_chunked_upload_session(path, stream)
logger.debug('chunked upload session data: {}'.format(
json.dumps(session_data)))
metadata = None
try:
# Step 3. Split the data into parts and upload them to box.
parts_manifest = await self._upload_parts(stream, session_data)
logger.debug('chunked upload parts manifest: {}'.format(
json.dumps(parts_manifest)))
data_sha = base64.standard_b64encode(
stream.writers['sha1'].digest).decode()
# Step 4. Complete the session and return the uploaded file's metadata.
retry = self.UPLOAD_COMMIT_RETRIES
while retry > 0:
--retry
try:
metadata = await self._complete_chunked_upload_session(
session_data, parts_manifest, data_sha)
break
except RetryChunkedUploadCommit:
continue
except Exception as err:
msg = 'An unexpected error has occurred during the multi-part upload.'
logger.error('{} upload_id={} error={!r}'.format(
msg, session_data, err))
aborted = await self._abort_chunked_upload(session_data, data_sha)
if not aborted:
msg += ' The abort action failed to clean up the temporary file parts generated ' \
'during the upload process. Please manually remove them.'
raise exceptions.UploadError(msg)
return metadata
async def _chunked_upload(self, stream, path):
"""Uploads the given stream to S3 over multiple chunks
"""
# Step 1. Create a multi-part upload session
session_upload_id = await self._create_upload_session(path)
try:
# Step 2. Break stream into chunks and upload them one by one
parts_metadata = await self._upload_parts(stream, path, session_upload_id)
# Step 3. Commit the parts and end the upload session
await self._complete_multipart_upload(path, session_upload_id, parts_metadata)
except Exception as err:
msg = 'An unexpected error has occurred during the multi-part upload.'
logger.error('{} upload_id={} error={!r}'.format(msg, session_upload_id, err))
aborted = await self._abort_chunked_upload(path, session_upload_id)
if aborted:
msg += ' The abort action failed to clean up the temporary file parts generated ' \
'during the upload process. Please manually remove them.'
raise exceptions.UploadError(msg)