def _client_side_chunk_join(self, final_path, chunk_list):
# If there's only one chunk, just "move" (copy and delete) the key and call it a day.
if len(chunk_list) == 1:
chunk_path = self._init_path(chunk_list[0].path)
abs_final_path = self._init_path(final_path)
# Let the copy raise an exception if it fails.
self._cloud_bucket.copy_key(abs_final_path, self._bucket_name, chunk_path)
# Attempt to clean up the old chunk.
try:
self._cloud_bucket.delete_key(chunk_path)
except IOError:
# We failed to delete a chunk. This sucks, but we shouldn't fail the push.
msg = "Failed to clean up chunk %s for move of %s"
logger.exception(msg, chunk_path, abs_final_path)
else:
# Concatenate and write all the chunks as one key.
concatenated = filelike.FilelikeStreamConcat(self._chunk_generator(chunk_list))
self.stream_write(final_path, concatenated)
# Attempt to clean up all the chunks.
for chunk in chunk_list:
try:
self._cloud_bucket.delete_key(self._init_path(chunk.path))
except IOError:
# We failed to delete a chunk. This sucks, but we shouldn't fail the push.
msg = "Failed to clean up chunk %s for reupload of %s"
logger.exception(msg, chunk.path, final_path)
def _client_side_chunk_join(self, final_path, chunk_list):
# If there's only one chunk, just "move" (copy and delete) the key and call it a day.
if len(chunk_list) == 1:
chunk_path = self._init_path(chunk_list[0].path)
abs_final_path = self._init_path(final_path)
# Let the copy raise an exception if it fails.
#
# TODO(kleesc): copy_from() is used instead of copy, since the latter is a managed transfer which uses S3's
# multipart api, which GCS and Rados does not support. Going forward, we should try moving
# non-aws implementations to use library's with better support (e.g GCS supports its own version
# of parallel uploads).
new_obj = self.get_cloud_bucket().Object(abs_final_path)
new_obj.copy_from(CopySource={"Bucket": self._bucket_name, "Key": chunk_path})
# Attempt to clean up the old chunk.
try:
self.get_cloud_bucket().Object(chunk_path).delete()
except (botocore.exceptions.ClientError, IOError):
# We failed to delete a chunk. This sucks, but we shouldn't fail the push.
msg = "Failed to clean up chunk %s for move of %s"
logger.exception(msg, chunk_path, abs_final_path)
else:
# Concatenate and write all the chunks as one key.
concatenated = filelike.FilelikeStreamConcat(self._chunk_generator(chunk_list))
self.stream_write(final_path, concatenated)
# Attempt to clean up all the chunks.
for chunk in chunk_list:
try:
self.get_cloud_bucket().Object(chunk.path).delete()
except (botocore.exceptions.ClientError, IOError):
# We failed to delete a chunk. This sucks, but we shouldn't fail the push.
msg = "Failed to clean up chunk %s for reupload of %s"
logger.exception(msg, chunk.path, final_path)
