def test_generate_multipart_md5(self):
with TmpFile() as tmp_filename:
with open(tmp_filename, 'w') as f:
f.truncate(10000)
md5 = calculate_multipart_etag(tmp_filename, 3333)
expected = '"728d2dbdd842b6a145cc3f3284d66861-4"'
self.assertEqual(md5, expected, 'Wrong calculated md5 for multipart upload')
def test_generate_multipart_md5(self):
with TmpFile() as tmp_filename:
with open(tmp_filename, 'w') as f:
f.truncate(10000)
md5 = calculate_multipart_etag(tmp_filename, 3333)
expected = '"728d2dbdd842b6a145cc3f3284d66861-4"'
self.assertEqual(md5, expected,
'Wrong calculated md5 for multipart upload')
def _check_multipart_copy_integrity(self, key, dest_bucket, dest_key_name, path):
from boto.exception import S3ResponseError
try:
dest_key = dest_bucket.get_key(dest_key_name)
md5 = calculate_multipart_etag(path, CHUNK_SIZE)
self._warn_if_etags_differ(key, dest_key, source_md5=md5)
except S3ResponseError:
self.logger.warning(
'Skipping copy integrity. We have no READ_ACP/WRITE_ACP permissions')
def _check_multipart_copy_integrity(self, key, dest_bucket, dest_key_name, path):
from boto.exception import S3ResponseError
try:
dest_key = dest_bucket.get_key(dest_key_name)
md5 = calculate_multipart_etag(path, CHUNK_SIZE)
self._warn_if_etags_differ(key, dest_key, source_md5=md5)
except S3ResponseError:
self.logger.warning(
'Skipping copy integrity. We have no READ_ACP/WRITE_ACP permissions')
def _upload_large_file(self, dump_path, key_name):
self.logger.debug('Using multipart S3 uploader')
md5 = None
if self.save_metadata:
md5 = calculate_multipart_etag(dump_path, CHUNK_SIZE)
metadata = self._create_key_metadata(dump_path, md5=md5)
with multipart_upload(self.bucket, key_name, metadata=metadata) as mp:
for chunk in split_file(dump_path):
self._upload_chunk(mp, chunk)
self.logger.debug(
'Uploaded chunk number {}'.format(chunk.number))
def _upload_large_file(self, dump_path, key_name):
self.logger.debug('Using multipart S3 uploader')
md5 = None
if self.save_metadata:
md5 = calculate_multipart_etag(dump_path, CHUNK_SIZE)
metadata = self._create_key_metadata(dump_path, md5=md5)
with multipart_upload(self.bucket, key_name, metadata=metadata) as mp:
for chunk in split_file(dump_path):
self._upload_chunk(mp, chunk)
self.logger.debug('Uploaded chunk number {}'.format(
chunk.number))
def _upload_large_file(self, dump_path, key_name):
self.logger.debug('Using multipart S3 uploader')
with multipart_upload(self.bucket, key_name) as mp:
for chunk in split_file(dump_path):
self._upload_chunk(mp, chunk)
self.logger.debug(
'Uploaded chunk number {}'.format(chunk.number))
with closing(self.bucket.get_key(key_name)) as key:
self._ensure_proper_key_permissions(key)
if self.save_metadata:
md5 = calculate_multipart_etag(dump_path, CHUNK_SIZE)
self._save_metadata_for_key(key, dump_path, md5=md5)
