def test_file_chunks_with_smaller_last_chunk(self):
with TmpFile() as tmp_filename:
with open(tmp_filename, 'w') as f:
f.truncate(10000)
chunks = list(split_file(tmp_filename, 3333))
self.assertEqual(len(chunks), 4,
'Incorrect number of chunks from file')
def _upload_large_file(self, dump_path, key_name):
self.logger.debug('Using multipart S3 uploader')
md5 = None
if self.save_metadata:
md5 = calculate_multipart_etag(dump_path, CHUNK_SIZE)
metadata = self._create_key_metadata(dump_path, md5=md5)
with multipart_upload(self.bucket, key_name, metadata=metadata) as mp:
for chunk in split_file(dump_path):
self._upload_chunk(mp, chunk)
self.logger.debug(
'Uploaded chunk number {}'.format(chunk.number))
def _upload_large_file(self, dump_path, key_name):
self.logger.debug('Using multipart S3 uploader')
md5 = None
if self.save_metadata:
md5 = calculate_multipart_etag(dump_path, CHUNK_SIZE)
metadata = self._create_key_metadata(dump_path, md5=md5)
with multipart_upload(self.bucket, key_name, metadata=metadata) as mp:
for chunk in split_file(dump_path):
self._upload_chunk(mp, chunk)
self.logger.debug('Uploaded chunk number {}'.format(
chunk.number))
def _upload_large_file(self, dump_path, key_name):
self.logger.debug('Using multipart S3 uploader')
with multipart_upload(self.bucket, key_name) as mp:
for chunk in split_file(dump_path):
self._upload_chunk(mp, chunk)
self.logger.debug(
'Uploaded chunk number {}'.format(chunk.number))
with closing(self.bucket.get_key(key_name)) as key:
self._ensure_proper_key_permissions(key)
if self.save_metadata:
md5 = calculate_multipart_etag(dump_path, CHUNK_SIZE)
self._save_metadata_for_key(key, dump_path, md5=md5)
def _upload_large_file(self, bucket, dump_path, key_name):
from boto.exception import S3ResponseError
self.logger.info('Using multipart S3 uploader')
with multipart_upload(bucket, key_name) as mp:
for chunk in split_file(dump_path):
self._upload_chunk(mp, chunk)
self.logger.info('Uploaded chunk number {}'.format(
chunk.number))
try:
with closing(bucket.get_key(key_name)) as key:
self._ensure_proper_key_permissions(key)
except S3ResponseError:
self.logger.warning('We could not ensure proper permissions. '
'We have no READ_ACP/WRITE_ACP permissions')
def _upload_large_file(self, bucket, dump_path, key_name):
from boto.exception import S3ResponseError
self.logger.info('Using multipart S3 uploader')
with multipart_upload(bucket, key_name) as mp:
for chunk in split_file(dump_path):
self._upload_chunk(mp, chunk)
self.logger.info(
'Uploaded chunk number {}'.format(chunk.number))
try:
with closing(bucket.get_key(key_name)) as key:
self._ensure_proper_key_permissions(key)
except S3ResponseError:
self.logger.warning(
'We could not ensure proper permissions. '
'We have no READ_ACP/WRITE_ACP permissions')
def test_file_chunks_with_smaller_last_chunk(self):
with TmpFile() as tmp_filename:
with open(tmp_filename, 'w') as f:
f.truncate(10000)
chunks = list(split_file(tmp_filename, 3333))
self.assertEqual(len(chunks), 4, 'Incorrect number of chunks from file')