def test_compute_hash_tempfile_py3(self):
# Note the missing 'b' in the mode!
with tempfile.TemporaryFile(mode='w+') as f:
with self.assertRaises(ValueError):
compute_hashes_from_fileobj(f, chunk_size=512)
# What about file-like objects without a mode? If it has an
# encoding we use it, otherwise attempt UTF-8 encoding to
# bytes for hashing.
f = StringIO('test data' * 500)
compute_hashes_from_fileobj(f, chunk_size=512)
def test_compute_hash_tempfile_py3(self):
# Note the missing 'b' in the mode!
with tempfile.TemporaryFile(mode='w+') as f:
with self.assertRaises(ValueError):
compute_hashes_from_fileobj(f, chunk_size=512)
# What about file-like objects without a mode? If it has an
# encoding we use it, otherwise attempt UTF-8 encoding to
# bytes for hashing.
f = StringIO('test data' * 500)
compute_hashes_from_fileobj(f, chunk_size=512)
def test_compute_hash_tempfile(self):
# Compute a hash from a file object. On Python 2 this uses a non-
# binary mode. On Python 3, however, binary mode is required for
# binary files. If not used, you will get UTF-8 code errors.
if six.PY2:
mode = "w+"
else:
mode = "wb+"
with tempfile.TemporaryFile(mode=mode) as f:
f.write(self._gen_data())
f.seek(0)
compute_hashes_from_fileobj(f, chunk_size=512)
def test_compute_hash_tempfile(self):
# Compute a hash from a file object. On Python 2 this uses a non-
# binary mode. On Python 3, however, binary mode is required for
# binary files. If not used, you will get UTF-8 code errors.
if six.PY2:
mode = "w+"
else:
mode = "wb+"
with tempfile.TemporaryFile(mode=mode) as f:
f.write(self._gen_data())
f.seek(0)
compute_hashes_from_fileobj(f, chunk_size=512)
def archive_multi_upload(self, filenames=None):
"""Upload multiple files - does not use multipart upload to get around:
https://github.com/boto/boto/issues/2209"""
count = 0
resume = True if 'resume' in vars(self.args) and self.args.resume \
else False;
vault = self.connection.get_vault(self.args.vault)
if filenames is None:
filenames = list_files(self.args.directory)
if resume:
archives = list(self.cache.get_archive_list(self.args.vault))
for filename in filenames:
if resume and os.path.basename(filename) in archives:
continue
with open(filename, 'rb') as fileobj:
name = os.path.basename(filename)
info('Uploading archive %s' % name)
linear_hash, tree_hash = compute_hashes_from_fileobj(fileobj)
for upload_attempt in range(5):
try:
fileobj.seek(0)
response = vault.layer1.upload_archive(vault.name,
fileobj, linear_hash, tree_hash, name)
except boto.glacier.exceptions.UnexpectedHTTPResponseError:
warn('Upload failed - retrying for archive %s' % name)
else:
self.cache.add_archive(self.args.vault, name,
response['ArchiveId'], tree_hash)
break
else:
raise ConsoleError('Failed upload for archive %s' % name)
count += 1
info('%i archives uploaded' % count)
def archive_upload(self):
# XXX: "Leading whitespace in archive descriptions is removed."
# XXX: "The description must be less than or equal to 1024 bytes. The
# allowable characters are 7 bit ASCII without control codes,
# specifically ASCII values 32-126 decimal or 0x20-0x7E
# hexadecimal."
if self.args.name is not None:
name = self.args.name
else:
try:
full_name = self.args.file.name
except:
raise RuntimeError('Archive name not specified. Use --name')
name = os.path.basename(full_name)
vault = self.connection.get_vault(self.args.vault)
archive_id = vault.create_archive_from_file(
file_obj=self.args.file, description=name)
linear_hash, tree_hash = compute_hashes_from_fileobj(self.args.file)
self.cache.add_archive(self.args.vault, name, archive_id, tree_hash)
def test_compute_hash_bytesio(self):
# Compute a hash from a file-like BytesIO object.
f = BytesIO(self._gen_data())
compute_hashes_from_fileobj(f, chunk_size=512)
def test_compute_hash_stringio(self):
# Python 2 binary data in StringIO example
f = StringIO(self._gen_data())
compute_hashes_from_fileobj(f, chunk_size=512)
def test_compute_hash_bytesio(self):
# Compute a hash from a file-like BytesIO object.
f = BytesIO(self._gen_data())
compute_hashes_from_fileobj(f, chunk_size=512)
def test_compute_hash_stringio(self):
# Python 2 binary data in StringIO example
f = StringIO(self._gen_data())
compute_hashes_from_fileobj(f, chunk_size=512)
