def test_callback_is_invoked_on_read(self):
chunk = ReadFileChunk.from_filename(
self.filename, start_byte=0, chunk_size=3, callback=self.callback)
chunk.read(1)
chunk.read(1)
chunk.read(1)
self.assertEqual(self.amounts_seen, [1, 1, 1])
def test_callback_is_invoked_on_read(self):
chunk = ReadFileChunk.from_filename(
self.filename, start_byte=0, chunk_size=3, callback=self.callback)
chunk.read(1)
chunk.read(1)
chunk.read(1)
self.assertEqual(self.amounts_seen, [1, 1, 1])
def multipart_upload(filename, part_size=PART_SIZE):
glacier = boto3.resource('glacier', region_name='us-west-2')
# There's no error if the vault already exists so we don't
# need to catch any exceptions here.
vault = glacier.create_vault(vaultName='botocore-integ-test-vault')
file_size = os.path.getsize(filename)
# Initiate a multipart upload
multipart_upload = vault.initiate_multipart_upload(
archiveDescription='multipart upload', partSize=str(part_size))
try:
# Upload each part
for i in range(file_size / part_size + 1):
range_from = i * part_size
range_to = min((i + 1) * part_size - 1, file_size - 1)
body = ReadFileChunk.from_filename(filename, range_from, part_size)
multipart_upload.upload_part(body=body,
range='bytes %d-%d/*' %
(range_from, range_to))
# Complete a multipart upload transaction
response = multipart_upload.complete(
checksum=calculate_tree_hash(open(filename, 'rb')), # NEEDED
archiveSize=str(file_size))
return vault.Archive(response['archiveId'])
except:
multipart_upload.abort()
raise
def multipart_upload(filename, part_size=PART_SIZE):
glacier = boto3.resource('glacier', region_name='us-west-2')
# There's no error if the vault already exists so we don't
# need to catch any exceptions here.
vault = glacier.create_vault(vaultName='botocore-integ-test-vault')
file_size = os.path.getsize(filename)
# Initiate a multipart upload
multipart_upload = vault.initiate_multipart_upload(
archiveDescription='multipart upload', partSize=str(part_size))
try:
# Upload each part
for i in range(file_size/part_size+1):
range_from = i*part_size
range_to = min((i+1)*part_size-1, file_size-1)
body = ReadFileChunk.from_filename(filename, range_from, part_size)
multipart_upload.upload_part(
body=body, range='bytes %d-%d/*' % (range_from, range_to))
# Complete a multipart upload transaction
response = multipart_upload.complete(
checksum=calculate_tree_hash(open(filename, 'rb')), # NEEDED
archiveSize=str(file_size))
return vault.Archive(response['archiveId'])
except:
multipart_upload.abort()
raise
def test_read_entire_chunk(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'onetwothreefourfivesixseveneightnineten')
chunk = ReadFileChunk.from_filename(
filename, start_byte=0, chunk_size=3)
self.assertEqual(chunk.read(), b'one')
self.assertEqual(chunk.read(), b'')
def test_iter_is_always_empty(self):
# This tests the workaround for the httplib bug (see
# the source for more info).
filename = os.path.join(self.tempdir, 'foo')
open(filename, 'wb').close()
chunk = ReadFileChunk.from_filename(
filename, start_byte=0, chunk_size=10)
self.assertEqual(list(chunk), [])
def test_callback_can_be_disabled(self):
chunk = ReadFileChunk.from_filename(
self.filename, start_byte=0, chunk_size=3, callback=self.callback)
chunk.disable_callback()
# Now reading from the ReadFileChunk should not invoke
# the callback.
chunk.read()
self.assertEqual(self.amounts_seen, [])
def test_file_chunk_supports_context_manager(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'abc')
with ReadFileChunk.from_filename(filename, start_byte=0,
chunk_size=2) as chunk:
val = chunk.read()
self.assertEqual(val, b'ab')
def test_iter_is_always_empty(self):
# This tests the workaround for the httplib bug (see
# the source for more info).
filename = os.path.join(self.tempdir, 'foo')
open(filename, 'wb').close()
chunk = ReadFileChunk.from_filename(
filename, start_byte=0, chunk_size=10)
self.assertEqual(list(chunk), [])
def test_read_entire_chunk(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'onetwothreefourfivesixseveneightnineten')
chunk = ReadFileChunk.from_filename(
filename, start_byte=0, chunk_size=3)
self.assertEqual(chunk.read(), b'one')
self.assertEqual(chunk.read(), b'')
def test_callback_can_be_disabled(self):
chunk = ReadFileChunk.from_filename(
self.filename, start_byte=0, chunk_size=3, callback=self.callback)
chunk.disable_callback()
# Now reading from the ReadFileChunk should not invoke
# the callback.
chunk.read()
self.assertEqual(self.amounts_seen, [])
def test_file_chunk_supports_context_manager(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'abc')
with ReadFileChunk.from_filename(filename,
start_byte=0,
chunk_size=2) as chunk:
val = chunk.read()
self.assertEqual(val, b'ab')
def test_reset_stream_emulation(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'onetwothreefourfivesixseveneightnineten')
chunk = ReadFileChunk.from_filename(
filename, start_byte=11, chunk_size=4)
self.assertEqual(chunk.read(), b'four')
chunk.seek(0)
self.assertEqual(chunk.read(), b'four')
def test_callback_will_also_be_triggered_by_seek(self):
chunk = ReadFileChunk.from_filename(
self.filename, start_byte=0, chunk_size=3, callback=self.callback)
chunk.read(2)
chunk.seek(0)
chunk.read(2)
chunk.seek(1)
chunk.read(2)
self.assertEqual(self.amounts_seen, [2, -2, 2, -1, 2])
def test_reset_stream_emulation(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'onetwothreefourfivesixseveneightnineten')
chunk = ReadFileChunk.from_filename(
filename, start_byte=11, chunk_size=4)
self.assertEqual(chunk.read(), b'four')
chunk.seek(0)
self.assertEqual(chunk.read(), b'four')
def test_callback_will_also_be_triggered_by_seek(self):
chunk = ReadFileChunk.from_filename(
self.filename, start_byte=0, chunk_size=3, callback=self.callback)
chunk.read(2)
chunk.seek(0)
chunk.read(2)
chunk.seek(1)
chunk.read(2)
self.assertEqual(self.amounts_seen, [2, -2, 2, -1, 2])
def test_tell_and_seek(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'onetwothreefourfivesixseveneightnineten')
chunk = ReadFileChunk.from_filename(
filename, start_byte=36, chunk_size=100000)
self.assertEqual(chunk.tell(), 0)
self.assertEqual(chunk.read(), b'ten')
self.assertEqual(chunk.tell(), 3)
chunk.seek(0)
self.assertEqual(chunk.tell(), 0)
def test_read_with_amount_size(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'onetwothreefourfivesixseveneightnineten')
chunk = ReadFileChunk.from_filename(
filename, start_byte=11, chunk_size=4)
self.assertEqual(chunk.read(1), b'f')
self.assertEqual(chunk.read(1), b'o')
self.assertEqual(chunk.read(1), b'u')
self.assertEqual(chunk.read(1), b'r')
self.assertEqual(chunk.read(1), b'')
def test_read_with_amount_size(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'onetwothreefourfivesixseveneightnineten')
chunk = ReadFileChunk.from_filename(
filename, start_byte=11, chunk_size=4)
self.assertEqual(chunk.read(1), b'f')
self.assertEqual(chunk.read(1), b'o')
self.assertEqual(chunk.read(1), b'u')
self.assertEqual(chunk.read(1), b'r')
self.assertEqual(chunk.read(1), b'')
def test_tell_and_seek(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'onetwothreefourfivesixseveneightnineten')
chunk = ReadFileChunk.from_filename(
filename, start_byte=36, chunk_size=100000)
self.assertEqual(chunk.tell(), 0)
self.assertEqual(chunk.read(), b'ten')
self.assertEqual(chunk.tell(), 3)
chunk.seek(0)
self.assertEqual(chunk.tell(), 0)
def test_callback_can_be_disabled(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'abc')
callback_calls = []
def callback(amount):
callback_calls.append(amount)
chunk = ReadFileChunk.from_filename(
filename, start_byte=0, chunk_size=3, callback=callback)
chunk.disable_callback()
# Now reading from the ReadFileChunk should not invoke
# the callback.
chunk.read()
self.assertEqual(callback_calls, [])
def test_callback_is_invoked_on_read(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'abc')
amounts_seen = []
def callback(amount):
amounts_seen.append(amount)
chunk = ReadFileChunk.from_filename(
filename, start_byte=0, chunk_size=3, callback=callback)
chunk.read(1)
chunk.read(1)
chunk.read(1)
self.assertEqual(amounts_seen, [1, 1, 1])
def test_callback_is_invoked_on_read(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'abc')
amounts_seen = []
def callback(amount):
amounts_seen.append(amount)
chunk = ReadFileChunk.from_filename(filename,
start_byte=0,
chunk_size=3,
callback=callback)
chunk.read(1)
chunk.read(1)
chunk.read(1)
self.assertEqual(amounts_seen, [1, 1, 1])
def test_callback_can_be_disabled(self):
filename = os.path.join(self.tempdir, 'foo')
with open(filename, 'wb') as f:
f.write(b'abc')
callback_calls = []
def callback(amount):
callback_calls.append(amount)
chunk = ReadFileChunk.from_filename(filename,
start_byte=0,
chunk_size=3,
callback=callback)
chunk.disable_callback()
# Now reading from the ReadFileChunk should not invoke
# the callback.
chunk.read()
self.assertEqual(callback_calls, [])
