def test_callback_is_invoked_on_read(self): chunk = ReadFileChunk.from_filename( self.filename, start_byte=0, chunk_size=3, callback=self.callback) chunk.read(1) chunk.read(1) chunk.read(1) self.assertEqual(self.amounts_seen, [1, 1, 1])
def multipart_upload(filename, part_size=PART_SIZE): glacier = boto3.resource('glacier', region_name='us-west-2') # There's no error if the vault already exists so we don't # need to catch any exceptions here. vault = glacier.create_vault(vaultName='botocore-integ-test-vault') file_size = os.path.getsize(filename) # Initiate a multipart upload multipart_upload = vault.initiate_multipart_upload( archiveDescription='multipart upload', partSize=str(part_size)) try: # Upload each part for i in range(file_size / part_size + 1): range_from = i * part_size range_to = min((i + 1) * part_size - 1, file_size - 1) body = ReadFileChunk.from_filename(filename, range_from, part_size) multipart_upload.upload_part(body=body, range='bytes %d-%d/*' % (range_from, range_to)) # Complete a multipart upload transaction response = multipart_upload.complete( checksum=calculate_tree_hash(open(filename, 'rb')), # NEEDED archiveSize=str(file_size)) return vault.Archive(response['archiveId']) except: multipart_upload.abort() raise
def multipart_upload(filename, part_size=PART_SIZE): glacier = boto3.resource('glacier', region_name='us-west-2') # There's no error if the vault already exists so we don't # need to catch any exceptions here. vault = glacier.create_vault(vaultName='botocore-integ-test-vault') file_size = os.path.getsize(filename) # Initiate a multipart upload multipart_upload = vault.initiate_multipart_upload( archiveDescription='multipart upload', partSize=str(part_size)) try: # Upload each part for i in range(file_size/part_size+1): range_from = i*part_size range_to = min((i+1)*part_size-1, file_size-1) body = ReadFileChunk.from_filename(filename, range_from, part_size) multipart_upload.upload_part( body=body, range='bytes %d-%d/*' % (range_from, range_to)) # Complete a multipart upload transaction response = multipart_upload.complete( checksum=calculate_tree_hash(open(filename, 'rb')), # NEEDED archiveSize=str(file_size)) return vault.Archive(response['archiveId']) except: multipart_upload.abort() raise
def test_read_entire_chunk(self): filename = os.path.join(self.tempdir, 'foo') with open(filename, 'wb') as f: f.write(b'onetwothreefourfivesixseveneightnineten') chunk = ReadFileChunk.from_filename( filename, start_byte=0, chunk_size=3) self.assertEqual(chunk.read(), b'one') self.assertEqual(chunk.read(), b'')
def test_iter_is_always_empty(self): # This tests the workaround for the httplib bug (see # the source for more info). filename = os.path.join(self.tempdir, 'foo') open(filename, 'wb').close() chunk = ReadFileChunk.from_filename( filename, start_byte=0, chunk_size=10) self.assertEqual(list(chunk), [])
def test_callback_can_be_disabled(self): chunk = ReadFileChunk.from_filename( self.filename, start_byte=0, chunk_size=3, callback=self.callback) chunk.disable_callback() # Now reading from the ReadFileChunk should not invoke # the callback. chunk.read() self.assertEqual(self.amounts_seen, [])
def test_file_chunk_supports_context_manager(self): filename = os.path.join(self.tempdir, 'foo') with open(filename, 'wb') as f: f.write(b'abc') with ReadFileChunk.from_filename(filename, start_byte=0, chunk_size=2) as chunk: val = chunk.read() self.assertEqual(val, b'ab')
def test_reset_stream_emulation(self): filename = os.path.join(self.tempdir, 'foo') with open(filename, 'wb') as f: f.write(b'onetwothreefourfivesixseveneightnineten') chunk = ReadFileChunk.from_filename( filename, start_byte=11, chunk_size=4) self.assertEqual(chunk.read(), b'four') chunk.seek(0) self.assertEqual(chunk.read(), b'four')
def test_callback_will_also_be_triggered_by_seek(self): chunk = ReadFileChunk.from_filename( self.filename, start_byte=0, chunk_size=3, callback=self.callback) chunk.read(2) chunk.seek(0) chunk.read(2) chunk.seek(1) chunk.read(2) self.assertEqual(self.amounts_seen, [2, -2, 2, -1, 2])
def test_tell_and_seek(self): filename = os.path.join(self.tempdir, 'foo') with open(filename, 'wb') as f: f.write(b'onetwothreefourfivesixseveneightnineten') chunk = ReadFileChunk.from_filename( filename, start_byte=36, chunk_size=100000) self.assertEqual(chunk.tell(), 0) self.assertEqual(chunk.read(), b'ten') self.assertEqual(chunk.tell(), 3) chunk.seek(0) self.assertEqual(chunk.tell(), 0)
def test_read_with_amount_size(self): filename = os.path.join(self.tempdir, 'foo') with open(filename, 'wb') as f: f.write(b'onetwothreefourfivesixseveneightnineten') chunk = ReadFileChunk.from_filename( filename, start_byte=11, chunk_size=4) self.assertEqual(chunk.read(1), b'f') self.assertEqual(chunk.read(1), b'o') self.assertEqual(chunk.read(1), b'u') self.assertEqual(chunk.read(1), b'r') self.assertEqual(chunk.read(1), b'')
def test_callback_can_be_disabled(self): filename = os.path.join(self.tempdir, 'foo') with open(filename, 'wb') as f: f.write(b'abc') callback_calls = [] def callback(amount): callback_calls.append(amount) chunk = ReadFileChunk.from_filename( filename, start_byte=0, chunk_size=3, callback=callback) chunk.disable_callback() # Now reading from the ReadFileChunk should not invoke # the callback. chunk.read() self.assertEqual(callback_calls, [])
def test_callback_is_invoked_on_read(self): filename = os.path.join(self.tempdir, 'foo') with open(filename, 'wb') as f: f.write(b'abc') amounts_seen = [] def callback(amount): amounts_seen.append(amount) chunk = ReadFileChunk.from_filename( filename, start_byte=0, chunk_size=3, callback=callback) chunk.read(1) chunk.read(1) chunk.read(1) self.assertEqual(amounts_seen, [1, 1, 1])
def test_callback_is_invoked_on_read(self): filename = os.path.join(self.tempdir, 'foo') with open(filename, 'wb') as f: f.write(b'abc') amounts_seen = [] def callback(amount): amounts_seen.append(amount) chunk = ReadFileChunk.from_filename(filename, start_byte=0, chunk_size=3, callback=callback) chunk.read(1) chunk.read(1) chunk.read(1) self.assertEqual(amounts_seen, [1, 1, 1])
def test_callback_can_be_disabled(self): filename = os.path.join(self.tempdir, 'foo') with open(filename, 'wb') as f: f.write(b'abc') callback_calls = [] def callback(amount): callback_calls.append(amount) chunk = ReadFileChunk.from_filename(filename, start_byte=0, chunk_size=3, callback=callback) chunk.disable_callback() # Now reading from the ReadFileChunk should not invoke # the callback. chunk.read() self.assertEqual(callback_calls, [])