def test_read_length(self):
io = IterIO(iter(['hello', 'world']))
for c in 'hel', 'low', 'orl', 'd':
assert io.read(3) == c
assert io.read(1) == ''
assert io.tell() == len('helloworld')
io = IterIO(iter(['hello', 'world']))
for c in 'hell', 'owor', 'ld':
assert io.read(4) == c
assert io.read(1) == ''
assert io.tell() == len('helloworld')
def test_deserializer(self):
deserializer = CSVDeserializer({}, None)
with open('tests/data/dummy_data.csv', 'rb') as f:
items = list(deserializer.deserialize(IterIO(f)))
expected_items = [
{'bar': 'hello', 'baz': 'world', 'id': '1'},
{'bar': 'foo', 'baz': 'bar', 'id': '2'},
{'bar': 'xdxd', 'baz': 'xdxd', 'id': '3'}
]
assert items == expected_items
def test_read_bytes(self):
io = IterIO(iter(['hello', 'world']))
for i, c in enumerate('helloworld'):
assert io.tell() == i
assert io.read(1) == c
assert io.read(1) == ''
assert io.tell() == len('helloworld')
def test_bypass_stream(self, write_stream_mock, open_stream_mock,
get_read_streams_mock, *othermocks):
# given
file_len = 50
file_obj = IterIO(BytesIO('a' * file_len))
get_read_streams_mock.return_value = [Stream('name', file_len, None)]
open_stream_mock.return_value = file_obj
options = create_stream_bypass_simple_config()
# when:
with closing(StreamBypass(options, meta())) as bypass:
bypass.execute()
# then:
write_stream_mock.assert_called_once_with(
Stream('name', file_len, None), file_obj)
self.assertEquals(bypass.bypass_state.stats['bytes_copied'], 50,
'Wrong number of bytes written')
def test_resume_bypass(self, write_stream_mock, open_stream_mock,
get_streams_mock, *othermocks):
# given
options = create_stream_bypass_simple_config()
options.persistence_options.update(
resume=True, persistence_state_id=self.tmp_bypass_resume_file)
options.persistence_options['options']['file_path'] = self.data_dir
file_len = 50
file_obj_b = IterIO(BytesIO('b' * file_len))
stream_a = Stream('file_a', file_len, None)
stream_b = Stream('file_b', file_len, None)
get_streams_mock.return_value = [stream_a, stream_b]
open_stream_mock.return_value = file_obj_b
# Initial state is:
# done = [(file_a, 50, None)] stats = {'bytes_copied': 50}
# when:
with closing(StreamBypass(options, meta())) as bypass:
bypass.execute()
# then:
write_stream_mock.assert_called_once_with(stream_b, file_obj_b)
assert bypass.bypass_state.stats['bytes_copied'] == 100,\
'Wrong number of bytes written'
def test_zlib_decompressor(self):
decompressor = ZLibDecompressor({}, None)
compressed = IterIO(BytesIO(zlib.compress('helloworld')))
assert IterIO(
decompressor.decompress(compressed)).read() == 'helloworld'
# Mutiple headers
decompressor = ZLibDecompressor({}, None)
compressed = zlib.compress('hello') + zlib.compress(
'world') + zlib.compress('foobar')
compressed = IterIO(BytesIO(compressed))
assert IterIO(
decompressor.decompress(compressed)).read() == 'helloworldfoobar'
# Parts are more than one chunk
decompressor = ZLibDecompressor({}, None)
parts = [randbytes(2**10), "hello", "world", randbytes(2**11)]
compressed = "".join(zlib.compress(part) for part in parts)
compressed = IterIO(BytesIO(compressed))
assert IterIO(
decompressor.decompress(compressed)).read() == "".join(parts)
def test_no_compression(self):
decompressor = NoDecompressor({}, None)
compressed = IterIO(BytesIO('helloworld'))
assert IterIO(
decompressor.decompress(compressed)).read() == 'helloworld'
def test_read_all(self):
io = IterIO(iter(['hello', 'world']))
assert io.read() == 'helloworld'
assert io.read() == ''
assert io.tell() == len('helloworld')
def test_line_mode(self):
io = IterIO(iter(['he\n\nllo', '\nworl\nd']), mode="lines")
assert list(io) == ['he\n', '\n', 'llo\n', 'worl\n', 'd']
def test_read_lines(self):
io = IterIO(iter(['he\n\nllo', '\nworl\nd']))
assert io.readlines() == ['he\n', '\n', 'llo\n', 'worl\n', 'd']