Exemple #1
0
 def test_read_length(self):
     io = IterIO(iter(['hello', 'world']))
     for c in 'hel', 'low', 'orl', 'd':
         assert io.read(3) == c
     assert io.read(1) == ''
     assert io.tell() == len('helloworld')
     io = IterIO(iter(['hello', 'world']))
     for c in 'hell', 'owor', 'ld':
         assert io.read(4) == c
     assert io.read(1) == ''
     assert io.tell() == len('helloworld')
    def test_deserializer(self):
        deserializer = CSVDeserializer({}, None)
        with open('tests/data/dummy_data.csv', 'rb') as f:
            items = list(deserializer.deserialize(IterIO(f)))

        expected_items = [
            {'bar': 'hello', 'baz': 'world', 'id': '1'},
            {'bar': 'foo', 'baz': 'bar', 'id': '2'},
            {'bar': 'xdxd', 'baz': 'xdxd', 'id': '3'}
        ]
        assert items == expected_items
Exemple #3
0
 def test_read_bytes(self):
     io = IterIO(iter(['hello', 'world']))
     for i, c in enumerate('helloworld'):
         assert io.tell() == i
         assert io.read(1) == c
     assert io.read(1) == ''
     assert io.tell() == len('helloworld')
Exemple #4
0
    def test_bypass_stream(self, write_stream_mock, open_stream_mock,
                           get_read_streams_mock, *othermocks):
        # given
        file_len = 50
        file_obj = IterIO(BytesIO('a' * file_len))
        get_read_streams_mock.return_value = [Stream('name', file_len, None)]
        open_stream_mock.return_value = file_obj
        options = create_stream_bypass_simple_config()

        # when:
        with closing(StreamBypass(options, meta())) as bypass:
            bypass.execute()

        # then:
        write_stream_mock.assert_called_once_with(
            Stream('name', file_len, None), file_obj)
        self.assertEquals(bypass.bypass_state.stats['bytes_copied'], 50,
                          'Wrong number of bytes written')
Exemple #5
0
    def test_resume_bypass(self, write_stream_mock, open_stream_mock,
                           get_streams_mock, *othermocks):
        # given
        options = create_stream_bypass_simple_config()
        options.persistence_options.update(
            resume=True, persistence_state_id=self.tmp_bypass_resume_file)
        options.persistence_options['options']['file_path'] = self.data_dir
        file_len = 50
        file_obj_b = IterIO(BytesIO('b' * file_len))
        stream_a = Stream('file_a', file_len, None)
        stream_b = Stream('file_b', file_len, None)
        get_streams_mock.return_value = [stream_a, stream_b]
        open_stream_mock.return_value = file_obj_b
        # Initial state is:
        # done = [(file_a, 50, None)] stats = {'bytes_copied': 50}

        # when:
        with closing(StreamBypass(options, meta())) as bypass:
            bypass.execute()

        # then:
        write_stream_mock.assert_called_once_with(stream_b, file_obj_b)
        assert bypass.bypass_state.stats['bytes_copied'] == 100,\
            'Wrong number of bytes written'
    def test_zlib_decompressor(self):
        decompressor = ZLibDecompressor({}, None)
        compressed = IterIO(BytesIO(zlib.compress('helloworld')))
        assert IterIO(
            decompressor.decompress(compressed)).read() == 'helloworld'

        # Mutiple headers
        decompressor = ZLibDecompressor({}, None)
        compressed = zlib.compress('hello') + zlib.compress(
            'world') + zlib.compress('foobar')
        compressed = IterIO(BytesIO(compressed))
        assert IterIO(
            decompressor.decompress(compressed)).read() == 'helloworldfoobar'

        # Parts are more than one chunk
        decompressor = ZLibDecompressor({}, None)
        parts = [randbytes(2**10), "hello", "world", randbytes(2**11)]
        compressed = "".join(zlib.compress(part) for part in parts)
        compressed = IterIO(BytesIO(compressed))
        assert IterIO(
            decompressor.decompress(compressed)).read() == "".join(parts)
 def test_no_compression(self):
     decompressor = NoDecompressor({}, None)
     compressed = IterIO(BytesIO('helloworld'))
     assert IterIO(
         decompressor.decompress(compressed)).read() == 'helloworld'
Exemple #8
0
 def test_read_all(self):
     io = IterIO(iter(['hello', 'world']))
     assert io.read() == 'helloworld'
     assert io.read() == ''
     assert io.tell() == len('helloworld')
Exemple #9
0
 def test_line_mode(self):
     io = IterIO(iter(['he\n\nllo', '\nworl\nd']), mode="lines")
     assert list(io) == ['he\n', '\n', 'llo\n', 'worl\n', 'd']
Exemple #10
0
 def test_read_lines(self):
     io = IterIO(iter(['he\n\nllo', '\nworl\nd']))
     assert io.readlines() == ['he\n', '\n', 'llo\n', 'worl\n', 'd']