def test_basic_unicode(self):
        io = IterIO([u"Hello", u"World", u"1", u"2", u"3"])
        self.assert_equal(io.tell(), 0)
        self.assert_equal(io.read(2), u"He")
        self.assert_equal(io.tell(), 2)
        self.assert_equal(io.read(3), u"llo")
        self.assert_equal(io.tell(), 5)
        io.seek(0)
        self.assert_equal(io.read(5), u"Hello")
        self.assert_equal(io.tell(), 5)
        self.assert_equal(io._buf, u"Hello")
        self.assert_equal(io.read(), u"World123")
        self.assert_equal(io.tell(), 13)
        io.close()
        assert io.closed

        io = IterIO([u"Hello\n", u"World!"])
        self.assert_equal(io.readline(), u'Hello\n')
        self.assert_equal(io._buf, u'Hello\n')
        self.assert_equal(io.read(), u'World!')
        self.assert_equal(io._buf, u'Hello\nWorld!')
        self.assert_equal(io.tell(), 12)
        io.seek(0)
        self.assert_equal(io.readlines(), [u'Hello\n', u'World!'])

        io = IterIO([u"foo\n", u"bar"])
        io.seek(-4, 2)
        self.assert_equal(io.read(4), u'\nbar')

        self.assert_raises(IOError, io.seek, 2, 100)
        io.close()
        self.assert_raises(ValueError, io.read)
    def test_basic_bytes(self):
        io = IterIO([b"Hello", b"World", b"1", b"2", b"3"])
        self.assert_equal(io.tell(), 0)
        self.assert_equal(io.read(2), b"He")
        self.assert_equal(io.tell(), 2)
        self.assert_equal(io.read(3), b"llo")
        self.assert_equal(io.tell(), 5)
        io.seek(0)
        self.assert_equal(io.read(5), b"Hello")
        self.assert_equal(io.tell(), 5)
        self.assert_equal(io._buf, b"Hello")
        self.assert_equal(io.read(), b"World123")
        self.assert_equal(io.tell(), 13)
        io.close()
        assert io.closed

        io = IterIO([b"Hello\n", b"World!"])
        self.assert_equal(io.readline(), b'Hello\n')
        self.assert_equal(io._buf, b'Hello\n')
        self.assert_equal(io.read(), b'World!')
        self.assert_equal(io._buf, b'Hello\nWorld!')
        self.assert_equal(io.tell(), 12)
        io.seek(0)
        self.assert_equal(io.readlines(), [b'Hello\n', b'World!'])

        io = IterIO([b"foo\n", b"bar"])
        io.seek(-4, 2)
        self.assert_equal(io.read(4), b'\nbar')

        self.assert_raises(IOError, io.seek, 2, 100)
        io.close()
        self.assert_raises(ValueError, io.read)
Ejemplo n.º 3
0
    def test_basic_bytes(self):
        io = IterIO([b"Hello", b"World", b"1", b"2", b"3"])
        assert io.tell() == 0
        assert io.read(2) == b"He"
        assert io.tell() == 2
        assert io.read(3) == b"llo"
        assert io.tell() == 5
        io.seek(0)
        assert io.read(5) == b"Hello"
        assert io.tell() == 5
        assert io._buf == b"Hello"
        assert io.read() == b"World123"
        assert io.tell() == 13
        io.close()
        assert io.closed

        io = IterIO([b"Hello\n", b"World!"])
        assert io.readline() == b'Hello\n'
        assert io._buf == b'Hello\n'
        assert io.read() == b'World!'
        assert io._buf == b'Hello\nWorld!'
        assert io.tell() == 12
        io.seek(0)
        assert io.readlines() == [b'Hello\n', b'World!']

        io = IterIO([b"foo\n", b"bar"])
        io.seek(-4, 2)
        assert io.read(4) == b'\nbar'

        pytest.raises(IOError, io.seek, 2, 100)
        io.close()
        pytest.raises(ValueError, io.read)
Ejemplo n.º 4
0
    def test_basic_unicode(self):
        io = IterIO([u"Hello", u"World", u"1", u"2", u"3"])
        assert io.tell() == 0
        assert io.read(2) == u"He"
        assert io.tell() == 2
        assert io.read(3) == u"llo"
        assert io.tell() == 5
        io.seek(0)
        assert io.read(5) == u"Hello"
        assert io.tell() == 5
        assert io._buf == u"Hello"
        assert io.read() == u"World123"
        assert io.tell() == 13
        io.close()
        assert io.closed

        io = IterIO([u"Hello\n", u"World!"])
        assert io.readline() == u'Hello\n'
        assert io._buf == u'Hello\n'
        assert io.read() == u'World!'
        assert io._buf == u'Hello\nWorld!'
        assert io.tell() == 12
        io.seek(0)
        assert io.readlines() == [u'Hello\n', u'World!']

        io = IterIO([u"foo\n", u"bar"])
        io.seek(-4, 2)
        assert io.read(4) == u'\nbar'

        pytest.raises(IOError, io.seek, 2, 100)
        io.close()
        pytest.raises(ValueError, io.read)
Ejemplo n.º 5
0
    def test_basic(self):
        io = IterIO(["Hello", "World", "1", "2", "3"])
        assert io.tell() == 0
        assert io.read(2) == "He"
        assert io.tell() == 2
        assert io.read(3) == "llo"
        assert io.tell() == 5
        io.seek(0)
        assert io.read(5) == "Hello"
        assert io.tell() == 5
        assert io._buf == "Hello"
        assert io.read() == "World123"
        assert io.tell() == 13
        io.close()
        assert io.closed

        io = IterIO(["Hello\n", "World!"])
        assert io.readline() == 'Hello\n'
        assert io._buf == 'Hello\n'
        assert io.read() == 'World!'
        assert io._buf == 'Hello\nWorld!'
        assert io.tell() == 12
        io.seek(0)
        assert io.readlines() == ['Hello\n', 'World!']

        io = IterIO(["foo\n", "bar"])
        io.seek(-4, 2)
        assert io.read(4) == '\nbar'

        self.assert_raises(IOError, io.seek, 2, 100)
        io.close()
        self.assert_raises(ValueError, io.read)
Ejemplo n.º 6
0
def test_itero():
    """Test the IterIO"""
    iterable = iter(["Hello", "World", "1", "2", "3"])
    io = IterIO(iterable)
    assert io.tell() == 0
    assert io.read(2) == "He"
    assert io.tell() == 2
    assert io.read(3) == "llo"
    assert io.tell() == 5
    io.seek(0)
    assert io.read(5) == "Hello"
    assert io.tell() == 5
    assert io._buf == "Hello"
    assert io.read() == "World123"
    assert io.tell() == 13
    io.close()
    assert io.closed

    io = IterIO(iter(["Hello\n", "World!"]))
    assert io.readline() == 'Hello\n'
    assert io._buf == 'Hello\n'
    assert io.read() == 'World!'
    assert io._buf == 'Hello\nWorld!'
    assert io.tell() == 12
    io.seek(0)
    assert io.readlines() == ['Hello\n', 'World!']

    io = IterIO(iter(["foo\n", "bar"]))
    io.seek(-4, 2)
    assert io.read(4) == '\nbar'

    assert_raises(IOError, io.seek, 2, 100)
    io.close()
    assert_raises(ValueError, io.read)
Ejemplo n.º 7
0
    def test_sentinel_cases(self):
        def producer_dummy_flush(out):
            out.flush()
        iterable = IterIO(producer_dummy_flush)
        self.assert_strict_equal(next(iterable), '')

        def producer_empty(out):
            pass
        iterable = IterIO(producer_empty)
        self.assert_raises(StopIteration, next, iterable)

        iterable = IterIO(producer_dummy_flush, b'')
        self.assert_strict_equal(next(iterable), b'')
        iterable = IterIO(producer_dummy_flush, u'')
        self.assert_strict_equal(next(iterable), u'')
Ejemplo n.º 8
0
def download_ed_catalog(catalog_url):
    print('download_ed_catalog({})', catalog_url)
    catalog_res = requests.get(catalog_url, stream=True)
    print('response: {}', catalog_res)

    # Read the request content, zip file and XML file via a stream to prevent
    # reading the whole data into memory at once. The memory used should be
    # constant regardless of the size of the input data.
    if not catalog_url.endswith('.zip'):
        catalog_xml = IterIO(catalog_res.iter_lines)
    else:
        with ZipFile(IterIO(catalog_res.iter_content(chunk_size=4096), sentinel=b'')) as zf:
            with zf.open(zf.filelist[0].filename, 'r') as xml_file:
                catalog_xml = xml_file
    return catalog_xml
Ejemplo n.º 9
0
    def response(self, flow):
        if flow.request.host == self.fwd_host:
            return

        if hasattr(flow, 'direct_response'):
            return

        if flow.response.status_code != 200:
            url = flow.request.req_url
            err_status = 400
            err_msg = 'Proxy Error'
            if flow.response.status_code == 404:
                err_status = 404
                err_msg = 'Not Found'

            self.send_error(flow, url, err_status, err_msg)
            return

        an_iter = flow.live.read_response_body(flow.request, flow.response)
        stream = IterIO(an_iter)

        try:
            self._set_response(flow, stream)
        except Exception as e:
            if hasattr(flow.request, 'req_url'):
                print(flow.request.req_url)
            print(type(e), e)
            import traceback
            traceback.print_exc()
Ejemplo n.º 10
0
    def test_sentinel_cases(self):
        def producer_dummy_flush(out):
            out.flush()

        iterable = IterIO(producer_dummy_flush)
        strict_eq(next(iterable), "")

        def producer_empty(out):
            pass

        iterable = IterIO(producer_empty)
        pytest.raises(StopIteration, next, iterable)

        iterable = IterIO(producer_dummy_flush, b"")
        strict_eq(next(iterable), b"")
        iterable = IterIO(producer_dummy_flush, u"")
        strict_eq(next(iterable), u"")
Ejemplo n.º 11
0
 def test_basic(self):
     def producer(out):
         out.write('1\n')
         out.write('2\n')
         out.flush()
         out.write('3\n')
     iterable = IterIO(producer)
     self.assert_equal(next(iterable), '1\n2\n')
     self.assert_equal(next(iterable), '3\n')
     self.assert_raises(StopIteration, next, iterable)
Ejemplo n.º 12
0
    def test_basic(self):
        def producer(out):
            out.write("1\n")
            out.write("2\n")
            out.flush()
            out.write("3\n")

        iterable = IterIO(producer)
        assert next(iterable) == "1\n2\n"
        assert next(iterable) == "3\n"
        pytest.raises(StopIteration, next, iterable)
Ejemplo n.º 13
0
    def test_basic(self):
        def producer(out):
            out.write('1\n')
            out.write('2\n')
            out.flush()
            out.write('3\n')

        iterable = IterIO(producer)
        assert next(iterable) == '1\n2\n'
        assert next(iterable) == '3\n'
        pytest.raises(StopIteration, next, iterable)
Ejemplo n.º 14
0
def _tar_iterator(tarball_bytes: Iterator[bytes]) \
        -> Iterator[Tuple[str, Optional[IO]]]:
    # The response is a tarball we need to extract into `output_dir`.
    with tempfile.TemporaryFile() as tarball:
        # `tarfile.open` needs to read from a real file, so we copy to one.
        shutil.copyfileobj(IterIO(tarball_bytes), tarball)
        # And rewind to the start.
        tarball.seek(0)
        tar = tarfile.open(fileobj=tarball)
        for tarinfo in tar.getmembers():
            # Drop the first segment, because it's just the name of the
            # directory that was tarred up, and we don't care.
            path_segments = tarinfo.name.split(os.sep)[1:]
            if path_segments:
                # Unfortunately we can't just pass `*path_segments`
                # because `os.path.join` explicitly expects an argument
                # for the first parameter.
                path = os.path.join(path_segments[0], *path_segments[1:])
                file_bytes = tar.extractfile(tarinfo.name)
                # Not None for files and links
                if file_bytes is not None:
                    yield path, tar.extractfile(tarinfo.name)
Ejemplo n.º 15
0
    def test_basic_native(self):
        io = IterIO(["Hello", "World", "1", "2", "3"])
        io.seek(0)
        assert io.tell() == 0
        assert io.read(2) == "He"
        assert io.tell() == 2
        assert io.read(3) == "llo"
        assert io.tell() == 5
        io.seek(0)
        assert io.read(5) == "Hello"
        assert io.tell() == 5
        assert io._buf == "Hello"
        assert io.read() == "World123"
        assert io.tell() == 13
        io.close()
        assert io.closed

        io = IterIO(["Hello\n", "World!"])
        assert io.readline() == "Hello\n"
        assert io._buf == "Hello\n"
        assert io.read() == "World!"
        assert io._buf == "Hello\nWorld!"
        assert io.tell() == 12
        io.seek(0)
        assert io.readlines() == ["Hello\n", "World!"]

        io = IterIO(["Line one\nLine ", "two\nLine three"])
        assert list(io) == ["Line one\n", "Line two\n", "Line three"]
        io = IterIO(iter("Line one\nLine two\nLine three"))
        assert list(io) == ["Line one\n", "Line two\n", "Line three"]
        io = IterIO(["Line one\nL", "ine", " two", "\nLine three"])
        assert list(io) == ["Line one\n", "Line two\n", "Line three"]

        io = IterIO(["foo\n", "bar"])
        io.seek(-4, 2)
        assert io.read(4) == "\nbar"

        pytest.raises(IOError, io.seek, 2, 100)
        io.close()
        pytest.raises(ValueError, io.read)
Ejemplo n.º 16
0
    def test_basic_native(self):
        io = IterIO(["Hello", "World", "1", "2", "3"])
        io.seek(0)
        assert io.tell() == 0
        assert io.read(2) == "He"
        assert io.tell() == 2
        assert io.read(3) == "llo"
        assert io.tell() == 5
        io.seek(0)
        assert io.read(5) == "Hello"
        assert io.tell() == 5
        assert io._buf == "Hello"
        assert io.read() == "World123"
        assert io.tell() == 13
        io.close()
        assert io.closed

        io = IterIO(["Hello\n", "World!"])
        assert io.readline() == 'Hello\n'
        assert io._buf == 'Hello\n'
        assert io.read() == 'World!'
        assert io._buf == 'Hello\nWorld!'
        assert io.tell() == 12
        io.seek(0)
        assert io.readlines() == ['Hello\n', 'World!']

        io = IterIO(['Line one\nLine ', 'two\nLine three'])
        assert list(io) == ['Line one\n', 'Line two\n', 'Line three']
        io = IterIO(iter('Line one\nLine two\nLine three'))
        assert list(io) == ['Line one\n', 'Line two\n', 'Line three']
        io = IterIO(['Line one\nL', 'ine', ' two', '\nLine three'])
        assert list(io) == ['Line one\n', 'Line two\n', 'Line three']

        io = IterIO(["foo\n", "bar"])
        io.seek(-4, 2)
        assert io.read(4) == '\nbar'

        pytest.raises(IOError, io.seek, 2, 100)
        io.close()
        pytest.raises(ValueError, io.read)
Ejemplo n.º 17
0
    def test_basic_native(self):
        io = IterIO(["Hello", "World", "1", "2", "3"])
        self.assert_equal(io.tell(), 0)
        self.assert_equal(io.read(2), "He")
        self.assert_equal(io.tell(), 2)
        self.assert_equal(io.read(3), "llo")
        self.assert_equal(io.tell(), 5)
        io.seek(0)
        self.assert_equal(io.read(5), "Hello")
        self.assert_equal(io.tell(), 5)
        self.assert_equal(io._buf, "Hello")
        self.assert_equal(io.read(), "World123")
        self.assert_equal(io.tell(), 13)
        io.close()
        assert io.closed

        io = IterIO(["Hello\n", "World!"])
        self.assert_equal(io.readline(), 'Hello\n')
        self.assert_equal(io._buf, 'Hello\n')
        self.assert_equal(io.read(), 'World!')
        self.assert_equal(io._buf, 'Hello\nWorld!')
        self.assert_equal(io.tell(), 12)
        io.seek(0)
        self.assert_equal(io.readlines(), ['Hello\n', 'World!'])

        io = IterIO(['Line one\nLine ', 'two\nLine three'])
        self.assert_equal(list(io), ['Line one\n', 'Line two\n', 'Line three'])
        io = IterIO(iter('Line one\nLine two\nLine three'))
        self.assert_equal(list(io), ['Line one\n', 'Line two\n', 'Line three'])
        io = IterIO(['Line one\nL', 'ine', ' two', '\nLine three'])
        self.assert_equal(list(io), ['Line one\n', 'Line two\n', 'Line three'])

        io = IterIO(["foo\n", "bar"])
        io.seek(-4, 2)
        self.assert_equal(io.read(4), '\nbar')

        self.assert_raises(IOError, io.seek, 2, 100)
        io.close()
        self.assert_raises(ValueError, io.read)
Ejemplo n.º 18
0
    def test_sentinel_cases(self):
        io = IterIO([])
        strict_eq(io.read(), "")
        io = IterIO([], b"")
        strict_eq(io.read(), b"")
        io = IterIO([], u"")
        strict_eq(io.read(), u"")

        io = IterIO([])
        strict_eq(io.read(), "")
        io = IterIO([b""])
        strict_eq(io.read(), b"")
        io = IterIO([u""])
        strict_eq(io.read(), u"")

        io = IterIO([])
        strict_eq(io.readline(), "")
        io = IterIO([], b"")
        strict_eq(io.readline(), b"")
        io = IterIO([], u"")
        strict_eq(io.readline(), u"")

        io = IterIO([])
        strict_eq(io.readline(), "")
        io = IterIO([b""])
        strict_eq(io.readline(), b"")
        io = IterIO([u""])
        strict_eq(io.readline(), u"")
Ejemplo n.º 19
0
    def test_sentinel_cases(self):
        io = IterIO([])
        strict_eq(io.read(), '')
        io = IterIO([], b'')
        strict_eq(io.read(), b'')
        io = IterIO([], u'')
        strict_eq(io.read(), u'')

        io = IterIO([])
        strict_eq(io.read(), '')
        io = IterIO([b''])
        strict_eq(io.read(), b'')
        io = IterIO([u''])
        strict_eq(io.read(), u'')

        io = IterIO([])
        strict_eq(io.readline(), '')
        io = IterIO([], b'')
        strict_eq(io.readline(), b'')
        io = IterIO([], u'')
        strict_eq(io.readline(), u'')

        io = IterIO([])
        strict_eq(io.readline(), '')
        io = IterIO([b''])
        strict_eq(io.readline(), b'')
        io = IterIO([u''])
        strict_eq(io.readline(), u'')
Ejemplo n.º 20
0
 def worker(inv=inv, key=inv[0]):
     rv[key] = InventoryFile.load(
         IterIO(requests.get(inv[1], stream=True).iter_content()),
         inv[0], posixpath.join)