def test_basic_unicode(self):
io = IterIO([u"Hello", u"World", u"1", u"2", u"3"])
self.assert_equal(io.tell(), 0)
self.assert_equal(io.read(2), u"He")
self.assert_equal(io.tell(), 2)
self.assert_equal(io.read(3), u"llo")
self.assert_equal(io.tell(), 5)
io.seek(0)
self.assert_equal(io.read(5), u"Hello")
self.assert_equal(io.tell(), 5)
self.assert_equal(io._buf, u"Hello")
self.assert_equal(io.read(), u"World123")
self.assert_equal(io.tell(), 13)
io.close()
assert io.closed
io = IterIO([u"Hello\n", u"World!"])
self.assert_equal(io.readline(), u'Hello\n')
self.assert_equal(io._buf, u'Hello\n')
self.assert_equal(io.read(), u'World!')
self.assert_equal(io._buf, u'Hello\nWorld!')
self.assert_equal(io.tell(), 12)
io.seek(0)
self.assert_equal(io.readlines(), [u'Hello\n', u'World!'])
io = IterIO([u"foo\n", u"bar"])
io.seek(-4, 2)
self.assert_equal(io.read(4), u'\nbar')
self.assert_raises(IOError, io.seek, 2, 100)
io.close()
self.assert_raises(ValueError, io.read)
def test_basic_bytes(self):
io = IterIO([b"Hello", b"World", b"1", b"2", b"3"])
self.assert_equal(io.tell(), 0)
self.assert_equal(io.read(2), b"He")
self.assert_equal(io.tell(), 2)
self.assert_equal(io.read(3), b"llo")
self.assert_equal(io.tell(), 5)
io.seek(0)
self.assert_equal(io.read(5), b"Hello")
self.assert_equal(io.tell(), 5)
self.assert_equal(io._buf, b"Hello")
self.assert_equal(io.read(), b"World123")
self.assert_equal(io.tell(), 13)
io.close()
assert io.closed
io = IterIO([b"Hello\n", b"World!"])
self.assert_equal(io.readline(), b'Hello\n')
self.assert_equal(io._buf, b'Hello\n')
self.assert_equal(io.read(), b'World!')
self.assert_equal(io._buf, b'Hello\nWorld!')
self.assert_equal(io.tell(), 12)
io.seek(0)
self.assert_equal(io.readlines(), [b'Hello\n', b'World!'])
io = IterIO([b"foo\n", b"bar"])
io.seek(-4, 2)
self.assert_equal(io.read(4), b'\nbar')
self.assert_raises(IOError, io.seek, 2, 100)
io.close()
self.assert_raises(ValueError, io.read)
def test_basic_bytes(self):
io = IterIO([b"Hello", b"World", b"1", b"2", b"3"])
assert io.tell() == 0
assert io.read(2) == b"He"
assert io.tell() == 2
assert io.read(3) == b"llo"
assert io.tell() == 5
io.seek(0)
assert io.read(5) == b"Hello"
assert io.tell() == 5
assert io._buf == b"Hello"
assert io.read() == b"World123"
assert io.tell() == 13
io.close()
assert io.closed
io = IterIO([b"Hello\n", b"World!"])
assert io.readline() == b'Hello\n'
assert io._buf == b'Hello\n'
assert io.read() == b'World!'
assert io._buf == b'Hello\nWorld!'
assert io.tell() == 12
io.seek(0)
assert io.readlines() == [b'Hello\n', b'World!']
io = IterIO([b"foo\n", b"bar"])
io.seek(-4, 2)
assert io.read(4) == b'\nbar'
pytest.raises(IOError, io.seek, 2, 100)
io.close()
pytest.raises(ValueError, io.read)
def test_basic_unicode(self):
io = IterIO([u"Hello", u"World", u"1", u"2", u"3"])
assert io.tell() == 0
assert io.read(2) == u"He"
assert io.tell() == 2
assert io.read(3) == u"llo"
assert io.tell() == 5
io.seek(0)
assert io.read(5) == u"Hello"
assert io.tell() == 5
assert io._buf == u"Hello"
assert io.read() == u"World123"
assert io.tell() == 13
io.close()
assert io.closed
io = IterIO([u"Hello\n", u"World!"])
assert io.readline() == u'Hello\n'
assert io._buf == u'Hello\n'
assert io.read() == u'World!'
assert io._buf == u'Hello\nWorld!'
assert io.tell() == 12
io.seek(0)
assert io.readlines() == [u'Hello\n', u'World!']
io = IterIO([u"foo\n", u"bar"])
io.seek(-4, 2)
assert io.read(4) == u'\nbar'
pytest.raises(IOError, io.seek, 2, 100)
io.close()
pytest.raises(ValueError, io.read)
def test_basic(self):
io = IterIO(["Hello", "World", "1", "2", "3"])
assert io.tell() == 0
assert io.read(2) == "He"
assert io.tell() == 2
assert io.read(3) == "llo"
assert io.tell() == 5
io.seek(0)
assert io.read(5) == "Hello"
assert io.tell() == 5
assert io._buf == "Hello"
assert io.read() == "World123"
assert io.tell() == 13
io.close()
assert io.closed
io = IterIO(["Hello\n", "World!"])
assert io.readline() == 'Hello\n'
assert io._buf == 'Hello\n'
assert io.read() == 'World!'
assert io._buf == 'Hello\nWorld!'
assert io.tell() == 12
io.seek(0)
assert io.readlines() == ['Hello\n', 'World!']
io = IterIO(["foo\n", "bar"])
io.seek(-4, 2)
assert io.read(4) == '\nbar'
self.assert_raises(IOError, io.seek, 2, 100)
io.close()
self.assert_raises(ValueError, io.read)
def test_itero():
"""Test the IterIO"""
iterable = iter(["Hello", "World", "1", "2", "3"])
io = IterIO(iterable)
assert io.tell() == 0
assert io.read(2) == "He"
assert io.tell() == 2
assert io.read(3) == "llo"
assert io.tell() == 5
io.seek(0)
assert io.read(5) == "Hello"
assert io.tell() == 5
assert io._buf == "Hello"
assert io.read() == "World123"
assert io.tell() == 13
io.close()
assert io.closed
io = IterIO(iter(["Hello\n", "World!"]))
assert io.readline() == 'Hello\n'
assert io._buf == 'Hello\n'
assert io.read() == 'World!'
assert io._buf == 'Hello\nWorld!'
assert io.tell() == 12
io.seek(0)
assert io.readlines() == ['Hello\n', 'World!']
io = IterIO(iter(["foo\n", "bar"]))
io.seek(-4, 2)
assert io.read(4) == '\nbar'
assert_raises(IOError, io.seek, 2, 100)
io.close()
assert_raises(ValueError, io.read)
def test_sentinel_cases(self):
def producer_dummy_flush(out):
out.flush()
iterable = IterIO(producer_dummy_flush)
self.assert_strict_equal(next(iterable), '')
def producer_empty(out):
pass
iterable = IterIO(producer_empty)
self.assert_raises(StopIteration, next, iterable)
iterable = IterIO(producer_dummy_flush, b'')
self.assert_strict_equal(next(iterable), b'')
iterable = IterIO(producer_dummy_flush, u'')
self.assert_strict_equal(next(iterable), u'')
def download_ed_catalog(catalog_url):
print('download_ed_catalog({})', catalog_url)
catalog_res = requests.get(catalog_url, stream=True)
print('response: {}', catalog_res)
# Read the request content, zip file and XML file via a stream to prevent
# reading the whole data into memory at once. The memory used should be
# constant regardless of the size of the input data.
if not catalog_url.endswith('.zip'):
catalog_xml = IterIO(catalog_res.iter_lines)
else:
with ZipFile(IterIO(catalog_res.iter_content(chunk_size=4096), sentinel=b'')) as zf:
with zf.open(zf.filelist[0].filename, 'r') as xml_file:
catalog_xml = xml_file
return catalog_xml
def response(self, flow):
if flow.request.host == self.fwd_host:
return
if hasattr(flow, 'direct_response'):
return
if flow.response.status_code != 200:
url = flow.request.req_url
err_status = 400
err_msg = 'Proxy Error'
if flow.response.status_code == 404:
err_status = 404
err_msg = 'Not Found'
self.send_error(flow, url, err_status, err_msg)
return
an_iter = flow.live.read_response_body(flow.request, flow.response)
stream = IterIO(an_iter)
try:
self._set_response(flow, stream)
except Exception as e:
if hasattr(flow.request, 'req_url'):
print(flow.request.req_url)
print(type(e), e)
import traceback
traceback.print_exc()
def test_sentinel_cases(self):
def producer_dummy_flush(out):
out.flush()
iterable = IterIO(producer_dummy_flush)
strict_eq(next(iterable), "")
def producer_empty(out):
pass
iterable = IterIO(producer_empty)
pytest.raises(StopIteration, next, iterable)
iterable = IterIO(producer_dummy_flush, b"")
strict_eq(next(iterable), b"")
iterable = IterIO(producer_dummy_flush, u"")
strict_eq(next(iterable), u"")
def test_basic(self):
def producer(out):
out.write('1\n')
out.write('2\n')
out.flush()
out.write('3\n')
iterable = IterIO(producer)
self.assert_equal(next(iterable), '1\n2\n')
self.assert_equal(next(iterable), '3\n')
self.assert_raises(StopIteration, next, iterable)
def test_basic(self):
def producer(out):
out.write("1\n")
out.write("2\n")
out.flush()
out.write("3\n")
iterable = IterIO(producer)
assert next(iterable) == "1\n2\n"
assert next(iterable) == "3\n"
pytest.raises(StopIteration, next, iterable)
def test_basic(self):
def producer(out):
out.write('1\n')
out.write('2\n')
out.flush()
out.write('3\n')
iterable = IterIO(producer)
assert next(iterable) == '1\n2\n'
assert next(iterable) == '3\n'
pytest.raises(StopIteration, next, iterable)
def _tar_iterator(tarball_bytes: Iterator[bytes]) \
-> Iterator[Tuple[str, Optional[IO]]]:
# The response is a tarball we need to extract into `output_dir`.
with tempfile.TemporaryFile() as tarball:
# `tarfile.open` needs to read from a real file, so we copy to one.
shutil.copyfileobj(IterIO(tarball_bytes), tarball)
# And rewind to the start.
tarball.seek(0)
tar = tarfile.open(fileobj=tarball)
for tarinfo in tar.getmembers():
# Drop the first segment, because it's just the name of the
# directory that was tarred up, and we don't care.
path_segments = tarinfo.name.split(os.sep)[1:]
if path_segments:
# Unfortunately we can't just pass `*path_segments`
# because `os.path.join` explicitly expects an argument
# for the first parameter.
path = os.path.join(path_segments[0], *path_segments[1:])
file_bytes = tar.extractfile(tarinfo.name)
# Not None for files and links
if file_bytes is not None:
yield path, tar.extractfile(tarinfo.name)
def test_basic_native(self):
io = IterIO(["Hello", "World", "1", "2", "3"])
io.seek(0)
assert io.tell() == 0
assert io.read(2) == "He"
assert io.tell() == 2
assert io.read(3) == "llo"
assert io.tell() == 5
io.seek(0)
assert io.read(5) == "Hello"
assert io.tell() == 5
assert io._buf == "Hello"
assert io.read() == "World123"
assert io.tell() == 13
io.close()
assert io.closed
io = IterIO(["Hello\n", "World!"])
assert io.readline() == "Hello\n"
assert io._buf == "Hello\n"
assert io.read() == "World!"
assert io._buf == "Hello\nWorld!"
assert io.tell() == 12
io.seek(0)
assert io.readlines() == ["Hello\n", "World!"]
io = IterIO(["Line one\nLine ", "two\nLine three"])
assert list(io) == ["Line one\n", "Line two\n", "Line three"]
io = IterIO(iter("Line one\nLine two\nLine three"))
assert list(io) == ["Line one\n", "Line two\n", "Line three"]
io = IterIO(["Line one\nL", "ine", " two", "\nLine three"])
assert list(io) == ["Line one\n", "Line two\n", "Line three"]
io = IterIO(["foo\n", "bar"])
io.seek(-4, 2)
assert io.read(4) == "\nbar"
pytest.raises(IOError, io.seek, 2, 100)
io.close()
pytest.raises(ValueError, io.read)
def test_basic_native(self):
io = IterIO(["Hello", "World", "1", "2", "3"])
io.seek(0)
assert io.tell() == 0
assert io.read(2) == "He"
assert io.tell() == 2
assert io.read(3) == "llo"
assert io.tell() == 5
io.seek(0)
assert io.read(5) == "Hello"
assert io.tell() == 5
assert io._buf == "Hello"
assert io.read() == "World123"
assert io.tell() == 13
io.close()
assert io.closed
io = IterIO(["Hello\n", "World!"])
assert io.readline() == 'Hello\n'
assert io._buf == 'Hello\n'
assert io.read() == 'World!'
assert io._buf == 'Hello\nWorld!'
assert io.tell() == 12
io.seek(0)
assert io.readlines() == ['Hello\n', 'World!']
io = IterIO(['Line one\nLine ', 'two\nLine three'])
assert list(io) == ['Line one\n', 'Line two\n', 'Line three']
io = IterIO(iter('Line one\nLine two\nLine three'))
assert list(io) == ['Line one\n', 'Line two\n', 'Line three']
io = IterIO(['Line one\nL', 'ine', ' two', '\nLine three'])
assert list(io) == ['Line one\n', 'Line two\n', 'Line three']
io = IterIO(["foo\n", "bar"])
io.seek(-4, 2)
assert io.read(4) == '\nbar'
pytest.raises(IOError, io.seek, 2, 100)
io.close()
pytest.raises(ValueError, io.read)
def test_basic_native(self):
io = IterIO(["Hello", "World", "1", "2", "3"])
self.assert_equal(io.tell(), 0)
self.assert_equal(io.read(2), "He")
self.assert_equal(io.tell(), 2)
self.assert_equal(io.read(3), "llo")
self.assert_equal(io.tell(), 5)
io.seek(0)
self.assert_equal(io.read(5), "Hello")
self.assert_equal(io.tell(), 5)
self.assert_equal(io._buf, "Hello")
self.assert_equal(io.read(), "World123")
self.assert_equal(io.tell(), 13)
io.close()
assert io.closed
io = IterIO(["Hello\n", "World!"])
self.assert_equal(io.readline(), 'Hello\n')
self.assert_equal(io._buf, 'Hello\n')
self.assert_equal(io.read(), 'World!')
self.assert_equal(io._buf, 'Hello\nWorld!')
self.assert_equal(io.tell(), 12)
io.seek(0)
self.assert_equal(io.readlines(), ['Hello\n', 'World!'])
io = IterIO(['Line one\nLine ', 'two\nLine three'])
self.assert_equal(list(io), ['Line one\n', 'Line two\n', 'Line three'])
io = IterIO(iter('Line one\nLine two\nLine three'))
self.assert_equal(list(io), ['Line one\n', 'Line two\n', 'Line three'])
io = IterIO(['Line one\nL', 'ine', ' two', '\nLine three'])
self.assert_equal(list(io), ['Line one\n', 'Line two\n', 'Line three'])
io = IterIO(["foo\n", "bar"])
io.seek(-4, 2)
self.assert_equal(io.read(4), '\nbar')
self.assert_raises(IOError, io.seek, 2, 100)
io.close()
self.assert_raises(ValueError, io.read)
def test_sentinel_cases(self):
io = IterIO([])
strict_eq(io.read(), "")
io = IterIO([], b"")
strict_eq(io.read(), b"")
io = IterIO([], u"")
strict_eq(io.read(), u"")
io = IterIO([])
strict_eq(io.read(), "")
io = IterIO([b""])
strict_eq(io.read(), b"")
io = IterIO([u""])
strict_eq(io.read(), u"")
io = IterIO([])
strict_eq(io.readline(), "")
io = IterIO([], b"")
strict_eq(io.readline(), b"")
io = IterIO([], u"")
strict_eq(io.readline(), u"")
io = IterIO([])
strict_eq(io.readline(), "")
io = IterIO([b""])
strict_eq(io.readline(), b"")
io = IterIO([u""])
strict_eq(io.readline(), u"")
def test_sentinel_cases(self):
io = IterIO([])
strict_eq(io.read(), '')
io = IterIO([], b'')
strict_eq(io.read(), b'')
io = IterIO([], u'')
strict_eq(io.read(), u'')
io = IterIO([])
strict_eq(io.read(), '')
io = IterIO([b''])
strict_eq(io.read(), b'')
io = IterIO([u''])
strict_eq(io.read(), u'')
io = IterIO([])
strict_eq(io.readline(), '')
io = IterIO([], b'')
strict_eq(io.readline(), b'')
io = IterIO([], u'')
strict_eq(io.readline(), u'')
io = IterIO([])
strict_eq(io.readline(), '')
io = IterIO([b''])
strict_eq(io.readline(), b'')
io = IterIO([u''])
strict_eq(io.readline(), u'')
def worker(inv=inv, key=inv[0]):
rv[key] = InventoryFile.load(
IterIO(requests.get(inv[1], stream=True).iter_content()),
inv[0], posixpath.join)