def test_truncate(self):
import _io
raw = _io.FileIO(self.tmpfile, 'w+')
raw.write('x' * 20)
b = _io.BufferedReader(raw)
assert b.seek(8) == 8
assert b.truncate() == 8
assert b.tell() == 8
def test_peek(self):
import _io
raw = _io.FileIO(self.tmpfile)
f = _io.BufferedReader(raw)
assert f.read(2) == 'a\n'
assert f.peek().startswith('b\nc')
assert f.read(3) == 'b\nc'
assert f.peek() == ''
def test_readinto(self):
import _io
a = bytearray('x' * 10)
raw = _io.FileIO(self.tmpfile)
f = _io.BufferedReader(raw)
assert f.readinto(a) == 5
f.close()
assert a == 'a\nb\ncxxxxx'
def test_monkeypatch_works(self):
import _io, os
raw = _io.FileIO(os.devnull)
f = _io.BufferedReader(raw)
with raises(ValueError) as e:
f.read(1024)
assert e.value.args[
0] == "rgc.nonmoving_raw_ptr_for_resizable_list() not supported under RevDB"
def test_read_pieces(self):
import _io
raw = _io.FileIO(self.tmpfile)
f = _io.BufferedReader(raw)
assert f.read(3) == "a\nb"
assert f.read(3) == "\nc"
assert f.read(3) == ""
assert f.read(3) == ""
f.close()
def w_test_nonblock_pipe_write(self, bufsize):
import _io as io
class NonBlockingPipe(io._BufferedIOBase):
"write() returns None when buffer is full"
def __init__(self, buffersize=4096):
self.buffersize = buffersize
self.buffer = b''
def readable(self): return True
def writable(self): return True
def write(self, data):
available = self.buffersize - len(self.buffer)
if available <= 0:
return None
self.buffer += data[:available]
return min(len(data), available)
def read(self, size=-1):
if not self.buffer:
return None
if size == -1:
size = len(self.buffer)
data = self.buffer[:size]
self.buffer = self.buffer[size:]
return data
sent = []
received = []
pipe = NonBlockingPipe()
rf = io.BufferedReader(pipe, bufsize)
wf = io.BufferedWriter(pipe, bufsize)
for N in 9999, 7574:
try:
i = 0
while True:
msg = bytes([i % 26 + 97] * N)
sent.append(msg)
wf.write(msg)
i += 1
except io.BlockingIOError as e:
sent[-1] = sent[-1][:e.characters_written]
received.append(rf.read())
msg = b'BLOCKED'
wf.write(msg)
sent.append(msg)
while True:
try:
wf.flush()
break
except io.BlockingIOError as e:
received.append(rf.read())
received += iter(rf.read, None)
rf.close()
wf.close()
sent, received = b''.join(sent), b''.join(received)
assert sent == received
def test_readinto_big(self):
import _io
a1 = bytearray('x')
a = bytearray('x' * 199)
raw = _io.FileIO(self.bigtmpfile)
f = _io.BufferedReader(raw)
assert f.readinto(a1) == 1
assert a1 == 'a'
assert f.readinto(a) == 99
assert a == '\nb\nc' + 'a\nb\nc' * 19 + 'x' * 100
def test_seek(self):
import _io
raw = _io.FileIO(self.tmpfile)
f = _io.BufferedReader(raw)
assert f.read() == "a\nb\nc"
f.seek(0)
assert f.read() == "a\nb\nc"
f.seek(-2, 2)
assert f.read() == "\nc"
f.close()
def test_slow_provider(self):
import _io
class MockIO(_io._IOBase):
def readable(self):
return True
def readinto(self, buf):
buf[:3] = "abc"
return 3
bufio = _io.BufferedReader(MockIO())
r = bufio.read(5)
assert r == "abcab"
def test_detach(self):
import _io
raw = _io.FileIO(self.tmpfile)
f = _io.BufferedReader(raw)
assert f.fileno() == raw.fileno()
assert f.detach() is raw
raises(ValueError, f.fileno)
raises(ValueError, f.close)
raises(ValueError, f.detach)
raises(ValueError, f.flush)
assert not raw.closed
raw.close()
def test_detached(self):
import _io
class MockRawIO(_io._RawIOBase):
def readable(self):
return True
raw = MockRawIO()
buf = _io.BufferedReader(raw)
assert buf.detach() is raw
raises(ValueError, buf.detach)
raises(ValueError, getattr, buf, 'mode')
raises(ValueError, buf.isatty)
repr(buf) # Should still work
def test_properties(self):
import _io
r = _io.BytesIO(b"\xc3\xa9\n\n")
b = _io.BufferedReader(r, 1000)
t = _io.TextIOWrapper(b)
assert t.readable()
assert t.seekable()
#
class CustomFile(object):
def isatty(self): return 'YES'
readable = writable = seekable = lambda self: False
t = _io.TextIOWrapper(CustomFile())
assert t.isatty() == 'YES'
def test_readinto_small_parts(self):
import _io, os, thread, time
read_fd, write_fd = os.pipe()
raw = _io.FileIO(read_fd)
f = _io.BufferedReader(raw)
a = bytearray(b'x' * 10)
os.write(write_fd, b"abcde")
def write_more():
time.sleep(0.5)
os.write(write_fd, b"fghij")
thread.start_new_thread(write_more, ())
assert f.readinto(a) == 10
assert a == 'abcdefghij'
def test_unseekable(self):
import _io
class Unseekable(_io.BytesIO):
def seekable(self):
return False
def seek(self, *args):
raise _io.UnsupportedOperation("not seekable")
def tell(self, *args):
raise _io.UnsupportedOperation("not seekable")
bufio = _io.BufferedReader(Unseekable(b"A" * 10))
raises(_io.UnsupportedOperation, bufio.tell)
raises(_io.UnsupportedOperation, bufio.seek, 0)
bufio.read(1)
raises(_io.UnsupportedOperation, bufio.seek, 0)
raises(_io.UnsupportedOperation, bufio.tell)
def test_constructor():
r = _io.BytesIO(b"\xc3\xa9\n\n")
b = _io.BufferedReader(r, 1000)
t = _io.TextIOWrapper(b)
t.__init__(b, encoding="latin1", newline="\r\n")
assert t.encoding == "latin1"
assert t.line_buffering == False
t.__init__(b, encoding="utf8", line_buffering=True)
assert t.encoding == "utf8"
assert t.line_buffering == True
assert t.readline() == u"\xe9\n"
raises(TypeError, t.__init__, b, newline=42)
raises(ValueError, t.__init__, b, newline='xyzzy')
t = _io.TextIOWrapper(b)
assert t.encoding
def test_readinto(self):
import _io
a = bytearray(b'x' * 10)
raw = _io.FileIO(self.tmpfile)
f = _io.BufferedReader(raw)
assert f.readinto(a) == 5
f.seek(0)
m = memoryview(bytearray(b"hello"))
assert f.readinto(m) == 5
exc = raises(TypeError, f.readinto, u"hello")
assert str(exc.value) == "must be read-write buffer, not str"
exc = raises(TypeError, f.readinto, memoryview(b"hello"))
assert str(exc.value) == "must be read-write buffer, not memoryview"
f.close()
assert a == b'a\nb\ncxxxxx'
def test_tell(self):
import _io
raw = _io.FileIO(self.tmpfile)
f = _io.BufferedReader(raw, buffer_size=2)
assert f.tell() == 0
d1 = f.read(1)
assert f.tell() == 1
d2 = f.read(2)
assert f.tell() == 3
assert f.seek(0) == 0
assert f.tell() == 0
d3 = f.read(3)
assert f.tell() == 3
assert d1 + d2 == d3
f.close()
def test_read_past_eof(self):
import _io
class MockIO(_io._IOBase):
stack = ["abc", "d", "efg"]
def readable(self):
return True
def readinto(self, buf):
if self.stack:
data = self.stack.pop(0)
buf[:len(data)] = data
return len(data)
else:
return 0
bufio = _io.BufferedReader(MockIO())
assert bufio.read(9000) == "abcdefg"
def test_valid_buffer(self):
import _io
class MockIO(_io._IOBase):
def readable(self):
return True
def readinto(self, buf):
# Check that `buf` is a valid memoryview object
assert buf.itemsize == 1
assert buf.strides == (1,)
assert buf.shape == (len(buf),)
return len(bytes(buf))
bufio = _io.BufferedReader(MockIO())
assert len(bufio.read(5)) == 5 # Note: PyPy zeros the buffer, CPython does not
def test_readinto(self):
import _io
a = bytearray('x' * 10)
raw = _io.FileIO(self.tmpfile)
f = _io.BufferedReader(raw)
assert f.readinto(a) == 5
exc = raises(TypeError, f.readinto, u"hello")
assert str(exc.value) == "cannot use unicode as modifiable buffer"
exc = raises(TypeError, f.readinto, buffer(b"hello"))
assert str(exc.value) == "must be read-write buffer, not buffer"
exc = raises(TypeError, f.readinto, buffer(bytearray("hello")))
assert str(exc.value) == "must be read-write buffer, not buffer"
exc = raises(TypeError, f.readinto, memoryview(b"hello"))
assert str(exc.value) == "must be read-write buffer, not memoryview"
f.close()
assert a == 'a\nb\ncxxxxx'
def open_maybe_gzip(filename, mode='r'):
# this _must_ be a str
filename = str(filename)
if filename.endswith(h5_constants.GZIP_SUFFIX):
raw = gzip.open(filename, mode + 'b', 2)
elif filename.endswith(h5_constants.LZ4_SUFFIX):
raw = lz4.open(filename, mode + 'b')
else:
return open(filename, mode)
bufsize = 1024 * 1024 # 1MB of buffering
if mode == 'r':
return io.BufferedReader(raw, buffer_size=bufsize)
elif mode == 'w':
return io.BufferedWriter(raw, buffer_size=bufsize)
else:
raise ValueError("Unsupported mode for compression: %s" % mode)
def _(remote,
file,
mode='r',
buffering=-1,
encoding=None,
errors=None,
newline=None,
closefd=True,
opener=None):
if 'b' in mode:
if 'r' in mode:
# return remote.File(file)
return _io.BufferedReader(remote.File(file))
elif 'w' in mode:
return _io.BufferedWriter(remote.File(file))
else:
raise NotImplementedError()
def test_read_non_blocking(self):
import _io
class MockRawIO(_io._RawIOBase):
def __init__(self, read_stack=()):
self._read_stack = list(read_stack)
def readable(self):
return True
def readinto(self, buf):
max_len = len(buf)
try:
data = self._read_stack[0]
except IndexError:
self._extraneous_reads += 1
return 0
if data is None:
del self._read_stack[0]
return None
n = len(data)
if len(data) <= max_len:
del self._read_stack[0]
buf[:n] = data
return n
else:
buf[:] = data[:max_len]
self._read_stack[0] = data[max_len:]
return max_len
def read(self, n=None):
try:
return self._read_stack.pop(0)
except IndexError:
return ""
# Inject some None's in there to simulate EWOULDBLOCK
rawio = MockRawIO((b"abc", b"d", None, b"efg", None, None, None))
bufio = _io.BufferedReader(rawio)
assert bufio.read(6) == "abcd"
assert bufio.read(1) == "e"
assert bufio.read() == "fg"
assert bufio.peek(1) == ""
assert bufio.read() is None
assert bufio.read() == ""
def test_read_interrupted(self):
import _io, errno
class MockRawIO(_io._RawIOBase):
def __init__(self):
self.count = 0
def readable(self):
return True
def readinto(self, buf):
self.count += 1
if self.count < 3:
raise IOError(errno.EINTR, "interrupted")
else:
buf[:3] = "abc"
return 3
rawio = MockRawIO()
bufio = _io.BufferedReader(rawio)
r = bufio.read(4)
assert r == "abca"
assert rawio.count == 4
def test_read1(self):
import _io
class RecordingFileIO(_io.FileIO):
def read(self, size=-1):
self.nbreads += 1
return _io.FileIO.read(self, size)
def readinto(self, buf):
self.nbreads += 1
return _io.FileIO.readinto(self, buf)
raw = RecordingFileIO(self.tmpfile)
raw.nbreads = 0
f = _io.BufferedReader(raw, buffer_size=3)
assert f.read(1) == 'a'
assert f.read1(1) == '\n'
assert raw.nbreads == 1
assert f.read1(100) == 'b'
assert raw.nbreads == 1
assert f.read1(100) == '\nc'
assert raw.nbreads == 2
assert f.read1(100) == ''
assert raw.nbreads == 3
f.close()
def test_newlines(self):
import _io
input_lines = ["unix\n", "windows\r\n", "os9\r", "last\n", "nonl"]
tests = [
[None, ['unix\n', 'windows\n', 'os9\n', 'last\n', 'nonl']],
['', input_lines],
['\n', ["unix\n", "windows\r\n", "os9\rlast\n", "nonl"]],
['\r\n', ["unix\nwindows\r\n", "os9\rlast\nnonl"]],
['\r', ["unix\nwindows\r", "\nos9\r", "last\nnonl"]],
]
# Try a range of buffer sizes to test the case where \r is the last
# character in TextIOWrapper._pending_line.
encoding = "ascii"
# XXX: str.encode() should return bytes
data = bytes(''.join(input_lines).encode(encoding))
for do_reads in (False, True):
for bufsize in range(1, 10):
for newline, exp_lines in tests:
bufio = _io.BufferedReader(_io.BytesIO(data), bufsize)
textio = _io.TextIOWrapper(bufio,
newline=newline,
encoding=encoding)
if do_reads:
got_lines = []
while True:
c2 = textio.read(2)
if c2 == '':
break
len(c2) == 2
got_lines.append(c2 + textio.readline())
else:
got_lines = list(textio)
for got_line, exp_line in zip(got_lines, exp_lines):
assert got_line == exp_line
assert len(got_lines) == len(exp_lines)
def test_readinto(self):
import _io
for methodname in ["readinto", "readinto1"]:
a = bytearray(b'x' * 10)
raw = _io.FileIO(self.tmpfile)
f = _io.BufferedReader(raw)
readinto = getattr(f, methodname)
assert readinto(a) == 5
f.seek(0)
m = memoryview(bytearray(b"hello"))
assert readinto(m) == 5
#
exc = raises(TypeError, readinto, u"hello")
msg = str(exc.value)
# print(msg)
assert " read-write b" in msg and msg.endswith(", not str")
#
exc = raises(TypeError, readinto, memoryview(b"hello"))
msg = str(exc.value)
# print(msg)
assert " read-write b" in msg and msg.endswith(", not memoryview")
#
f.close()
assert a == b'a\nb\ncxxxxx'
def test_newlinetranslate(self):
import _io
r = _io.BytesIO(b"abc\r\ndef\rg")
b = _io.BufferedReader(r, 1000)
t = _io.TextIOWrapper(b)
assert t.read() == u"abc\ndef\ng"
def test_repr(self):
import _io
raw = _io.FileIO(self.tmpfile)
f = _io.BufferedReader(raw)
assert repr(f) == '<_io.BufferedReader name=%r>' % (self.tmpfile, )
def test_readlines(self):
import _io
raw = _io.FileIO(self.tmpfile)
f = _io.BufferedReader(raw)
assert f.readlines() == ['a\n', 'b\n', 'c']