def test_read_until(self):
buf = StringBuffer()
buf.feed(b'hello\nworld\n')
data = buf.read_until(b'\n')
self.assertEqual(data, b'hello\n')
data = buf.read_until(b'\n')
self.assertEqual(data, b'world\n')
data = buf.read_until(b'\n')
self.assertEqual(data, None)
def test_read(self):
buf = StringBuffer()
buf.feed(b'hello world')
data = buf.read(5)
self.assertEqual(data, b'hello')
data = buf.read(6)
self.assertEqual(data, b' world')
data = buf.read(100)
self.assertEqual(data, None)
def test_read_until_regex(self):
regex = re.compile(b'~~')
buf = StringBuffer()
buf.feed(b'hello~~world~~')
data = buf.read_until_regex(regex)
self.assertEqual(data, b'hello~~')
data = buf.read_until_regex(regex)
self.assertEqual(data, b'world~~')
data = buf.read_until_regex(regex)
self.assertEqual(data, None)
def test_read_until(self):
buf = StringBuffer()
buf.feed(b'hello\nworld\n')
data = buf.read_until(b'\n')
self.assertEqual(data, b'hello\n')
data = buf.read_until(b'\n')
self.assertEqual(data, b'world\n')
data = buf.read_until(b'\n')
self.assertEqual(data, None)
def test_read(self):
buf = StringBuffer()
buf.feed(b'hello world')
data = buf.read(5)
self.assertEqual(data, b'hello')
data = buf.read(6)
self.assertEqual(data, b' world')
data = buf.read(100)
self.assertEqual(data, None)
def test_read_until_regex(self):
regex = re.compile(b'~~')
buf = StringBuffer()
buf.feed(b'hello~~world~~')
data = buf.read_until_regex(regex)
self.assertEqual(data, b'hello~~')
data = buf.read_until_regex(regex)
self.assertEqual(data, b'world~~')
data = buf.read_until_regex(regex)
self.assertEqual(data, None)
def test_close(self):
buf = StringBuffer()
buf.feed(b'hello world')
buf.close()
self.assertTrue(buf.closed)
self.assertRaises(ValueError, buf.read, 5)
def test_clear(self):
buf = StringBuffer()
buf.feed(b'hello world')
buf.clear()
data = buf.read(5)
self.assertEqual(data, None)
def __init__(self):
self._read_buffer = StringBuffer(self.MAX_BUFFER_SIZE)
self._write_buffer = []
self._connected = False
self._closed = False
class AbstractBaseStream(six.with_metaclass(abc.ABCMeta)):
"""Abstract base class for a stream-like object
"""
error_cls = None # to be defined by subclass
MAX_BUFFER_SIZE = 100*1024*1024
READ_CHUNK_SIZE = 4*1024
def __init__(self):
self._read_buffer = StringBuffer(self.MAX_BUFFER_SIZE)
self._write_buffer = []
self._connected = False
self._closed = False
def read_bytes(self, nbytes):
assert isinstance(nbytes, six.integer_types)
assert nbytes > 0
return self._do_read(nbytes=nbytes)
def read_until(self, delimiter):
return self._do_read(delimiter=delimiter)
def read_until_regex(self, regex):
return self._do_read(regex=re.compile(regex))
def write(self, data):
self._check_closed()
if not self._connected:
self._write_buffer.append(data)
return False
else:
return self._write(data)
def shutdown(self):
self._check_closed()
self._shutdown()
def close(self):
if self._closed:
return
self._read_buffer.clear()
self._write_buffer = []
self._close()
self._closed = True
@property
def closed(self):
return self._closed
def _set_connected(self):
self._connected = True
buf, self._write_buffer = self._write_buffer, []
if buf:
self._write(b''.join(buf))
# internal
def _do_read(self, delimiter=None, nbytes=None, regex=None):
# See if we've already got the data from a previous read
data = self._read_from_buffer(delimiter, nbytes, regex)
if data is not None:
return data
self._check_closed()
while not self.closed:
self._read(self.READ_CHUNK_SIZE)
data = self._read_from_buffer(delimiter, nbytes, regex)
if data is not None:
return data
return b''
def _read_from_buffer(self, delimiter=None, nbytes=None, regex=None):
if nbytes is not None:
return self._read_buffer.read(nbytes)
elif delimiter is not None:
return self._read_buffer.read_until(delimiter)
elif regex is not None:
return self._read_buffer.read_until_regex(regex)
def _check_closed(self):
if self._closed:
raise self.error_cls('stream is closed')
# internal, to be implemented by subclasses
@abc.abstractmethod
def _read(self, n):
raise NotImplementedError
@abc.abstractmethod
def _write(self, data):
raise NotImplementedError
@abc.abstractmethod
def _shutdown(self):
raise NotImplementedError
@abc.abstractmethod
def _close(self):
raise NotImplementedError
def __init__(self):
self._read_buffer = StringBuffer(self.MAX_BUFFER_SIZE)
self._write_buffer = []
self._connected = False
self._closed = False
class AbstractBaseStream(six.with_metaclass(abc.ABCMeta)):
"""Abstract base class for a stream-like object
"""
error_cls = None # to be defined by subclass
MAX_BUFFER_SIZE = 100 * 1024 * 1024
READ_CHUNK_SIZE = 4 * 1024
def __init__(self):
self._read_buffer = StringBuffer(self.MAX_BUFFER_SIZE)
self._write_buffer = []
self._connected = False
self._closed = False
def read_bytes(self, nbytes):
assert isinstance(nbytes, six.integer_types)
assert nbytes > 0
return self._do_read(nbytes=nbytes)
def read_until(self, delimiter):
return self._do_read(delimiter=delimiter)
def read_until_regex(self, regex):
return self._do_read(regex=re.compile(regex))
def write(self, data):
self._check_closed()
if not self._connected:
self._write_buffer.append(data)
return False
else:
return self._write(data)
def shutdown(self):
self._check_closed()
self._shutdown()
def close(self):
if self._closed:
return
self._read_buffer.clear()
self._write_buffer = []
self._close()
self._closed = True
@property
def closed(self):
return self._closed
def _set_connected(self):
self._connected = True
buf, self._write_buffer = self._write_buffer, []
if buf:
self._write(b''.join(buf))
# internal
def _do_read(self, delimiter=None, nbytes=None, regex=None):
# See if we've already got the data from a previous read
data = self._read_from_buffer(delimiter, nbytes, regex)
if data is not None:
return data
self._check_closed()
while not self.closed:
self._read(self.READ_CHUNK_SIZE)
data = self._read_from_buffer(delimiter, nbytes, regex)
if data is not None:
return data
return b''
def _read_from_buffer(self, delimiter=None, nbytes=None, regex=None):
if nbytes is not None:
return self._read_buffer.read(nbytes)
elif delimiter is not None:
return self._read_buffer.read_until(delimiter)
elif regex is not None:
return self._read_buffer.read_until_regex(regex)
def _check_closed(self):
if self._closed:
raise self.error_cls('stream is closed')
# internal, to be implemented by subclasses
@abc.abstractmethod
def _read(self, n):
raise NotImplementedError
@abc.abstractmethod
def _write(self, data):
raise NotImplementedError
@abc.abstractmethod
def _shutdown(self):
raise NotImplementedError
@abc.abstractmethod
def _close(self):
raise NotImplementedError
def test_close(self):
buf = StringBuffer()
buf.feed(b'hello world')
buf.close()
self.assertTrue(buf.closed)
self.assertRaises(ValueError, buf.read, 5)
def test_clear(self):
buf = StringBuffer()
buf.feed(b'hello world')
buf.clear()
data = buf.read(5)
self.assertEqual(data, None)
