def test_readline_limit_with_existing_data(self):
# Read one line. The data is in StreamReader's buffer
# before the event loop is run.
stream = asyncio.StreamReader(limit=3, loop=self.loop)
stream.feed_data(b'li')
stream.feed_data(b'ne1\nline2\n')
self.assertRaises(ValueError, self.loop.run_until_complete,
stream.readline())
# The buffer should contain the remaining data after exception
self.assertEqual(b'line2\n', stream._buffer)
stream = asyncio.StreamReader(limit=3, loop=self.loop)
stream.feed_data(b'li')
stream.feed_data(b'ne1')
stream.feed_data(b'li')
self.assertRaises(ValueError, self.loop.run_until_complete,
stream.readline())
# No b'\n' at the end. The 'limit' is set to 3. So before
# waiting for the new data in buffer, 'readline' will consume
# the entire buffer, and since the length of the consumed data
# is more than 3, it will raise a ValueError. The buffer is
# expected to be empty now.
self.assertEqual(b'', stream._buffer)
def test_readline_limit(self):
# Read one line. StreamReaders are fed with data after
# their 'readline' methods are called.
stream = asyncio.StreamReader(limit=7, loop=self.loop)
def cb():
stream.feed_data(b'chunk1')
stream.feed_data(b'chunk2')
stream.feed_data(b'chunk3\n')
stream.feed_eof()
self.loop.call_soon(cb)
self.assertRaises(ValueError, self.loop.run_until_complete,
stream.readline())
# The buffer had just one line of data, and after raising
# a ValueError it should be empty.
self.assertEqual(b'', stream._buffer)
stream = asyncio.StreamReader(limit=7, loop=self.loop)
def cb():
stream.feed_data(b'chunk1')
stream.feed_data(b'chunk2\n')
stream.feed_data(b'chunk3\n')
stream.feed_eof()
self.loop.call_soon(cb)
self.assertRaises(ValueError, self.loop.run_until_complete,
stream.readline())
self.assertEqual(b'chunk3\n', stream._buffer)
def test_exception(self):
stream = asyncio.StreamReader(loop=self.loop)
self.assertIsNone(stream.exception())
exc = ValueError()
stream.set_exception(exc)
self.assertIs(stream.exception(), exc)
def test_readline_eof(self):
stream = asyncio.StreamReader(loop=self.loop)
stream.feed_data(b'some data')
stream.feed_eof()
line = self.loop.run_until_complete(stream.readline())
self.assertEqual(b'some data', line)
def test_read_zero(self):
# Read zero bytes.
stream = asyncio.StreamReader(loop=self.loop)
stream.feed_data(self.DATA)
data = self.loop.run_until_complete(stream.read(0))
self.assertEqual(b'', data)
self.assertEqual(self.DATA, stream._buffer)
def test_streamreader_constructor(self):
self.addCleanup(asyncio.set_event_loop, None)
asyncio.set_event_loop(self.loop)
# asyncio issue #184: Ensure that StreamReaderProtocol constructor
# retrieves the current loop if the loop parameter is not set
reader = asyncio.StreamReader()
self.assertIs(reader._loop, self.loop)
def connect_read_pipe(file):
loop = asyncio.get_event_loop()
stream_reader = asyncio.StreamReader(loop=loop)
def factory():
return asyncio.StreamReaderProtocol(stream_reader)
transport, _ = yield From(loop.connect_read_pipe(factory, file))
raise Return(stream_reader, transport)
def test_readexactly_exception(self):
stream = asyncio.StreamReader(loop=self.loop)
stream.feed_data(b'line\n')
data = self.loop.run_until_complete(stream.readexactly(2))
self.assertEqual(b'li', data)
stream.set_exception(ValueError())
self.assertRaises(ValueError, self.loop.run_until_complete,
stream.readexactly(2))
def test_read_line_breaks(self):
# Read bytes without line breaks.
stream = asyncio.StreamReader(loop=self.loop)
stream.feed_data(b'line1')
stream.feed_data(b'line2')
data = self.loop.run_until_complete(stream.read(5))
self.assertEqual(b'line1', data)
self.assertEqual(b'line2', stream._buffer)
def test_readline_read_byte_count(self):
stream = asyncio.StreamReader(loop=self.loop)
stream.feed_data(self.DATA)
self.loop.run_until_complete(stream.readline())
data = self.loop.run_until_complete(stream.read(7))
self.assertEqual(b'line2\nl', data)
self.assertEqual(b'ine3\n', stream._buffer)
def test_exception_cancel(self):
stream = asyncio.StreamReader(loop=self.loop)
t = asyncio.Task(stream.readline(), loop=self.loop)
test_utils.run_briefly(self.loop)
t.cancel()
test_utils.run_briefly(self.loop)
# The following line fails if set_exception() isn't careful.
stream.set_exception(RuntimeError('message'))
test_utils.run_briefly(self.loop)
self.assertIs(stream._waiter, None)
def test_readline_nolimit_nowait(self):
# All needed data for the first 'readline' call will be
# in the buffer.
stream = asyncio.StreamReader(loop=self.loop)
stream.feed_data(self.DATA[:6])
stream.feed_data(self.DATA[6:])
line = self.loop.run_until_complete(stream.readline())
self.assertEqual(b'line1\n', line)
self.assertEqual(b'line2\nline3\n', stream._buffer)
def test_readexactly_zero_or_less(self):
# Read exact number of bytes (zero or less).
stream = asyncio.StreamReader(loop=self.loop)
stream.feed_data(self.DATA)
data = self.loop.run_until_complete(stream.readexactly(0))
self.assertEqual(b'', data)
self.assertEqual(self.DATA, stream._buffer)
data = self.loop.run_until_complete(stream.readexactly(-1))
self.assertEqual(b'', data)
self.assertEqual(self.DATA, stream._buffer)
def test_read(self):
# Read bytes.
stream = asyncio.StreamReader(loop=self.loop)
read_task = asyncio.Task(stream.read(30), loop=self.loop)
def cb():
stream.feed_data(self.DATA)
self.loop.call_soon(cb)
data = self.loop.run_until_complete(read_task)
self.assertEqual(self.DATA, data)
self.assertEqual(b'', stream._buffer)
def test_read_eof(self):
# Read bytes, stop at eof.
stream = asyncio.StreamReader(loop=self.loop)
read_task = asyncio.Task(stream.read(1024), loop=self.loop)
def cb():
stream.feed_eof()
self.loop.call_soon(cb)
data = self.loop.run_until_complete(read_task)
self.assertEqual(b'', data)
self.assertEqual(b'', stream._buffer)
def test_exception_waiter(self):
stream = asyncio.StreamReader(loop=self.loop)
@asyncio.coroutine
def set_err():
self.loop.call_soon(stream.set_exception, ValueError())
t1 = asyncio.Task(stream.readline(), loop=self.loop)
t2 = asyncio.Task(set_err(), loop=self.loop)
self.loop.run_until_complete(asyncio.wait([t1, t2], loop=self.loop))
self.assertRaises(ValueError, t1.result)
def __init__(self, loop=None):
super(CoreMqServerProtocol, self).__init__()
self.LOOP = loop or asyncio.get_event_loop()
self.uuid = str(uuid.uuid4())
self.transport = None
self.peer = None
self.local_ip = None
self.reader = asyncio.StreamReader()
self.subscriptions = []
self.options = dict()
self.is_replicant = False
self.hostname = None
ServerState.connections[self.uuid] = self
def __init__(self, factory, loop=None, logger=None, subscriptions=None):
super(CoreMqClientProtocol, self).__init__()
self.factory = factory
self.loop = loop or factory.loop
self.transport = None
self.reader = asyncio.StreamReader()
self.writer = None
self.uuid = None
self.logger = factory.get_logger(logger)
self.subscriptions = subscriptions or []
self.options = dict()
self.server = None
self.connected_future = asyncio.Future()
def _start_tls(self):
'''Start SSL protocol on the socket.'''
socket_ = self._writer.get_extra_info('socket')
try:
trollius.get_event_loop().remove_reader(socket_.fileno())
except ValueError as error:
raise trollius.ConnectionAbortedError() from error
self._writer.write(b'HTTP/1.1 200 Connection established\r\n\r\n')
yield From(self._writer.drain())
try:
trollius.get_event_loop().remove_writer(socket_.fileno())
except ValueError as error:
raise trollius.ConnectionAbortedError() from error
ssl_socket = ssl.wrap_socket(socket_,
server_side=True,
certfile=self._cert_filename,
keyfile=self._key_filename,
do_handshake_on_connect=False)
# FIXME: this isn't how to START TLS
for dummy in range(1200):
try:
ssl_socket.do_handshake()
break
except ssl.SSLError as error:
if error.errno in (ssl.SSL_ERROR_WANT_READ,
ssl.SSL_ERROR_WANT_WRITE):
_logger.debug('Do handshake %s', error)
yield From(trollius.sleep(0.05))
else:
raise
else:
_logger.error(_('Unable to handshake.'))
ssl_socket.close()
self._reject_request('Could not start TLS')
raise trollius.ConnectionAbortedError('Could not start TLS')
loop = trollius.get_event_loop()
reader = trollius.StreamReader(loop=loop)
protocol = trollius.StreamReaderProtocol(reader, loop=loop)
transport, dummy = yield From(
loop.create_connection(lambda: protocol, sock=ssl_socket))
writer = trollius.StreamWriter(transport, protocol, reader, loop)
self._reader = reader
self._writer = writer
def test_at_eof(self):
stream = asyncio.StreamReader(loop=self.loop)
self.assertFalse(stream.at_eof())
stream.feed_data(b'some data\n')
self.assertFalse(stream.at_eof())
self.loop.run_until_complete(stream.readline())
self.assertFalse(stream.at_eof())
stream.feed_data(b'some data\n')
stream.feed_eof()
self.loop.run_until_complete(stream.readline())
self.assertTrue(stream.at_eof())
def test_chunked_decoding_deflate(self):
import zlib
data = zlib.compress(b'foo')
fp = asyncio.StreamReader()
fp.feed_data(data)
r = HTTPResponse(fp, headers={'content-encoding': 'deflate'},
preload_content=False)
#yield From(r.init())
_d1 = yield From(r.read(3))
_d2 = yield From(r.read(1))
_d3 = yield From(r.read(2))
self.assertEqual(_d1, b'')
self.assertEqual(_d2, b'f')
self.assertEqual(_d3, b'oo')
def test_read_until_eof(self):
# Read all bytes until eof.
stream = asyncio.StreamReader(loop=self.loop)
read_task = asyncio.Task(stream.read(-1), loop=self.loop)
def cb():
stream.feed_data(b'chunk1\n')
stream.feed_data(b'chunk2')
stream.feed_eof()
self.loop.call_soon(cb)
data = self.loop.run_until_complete(read_task)
self.assertEqual(b'chunk1\nchunk2', data)
self.assertEqual(b'', stream._buffer)
def test_readline(self):
# Read one line. 'readline' will need to wait for the data
# to come from 'cb'
stream = asyncio.StreamReader(loop=self.loop)
stream.feed_data(b'chunk1 ')
read_task = asyncio.Task(stream.readline(), loop=self.loop)
def cb():
stream.feed_data(b'chunk2 ')
stream.feed_data(b'chunk3 ')
stream.feed_data(b'\n chunk4')
self.loop.call_soon(cb)
line = self.loop.run_until_complete(read_task)
self.assertEqual(b'chunk1 chunk2 chunk3 \n', line)
self.assertEqual(b' chunk4', stream._buffer)
def test_readexactly(self):
# Read exact number of bytes.
stream = asyncio.StreamReader(loop=self.loop)
n = 2 * len(self.DATA)
read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
def cb():
stream.feed_data(self.DATA)
stream.feed_data(self.DATA)
stream.feed_data(self.DATA)
self.loop.call_soon(cb)
data = self.loop.run_until_complete(read_task)
self.assertEqual(self.DATA + self.DATA, data)
self.assertEqual(self.DATA, stream._buffer)
def task():
rfd, wfd = os.pipe()
args = [sys.executable, '-c', code, str(wfd)]
pipe = os.fdopen(rfd, 'rb', 0)
reader = asyncio.StreamReader(loop=loop)
protocol = asyncio.StreamReaderProtocol(reader, loop=loop)
transport, _ = yield From(loop.connect_read_pipe(lambda: protocol, pipe))
kwds = {}
if sys.version_info >= (3, 2):
kwds['pass_fds'] = (wfd, )
proc = yield From(asyncio.create_subprocess_exec(*args, **kwds))
yield From(proc.wait())
os.close(wfd)
data = yield From(reader.read())
print("read = %r" % data.decode())
def test_chunked_decoding_gzip(self):
import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
data = compress.compress(b'foo')
data += compress.flush()
fp = asyncio.StreamReader()
fp.feed_data(data)
r = HTTPResponse(fp, headers={'content-encoding': 'gzip'},
preload_content=False)
yield From(r.init())
_d1 = yield From(r.read(10))
self.assertEqual(_d1, b'')
_d2 = yield From(r.read(5))
self.assertEqual(_d2, b'foo')
_d3 = yield From(r.read(2))
self.assertEqual(_d3, b'')
def test_readexactly_eof(self):
# Read exact number of bytes (eof).
stream = asyncio.StreamReader(loop=self.loop)
n = 2 * len(self.DATA)
read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
def cb():
stream.feed_data(self.DATA)
stream.feed_eof()
self.loop.call_soon(cb)
with self.assertRaises(asyncio.IncompleteReadError) as cm:
self.loop.run_until_complete(read_task)
self.assertEqual(cm.exception.partial, self.DATA)
self.assertEqual(cm.exception.expected, n)
self.assertEqual(str(cm.exception),
'18 bytes read on a total of 36 expected bytes')
self.assertEqual(b'', stream._buffer)
def test_read_all_from_pipe_reader(self):
# See asyncio issue 168. This test is derived from the example
# subprocess_attach_read_pipe.py, but we configure the
# StreamReader's limit so that twice it is less than the size
# of the data writter. Also we must explicitly attach a child
# watcher to the event loop.
code = """\
import os, sys
fd = int(sys.argv[1])
os.write(fd, b'data')
os.close(fd)
"""
rfd, wfd = os.pipe()
args = [sys.executable, '-c', code, str(wfd)]
pipe = io.open(rfd, 'rb', 0)
reader = asyncio.StreamReader(loop=self.loop, limit=1)
protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop)
transport, _ = self.loop.run_until_complete(
self.loop.connect_read_pipe(lambda: protocol, pipe))
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(self.loop)
try:
asyncio.set_child_watcher(watcher)
kw = {'loop': self.loop}
if six.PY3:
kw['pass_fds'] = set((wfd, ))
create = asyncio.create_subprocess_exec(*args, **kw)
proc = self.loop.run_until_complete(create)
self.loop.run_until_complete(proc.wait())
finally:
asyncio.set_child_watcher(None)
os.close(wfd)
data = self.loop.run_until_complete(reader.read(-1))
self.assertEqual(data, b'data')
def _test_pipe(self):
ADDRESS = r'\\.\pipe\_test_pipe-%s' % os.getpid()
with self.assertRaises(FileNotFoundError):
yield From(self.loop.create_pipe_connection(
asyncio.Protocol, ADDRESS))
[server] = yield From(self.loop.start_serving_pipe(
UpperProto, ADDRESS))
self.assertIsInstance(server, windows_events.PipeServer)
clients = []
for i in range(5):
stream_reader = asyncio.StreamReader(loop=self.loop)
protocol = asyncio.StreamReaderProtocol(stream_reader,
loop=self.loop)
trans, proto = yield From(self.loop.create_pipe_connection(
lambda: protocol, ADDRESS))
self.assertIsInstance(trans, asyncio.Transport)
self.assertEqual(protocol, proto)
clients.append((stream_reader, trans))
for i, (r, w) in enumerate(clients):
w.write('lower-{0}\n'.format(i).encode())
for i, (r, w) in enumerate(clients):
response = yield From(r.readline())
self.assertEqual(response, 'LOWER-{0}\n'.format(i).encode())
w.close()
server.close()
with self.assertRaises(FileNotFoundError):
yield From(self.loop.create_pipe_connection(
asyncio.Protocol, ADDRESS))
raise Return('done')
def test_ctor_global_loop(self, m_events):
stream = asyncio.StreamReader()
self.assertIs(stream._loop, m_events.get_event_loop.return_value)
