def test_constructor_heterogenous_futures(self):
fut1 = futures.Future(loop=self.one_loop)
fut2 = futures.Future(loop=self.other_loop)
with self.assertRaises(ValueError):
tasks.gather(fut1, fut2)
with self.assertRaises(ValueError):
tasks.gather(fut1, loop=self.other_loop)
def test_constructor_homogenous_futures(self):
children = [futures.Future(loop=self.other_loop) for i in range(3)]
fut = tasks.gather(*children)
self.assertIs(fut._loop, self.other_loop)
self._run_loop(self.other_loop)
self.assertFalse(fut.done())
fut = tasks.gather(*children, loop=self.other_loop)
self.assertIs(fut._loop, self.other_loop)
self._run_loop(self.other_loop)
self.assertFalse(fut.done())
def _check_empty_sequence(self, seq_or_iter):
events.set_event_loop(self.one_loop)
self.addCleanup(events.set_event_loop, None)
fut = tasks.gather(*seq_or_iter)
self.assertIsInstance(fut, futures.Future)
self.assertIs(fut._loop, self.one_loop)
self._run_loop(self.one_loop)
self.assertTrue(fut.done())
self.assertEqual(fut.result(), [])
fut = tasks.gather(*seq_or_iter, loop=self.other_loop)
self.assertIs(fut._loop, self.other_loop)
def test_constructor_loop_selection(self):
@tasks.coroutine
def coro():
return 'abc'
gen1 = coro()
gen2 = coro()
fut = tasks.gather(gen1, gen2)
self.assertIs(fut._loop, self.one_loop)
gen1.close()
gen2.close()
gen3 = coro()
gen4 = coro()
fut = tasks.gather(gen3, gen4, loop=self.other_loop)
self.assertIs(fut._loop, self.other_loop)
gen3.close()
gen4.close()
def test():
# Create connection
connection = yield from Connection.create(port=PORT, poolsize=2)
yield from connection.delete([ 'my-list' ])
results = []
# Sink: receive items using blocking pop
@asyncio.coroutine
def sink():
for i in range(0, 5):
reply = yield from connection.blpop(['my-list'])
self.assertIsInstance(reply, BlockingPopReply)
results.append(reply.value)
# Source: Push items on the queue
@asyncio.coroutine
def source():
for i in range(0, 5):
result = yield from connection.rpush('my-list', [str(i)])
yield from asyncio.sleep(.5)
# Run both coroutines.
f1 = asyncio.Task(source())
f2 = asyncio.Task(sink())
yield from gather(f1, f2)
# Test results.
self.assertEqual(results, [ str(i) for i in range(0, 5) ])
def mailer(event_loop, unused_tcp_port, request, recipients):
auth_mechanism, fail_on = request.param
callback = partial(MockSMTPServer.connected, auth_mechanism=auth_mechanism,
fail_on=fail_on, recipients=recipients)
task = start_server(callback, '127.0.0.1', unused_tcp_port)
server = event_loop.run_until_complete(task)
mailer = SMTPMailer(host='127.0.0.1', port=unused_tcp_port, username='******',
password='******', timeout=1)
event_loop.run_until_complete(mailer.start(Context()))
yield mailer
server.close()
tasks = Task.all_tasks(event_loop)
event_loop.run_until_complete(gather(*tasks))
def test_shield_gather(self):
child1 = futures.Future(loop=self.loop)
child2 = futures.Future(loop=self.loop)
parent = tasks.gather(child1, child2, loop=self.loop)
outer = tasks.shield(parent, loop=self.loop)
test_utils.run_briefly(self.loop)
outer.cancel()
test_utils.run_briefly(self.loop)
self.assertTrue(outer.cancelled())
child1.set_result(1)
child2.set_result(2)
test_utils.run_briefly(self.loop)
self.assertEqual(parent.result(), [1, 2])
def _check_success(self, **kwargs):
a, b, c = [futures.Future(loop=self.one_loop) for i in range(3)]
fut = tasks.gather(*self.wrap_futures(a, b, c), **kwargs)
cb = Mock()
fut.add_done_callback(cb)
b.set_result(1)
a.set_result(2)
self._run_loop(self.one_loop)
self.assertEqual(cb.called, False)
self.assertFalse(fut.done())
c.set_result(3)
self._run_loop(self.one_loop)
cb.assert_called_once_with(fut)
self.assertEqual(fut.result(), [2, 1, 3])
def test_gather_shield(self):
child1 = futures.Future(loop=self.loop)
child2 = futures.Future(loop=self.loop)
inner1 = tasks.shield(child1, loop=self.loop)
inner2 = tasks.shield(child2, loop=self.loop)
parent = tasks.gather(inner1, inner2, loop=self.loop)
test_utils.run_briefly(self.loop)
parent.cancel()
# This should cancel inner1 and inner2 but bot child1 and child2.
test_utils.run_briefly(self.loop)
self.assertIsInstance(parent.exception(), futures.CancelledError)
self.assertTrue(inner1.cancelled())
self.assertTrue(inner2.cancelled())
child1.set_result(1)
child2.set_result(2)
test_utils.run_briefly(self.loop)
def test_one_cancellation(self):
a, b, c, d, e = [futures.Future(loop=self.one_loop) for i in range(5)]
fut = tasks.gather(a, b, c, d, e)
cb = Mock()
fut.add_done_callback(cb)
a.set_result(1)
b.cancel()
self._run_loop(self.one_loop)
self.assertTrue(fut.done())
cb.assert_called_once_with(fut)
self.assertFalse(fut.cancelled())
self.assertIsInstance(fut.exception(), futures.CancelledError)
# Does nothing
c.set_result(3)
d.cancel()
e.set_exception(RuntimeError())
def test_one_exception(self):
a, b, c, d, e = [futures.Future(loop=self.one_loop) for i in range(5)]
fut = tasks.gather(*self.wrap_futures(a, b, c, d, e))
cb = Mock()
fut.add_done_callback(cb)
exc = ZeroDivisionError()
a.set_result(1)
b.set_exception(exc)
self._run_loop(self.one_loop)
self.assertTrue(fut.done())
cb.assert_called_once_with(fut)
self.assertIs(fut.exception(), exc)
# Does nothing
c.set_result(3)
d.cancel()
e.set_exception(RuntimeError())
def test_gather_shield(self):
child1 = futures.Future(loop=self.loop)
child2 = futures.Future(loop=self.loop)
inner1 = tasks.shield(child1, loop=self.loop)
inner2 = tasks.shield(child2, loop=self.loop)
parent = tasks.gather(inner1, inner2, loop=self.loop)
test_utils.run_briefly(self.loop)
parent.cancel()
# This should cancel inner1 and inner2 but bot child1 and child2.
test_utils.run_briefly(self.loop)
self.assertIsInstance(parent.exception(), futures.CancelledError)
self.assertTrue(inner1.cancelled())
self.assertTrue(inner2.cancelled())
child1.set_result(1)
child2.set_result(2)
test_utils.run_briefly(self.loop)
def test_one_cancellation(self):
a, b, c, d, e = [futures.Future(loop=self.one_loop) for i in range(5)]
fut = tasks.gather(a, b, c, d, e)
cb = Mock()
fut.add_done_callback(cb)
a.set_result(1)
b.cancel()
self._run_loop(self.one_loop)
self.assertTrue(fut.done())
cb.assert_called_once_with(fut)
self.assertFalse(fut.cancelled())
self.assertIsInstance(fut.exception(), futures.CancelledError)
# Does nothing
c.set_result(3)
d.cancel()
e.set_exception(RuntimeError())
def test_one_exception(self):
a, b, c, d, e = [futures.Future(loop=self.one_loop) for i in range(5)]
fut = tasks.gather(*self.wrap_futures(a, b, c, d, e))
cb = Mock()
fut.add_done_callback(cb)
exc = ZeroDivisionError()
a.set_result(1)
b.set_exception(exc)
self._run_loop(self.one_loop)
self.assertTrue(fut.done())
cb.assert_called_once_with(fut)
self.assertIs(fut.exception(), exc)
# Does nothing
c.set_result(3)
d.cancel()
e.set_exception(RuntimeError())
def test_return_exceptions(self):
a, b, c, d = [futures.Future(loop=self.one_loop) for i in range(4)]
fut = tasks.gather(*self.wrap_futures(a, b, c, d),
return_exceptions=True)
cb = Mock()
fut.add_done_callback(cb)
exc = ZeroDivisionError()
exc2 = RuntimeError()
b.set_result(1)
c.set_exception(exc)
a.set_result(3)
self._run_loop(self.one_loop)
self.assertFalse(fut.done())
d.set_exception(exc2)
self._run_loop(self.one_loop)
self.assertTrue(fut.done())
cb.assert_called_once_with(fut)
self.assertEqual(fut.result(), [3, 1, exc, exc2])
def test_return_exceptions(self):
a, b, c, d = [futures.Future(loop=self.one_loop) for i in range(4)]
fut = tasks.gather(*self.wrap_futures(a, b, c, d),
return_exceptions=True)
cb = Mock()
fut.add_done_callback(cb)
exc = ZeroDivisionError()
exc2 = RuntimeError()
b.set_result(1)
c.set_exception(exc)
a.set_result(3)
self._run_loop(self.one_loop)
self.assertFalse(fut.done())
d.set_exception(exc2)
self._run_loop(self.one_loop)
self.assertTrue(fut.done())
cb.assert_called_once_with(fut)
self.assertEqual(fut.result(), [3, 1, exc, exc2])
def _cancel_all_tasks(loop):
to_cancel = _all_tasks(loop)
if not to_cancel:
return
for task in to_cancel:
task.cancel()
loop.run_until_complete(
tasks.gather(*to_cancel, loop=loop, return_exceptions=True))
for task in to_cancel:
if task.cancelled():
continue
if task.exception() is not None:
loop.call_exception_handler({
'message': 'unhandled exception during asyncio.run() shutdown',
'exception': task.exception(),
'task': task,
})
async def test_tasklet_cancellation(event_loop: AbstractEventLoop):
gates = [Event(), Event()]
statuses = [event_loop.create_future(), event_loop.create_future()]
def on_done(status: Future, task: Future) -> None:
status.set_result("cancelled" if task.cancelled() else "done")
@tasklet
async def suspend(e: Event):
await e.wait()
(await suspend(gates[0])).add_done_callback(partial(on_done, statuses[0]))
# Next call should cancel the previously scheduled task
(await suspend(gates[1])).add_done_callback(partial(on_done, statuses[1]))
# Opening the gate should complete the newly scheduled task
gates[1].set()
await wait_for(gather(*statuses), timeout=1)
assert statuses[0].result() == "cancelled"
assert statuses[1].result() == "done"
def test_result_exception_one_cancellation(self):
a, b, c, d, e, f = [futures.Future(loop=self.one_loop)
for i in range(6)]
fut = tasks.gather(a, b, c, d, e, f, return_exceptions=True)
cb = Mock()
fut.add_done_callback(cb)
a.set_result(1)
zde = ZeroDivisionError()
b.set_exception(zde)
c.cancel()
self._run_loop(self.one_loop)
self.assertFalse(fut.done())
d.set_result(3)
e.cancel()
rte = RuntimeError()
f.set_exception(rte)
res = self.one_loop.run_until_complete(fut)
self.assertIsInstance(res[2], futures.CancelledError)
self.assertIsInstance(res[4], futures.CancelledError)
res[2] = res[4] = None
self.assertEqual(res, [1, zde, None, 3, None, rte])
cb.assert_called_once_with(fut)
def test_result_exception_one_cancellation(self):
a, b, c, d, e, f = [
futures.Future(loop=self.one_loop) for i in range(6)
]
fut = tasks.gather(a, b, c, d, e, f, return_exceptions=True)
cb = Mock()
fut.add_done_callback(cb)
a.set_result(1)
zde = ZeroDivisionError()
b.set_exception(zde)
c.cancel()
self._run_loop(self.one_loop)
self.assertFalse(fut.done())
d.set_result(3)
e.cancel()
rte = RuntimeError()
f.set_exception(rte)
res = self.one_loop.run_until_complete(fut)
self.assertIsInstance(res[2], futures.CancelledError)
self.assertIsInstance(res[4], futures.CancelledError)
res[2] = res[4] = None
self.assertEqual(res, [1, zde, None, 3, None, rte])
cb.assert_called_once_with(fut)
def _cancel_all_tasks(self, loop):
# https://github.com/python/cpython/blob/7f7dc673540c47db544878bb32d20d9bd1445b94/Lib/asyncio/runners.py#L55
from asyncio import tasks
to_cancel = tasks.all_tasks(loop)
if not to_cancel:
return
for task in to_cancel:
task.cancel()
loop.run_until_complete(
tasks.gather(*to_cancel, loop=loop, return_exceptions=True))
for task in to_cancel:
if task.cancelled():
continue
if task.exception() is not None:
loop.call_exception_handler({
"message":
"unhandled exception during asyncio.run() shutdown",
"exception": task.exception(),
"task": task,
})
def outer():
nonlocal proof, gatherer
gatherer = tasks.gather(child1, child2, loop=self.one_loop)
yield from gatherer
proof += 100
def outer():
yield from tasks.gather(inner(a), inner(b), loop=self.one_loop)
def outer():
nonlocal proof, gatherer
gatherer = tasks.gather(child1, child2, loop=self.one_loop)
yield from gatherer
proof += 100
async def test_2(self) -> None:
await wait_for(gather(self.wg.wait(), self.wg.wait()),
timeout=SMOL_TIME)
def getter(key_f, value_f):
""" Coroutine which processes one item. """
key, value = yield from gather(key_f, value_f, loop=self._result._loop)
key, value = self._parse(key, value)
return (key, value)
def create_server_tfo(self, protocol_factory, host=None, port=None,
*,
family=socket.AF_UNSPEC,
flags=socket.AI_PASSIVE,
sock=None,
backlog=100,
ssl=None,
reuse_address=None):
if isinstance(ssl, bool):
raise TypeError('ssl argument must be an SSLContext or None')
if host is not None or port is not None:
if sock is not None:
raise ValueError(
'host/port and sock can not be specified at the same time')
AF_INET6 = getattr(socket, 'AF_INET6', 0)
if reuse_address is None:
reuse_address = os.name == 'posix' and sys.platform != 'cygwin'
sockets = []
if host == '':
hosts = [None]
elif (isinstance(host, str) or
not isinstance(host, collections.Iterable)):
hosts = [host]
else:
hosts = host
fs = [self._create_server_getaddrinfo(host, port, family=family,
flags=flags)
for host in hosts]
infos = yield from tasks.gather(*fs, loop=self)
infos = set(itertools.chain.from_iterable(infos))
completed = False
try:
for res in infos:
af, socktype, proto, canonname, sa = res
try:
sock = socket.socket(af, socktype, proto)
except socket.error:
continue
sockets.append(sock)
if reuse_address:
sock.setsockopt(
socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
# set TCP_FASTOPEN
sock.setsockopt(socket.SOL_TCP, TCP_FASTOPEN, 20) # last arg is max_pending_tfo_request
if af == AF_INET6 and hasattr(socket, 'IPPROTO_IPV6'):
sock.setsockopt(socket.IPPROTO_IPV6,
socket.IPV6_V6ONLY,
True)
try:
sock.bind(sa)
except OSError as err:
raise OSError(err.errno, 'error while attempting '
'to bind on address %r: %s'
% (sa, err.strerror.lower()))
completed = True
finally:
if not completed:
for sock in sockets:
sock.close()
else:
if sock is None:
raise ValueError('Neither host/port nor sock were specified')
if not (sock.type & socket.SOCK_STREAM) == socket.SOCK_STREAM:
raise ValueError(
'A Stream Socket was expected, got {!r}'.format(sock))
sockets = [sock]
server = Server(self, sockets)
for sock in sockets:
sock.listen(backlog)
sock.setblocking(False)
self._start_serving(protocol_factory, sock, ssl, server, backlog)
return server
def aslist(self):
""" Return the result as a Python ``list``. """
return gather(* list(self._result))
def getter(key_f, value_f):
""" Coroutine which processes one item. """
key, value = yield from gather(key_f, value_f)
key, value = self._parse(key, value)
return { key: value }
def asset(self):
""" Return the result as a Python ``set``. """
result = yield from gather(* list(self._result))
return set(result)
def asset(self):
""" Return the result as a Python ``set``. """
result = yield from gather(*list(self._result),
loop=self._result._loop)
return set(result)
def aslist(self):
""" Return the result as a Python ``list``. """
return gather(*list(self._result), loop=self._result._loop)
def outer():
yield from tasks.gather(inner(a), inner(b), loop=self.one_loop)
