async def test_can_nest_nurseries():
"""
When nesting nurseries, nested exceptions are gathered into a tree
of MultiErrors.
"""
outer_error = RuntimeError('outer')
inner_error = RuntimeError('inner')
async def child(outer):
if outer:
raise outer_error
else:
raise inner_error
with pytest.raises(MultiError) as exc_info:
async with Nursery() as outer:
outer.start_soon(child(outer=True))
async with Nursery() as inner:
inner.start_soon(child(outer=False))
assert len(exc_info.value) == 2
first, second = exc_info.value
assert first is outer_error
assert isinstance(second, MultiError)
assert second[0] is inner_error
async def test_nursery_cant_be_reused():
"""
An already used nursery raises NurseryClosed when used again.
"""
nursery = Nursery()
async with nursery:
pass
with pytest.raises(NurseryClosed):
async with nursery:
pass
with pytest.raises(NurseryClosed):
nursery.start_soon(asyncio.sleep(0))
async def test_nursery_already_stopped(self):
async with Nursery() as nursery:
pass
exc = pytest.raises(NurseryError, nursery.start_soon, self.async_error,
'fail')
exc.match('This nursery has already been closed')
async def test_timeout_is_respected():
"""
When using ``async_timeout``, the timeout is respected and all tasks
(both parent and children) are properly cancelled.
"""
parent_cancelled = child_cancelled = False
async def sleepy():
nonlocal child_cancelled
try:
await asyncio.sleep(1000 * 1000)
except asyncio.CancelledError:
child_cancelled = True
raise
# The outer timeout is for terminating the test
# in case the code doesn't work as intended.
async with async_timeout.timeout(1):
async with async_timeout.timeout(0.01):
try:
async with Nursery() as nursery:
nursery.start_soon(sleepy())
await asyncio.sleep(1000 * 1000)
except asyncio.CancelledError:
parent_cancelled = True
assert parent_cancelled and child_cancelled
async def test_success(self, queue):
async with Nursery() as nursery:
nursery.start_soon(queue.put, 'a')
nursery.start_soon(queue.put, 'b')
assert queue.get_nowait() == 'a'
assert queue.get_nowait() == 'b'
async def test_can_cancel_remaining():
"""
A child task can cancel other remaining children.
"""
cancelled = False
async def runs_forever():
nonlocal cancelled
try:
await asyncio.sleep(1000)
except asyncio.CancelledError:
cancelled = True
raise
async def canceller(nursery):
nursery.cancel_remaining()
async with Nursery() as nursery:
nursery.start_soon(runs_forever())
nursery.start_soon(canceller(nursery))
# This allows children to be actually executed
await asyncio.sleep(0)
# This allows to execute cancellation logic in `runs_forever`
await asyncio.sleep(0)
# assert within context manager to be sure
# `runs_forever` hasn't been cancelled by the nursery
assert cancelled
async def test_host_cancelled_before_aexit(self, queue):
with pytest.raises(asyncio.CancelledError):
async with Nursery() as nursery:
nursery.start_soon(self.delayed_put, queue, 'a')
raise CancelledError
assert queue.empty()
async def test_host_cancelled_during_aexit(self, event_loop, queue):
with pytest.raises(asyncio.CancelledError):
async with Nursery() as nursery:
nursery.start_soon(self.delayed_put, queue, 'a')
event_loop.call_soon(asyncio.Task.current_task().cancel)
assert queue.empty()
async def test_child_crash_propagation():
"""
An error in one child cancels other children
and the context manager's body.
"""
looper_cancelled = False
async def looper():
nonlocal looper_cancelled
try:
while True:
await asyncio.sleep(0)
except asyncio.CancelledError:
looper_cancelled = True
error = ValueError('crashed')
async def crasher():
raise error
with pytest.raises(ValueError) as excinfo:
async with Nursery() as nursery:
nursery.start_soon(looper())
nursery.start_soon(crasher())
assert looper_cancelled
assert excinfo.value is error
async def create_handy_nursery():
try:
async with Nursery() as nursery:
yield nursery
except MultiError as e:
if len(e.exceptions) == 1:
raise e.exceptions[0]
raise
async def test_host_exception(self, queue):
with pytest.raises(Exception) as exc:
async with Nursery() as nursery:
nursery.start_soon(self.delayed_put, queue, 'a', 5)
raise Exception('dummy error')
exc.match('dummy error')
assert queue.empty()
async def test_multi_error(self):
with pytest.raises(MultiError) as exc:
async with Nursery() as nursery:
nursery.start_soon(self.async_error, 'task1')
nursery.start_soon(self.async_error, 'task2')
assert len(exc.value.exceptions) == 2
assert sorted(str(e)
for e in exc.value.exceptions) == ['task1', 'task2']
async def test_multi_error_host(self):
with pytest.raises(MultiError) as exc:
async with Nursery() as nursery:
nursery.start_soon(self.async_error, 'child', delay=2)
await asyncio.sleep(0.1)
raise Exception('host')
assert len(exc.value.exceptions) == 2
assert [str(e) for e in exc.value.exceptions] == ['host', 'child']
async def test_parent_block_error_basic():
"""
Context manager's body raises properly.
"""
error = ValueError('whoops')
with pytest.raises(ValueError) as excinfo:
async with Nursery():
raise error
assert excinfo.value is error
async def test_child_crash_wakes_parent():
"""
If a child task crashes, the context manager's body is cancelled.
"""
async def crasher():
raise ValueError
with pytest.raises(ValueError):
async with Nursery() as nursery:
nursery.start_soon(crasher())
await asyncio.sleep(1000 * 1000)
async def test_two_child_crashes():
"""
Multiple child crashes are propagated as a MultiError.
"""
async def crasher(etype):
raise etype
with pytest.raises(MultiError) as excinfo:
async with Nursery() as nursery:
nursery.start_soon(crasher(KeyError))
nursery.start_soon(crasher(ValueError))
assert set(type(exc) for exc in excinfo.value) == {ValueError, KeyError}
async def test_child_crash_basic():
"""
A child exception is propagated to the parent coroutine.
"""
error = ValueError('whoops')
async def child():
raise error
try:
async with Nursery() as nursery:
nursery.start_soon(child())
except ValueError as exc:
assert exc is error
async def test_parent_and_child_both_crash():
"""
When both parent and child crash, errors are propagated
as a MultiError.
"""
async def crasher():
raise ValueError
with pytest.raises(MultiError) as excinfo:
async with Nursery() as nursery:
nursery.start_soon(crasher())
raise KeyError
assert set(type(exc) for exc in excinfo.value) == {ValueError, KeyError}
async def test_child_crash_basic():
"""
A child exception is propagated to the parent coroutine.
"""
error = ValueError('whoops')
async def child():
raise error
with pytest.raises(MultiError) as excinfo:
async with Nursery() as nursery:
nursery.start_soon(child())
assert len(excinfo.value) == 1
assert excinfo.value[0] is error
async def test_child_crash_wakes_parent():
"""
If a child task crashes, the context manager's body is cancelled.
"""
error = ValueError('crashed')
async def crasher():
raise error
with pytest.raises(MultiError) as excinfo:
async with Nursery() as nursery:
nursery.start_soon(crasher())
await asyncio.sleep(1000 * 1000)
assert len(excinfo.value) == 1
assert excinfo.value[0] is error
async def test_child_can_spawn_children():
"""
A child task can spawn even more children tasks.
"""
i = 0
async def child(nursery):
nonlocal i
i += 1
if i == 3:
return
await asyncio.sleep(0)
nursery.start_soon(child(nursery))
async with Nursery() as nursery:
nursery.start_soon(child(nursery))
async def test_basic_interleave():
"""
Children are running one after another; when all finished, the task nursery
is joined with no errors.
"""
async def looper(whoami, record):
for i in range(3):
record.append((whoami, i))
await asyncio.sleep(0)
record = []
async with Nursery() as nursery:
nursery.start_soon(looper('a', record))
nursery.start_soon(looper('b', record))
assert record == [('a', 0), ('b', 0), ('a', 1), ('b', 1), ('a', 2),
('b', 2)]
async def test_shielded_child_continues_running():
"""
A shielded child continues running.
"""
work_done = False
async def worker():
nonlocal work_done
await asyncio.sleep(0)
work_done = True
try:
async with Nursery() as nursery:
nursery.start_soon(asyncio.shield(worker()))
raise RuntimeError
except RuntimeError:
pass
assert work_done
async def fetch_articles_scores(urls):
morph = pymorphy2.MorphAnalyzer()
charged_words = load_charged_words('charged_dict/negative_words.txt')
async with aiohttp.ClientSession() as session:
async with Nursery() as nursery:
tasks = []
for num, url in enumerate(urls):
task = nursery.start_soon(
process_article(session, morph, charged_words, url))
tasks.append(task)
results = await asyncio.wait(tasks)
scores = []
for result in results[0]:
scores.append({
"status": str(result.result()[0]),
"url": result.result()[1],
"score": result.result()[2],
"words_count": result.result()[3]
})
return scores
async def start(self, loop: asyncio.AbstractEventLoop = None) -> None:
assert not self._closed_future, 'Consumer already started.'
loop = loop or asyncio.get_event_loop()
self._closed_future = asyncio.Future(loop=loop)
self._closed_ok = asyncio.Event(loop=loop)
reconnect_attempts = 0
while True:
connection_closed_future: asyncio.Future[None] = asyncio.Future(
loop=loop)
try:
await self._connect(
connection_closed_future=connection_closed_future,
loop=loop,
)
reconnect_attempts = 0
async with Nursery() as nursery:
nursery.start_soon(connection_closed_future)
nursery.start_soon(self._closed_future)
nursery.start_soon(self._process_queue(loop=loop))
except (aioamqp.AioamqpException, OSError) as exc:
logger.exception(str(exc))
reconnect_attempts += 1
timeout = min(
self.default_reconnect_timeout * reconnect_attempts,
self.max_reconnect_timeout)
logger.info('Trying to reconnect in %d seconds.', timeout)
await asyncio.sleep(timeout, loop=loop)
except _ConsumerCloseException:
break
await self._disconnect()
self._closed_future = None
self._closed_ok.set()
async def process(self):
cancelled = set()
async def cancelled_watcher():
try:
while True:
await asyncio.sleep(100)
except asyncio.CancelledError:
cancelled.add(True)
try:
async with Nursery() as nursery:
nursery.start_soon(cancelled_watcher())
nursery.start_soon(self._process_data(self._reader))
nursery.start_soon(self._process_heartbeat(self._writer))
except MultiError as e:
if len(e.exceptions) == 1:
raise e.exceptions[0] from e
if cancelled:
raise asyncio.CancelledError
async def test_timeout_is_respected():
"""
When using ``async_timeout``, the timeout is respected and all tasks
(both parent and children) are properly cancelled.
"""
parent_cancelled = child_cancelled = False
async def sleepy():
nonlocal child_cancelled
try:
await asyncio.sleep(1000 * 1000)
except asyncio.CancelledError:
child_cancelled = True
async with async_timeout.timeout(0.01):
try:
async with Nursery() as nursery:
nursery.start_soon(sleepy())
await asyncio.sleep(1000 * 1000)
except asyncio.CancelledError:
parent_cancelled = True
assert parent_cancelled and child_cancelled
async def main(self):
async with Nursery() as ns:
ns.start_soon(self.queue_monitor_task)
ns.start_soon(self.cb_feed.run())
ns.start_soon(self.pull_sink())
def test_nursery_not_opened(self):
exc = pytest.raises(NurseryError,
Nursery().start_soon, self.async_error, 'fail')
exc.match('This nursery has not been opened yet')
async def test_nursery_already_running(self):
with pytest.raises(NurseryError) as exc:
async with Nursery() as nursery, nursery:
pass
exc.match('This nursery is already running')