async def test_aio_concurrent_cancel(patches):
concurrent = aio.concurrent(["CORO"])
patched = patches(
("concurrent.cancel_tasks", dict(new_callable=AsyncMock)),
("concurrent.close", dict(new_callable=AsyncMock)),
("concurrent.close_coros", dict(new_callable=AsyncMock)),
("concurrent.sem", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
waiter = MagicMock()
class SubmitTask:
def __init__(self):
self.cancel = MagicMock()
def __await__(self):
waiter()
yield
concurrent.submit_task = SubmitTask()
with patched as (m_cancel, m_close, m_coros, m_sem):
assert not await concurrent.cancel()
assert (list(m_close.call_args) == [(), {}])
assert (list(m_sem.return_value.release.call_args) == [(), {}])
assert (list(m_cancel.call_args) == [(), {}])
assert (list(m_coros.call_args) == [(), {}])
assert (list(waiter.call_args) == [(), {}])
async def test_aio_concurrent_cancel_tasks(patches, bad):
concurrent = aio.concurrent(["CORO"])
patched = patches(
("concurrent.running_tasks", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
tasks = []
waiter = MagicMock()
class Task:
def __init__(self, i):
self.i = i
self.cancel = MagicMock()
def __await__(self):
waiter()
if self.i == bad:
raise BaseException("AN ERROR OCCURRED")
for i in range(0, 7):
tasks.append(Task(i))
with patched as (m_running, ):
m_running.return_value = tasks
assert not await concurrent.cancel_tasks()
assert (list(list(c) for c in waiter.call_args_list) == [[(), {}]] * 7)
for task in tasks:
assert (list(task.cancel.call_args) == [(), {}])
def test_aio_concurrent_validate_coro(patches, awaitable, state):
concurrent = aio.concurrent(["CORO"])
patched = patches("inspect.getcoroutinestate", prefix="tools.base.aio")
# we cant patch inspect.isawaitable without fooing unittest
def unawaitable():
pass
async def coro():
pass
awaits = (coro() if awaitable else unawaitable)
with patched as (m_state, ):
m_state.return_value = state
if awaitable and state == inspect.CORO_CREATED:
assert not concurrent.validate_coro(awaits)
else:
with pytest.raises(aio.ConcurrentError) as e:
concurrent.validate_coro(awaits)
if not awaitable:
assert (
e.value.args[0] == f'Provided input was not a coroutine: {awaits}')
assert not m_state.called
return
awaits.close()
assert (list(m_state.call_args) == [(awaits, ), {}])
if state != inspect.CORO_CREATED:
assert (e.value.args[0] ==
f'Provided coroutine has already been fired: {awaits}')
async def test_aio_concurrent_close_coros(patches, consumes_generator, bad):
concurrent = aio.concurrent(["CORO"])
patched = patches(
"concurrent.close",
("concurrent.iter_coros", dict(new_callable=PropertyMock)),
("concurrent.consumes_generator", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
coros = []
for i in range(0, 7):
coro = MagicMock()
if i == bad:
coro.close.side_effect = BaseException("AN ERROR OCCURRED")
coros.append(coro)
async def iter_coros():
for coro in coros:
yield coro
with patched as (m_close, m_iter, m_isgen):
m_isgen.return_value = consumes_generator
m_iter.return_value = iter_coros
assert not await concurrent.close_coros()
if consumes_generator:
assert not m_iter.called
return
assert (list(m_iter.call_args) == [(), {}])
for coro in coros:
assert (list(coro.close.call_args) == [(), {}])
async def check_yapf(self) -> None:
"""Run flake8 on files and/or repo"""
futures = aio.concurrent(
self.yapf_format(python_file) for python_file in self.yapf_files)
async for (python_file, (reformatted, encoding, changed)) in futures:
self.yapf_result(python_file, reformatted, changed)
async def test_aio_concurrent_ready(patches, closed_before, closed_after,
nolimit):
concurrent = aio.concurrent(["CORO"])
patched = patches(("concurrent.closed", dict(new_callable=PropertyMock)),
("concurrent.nolimit", dict(new_callable=PropertyMock)),
("concurrent.sem", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
class DummyCloser:
order_mock = MagicMock()
close_calls = 0
async def _acquire(self):
self.order_mock("ACQUIRE")
def _nolimit(self):
self.order_mock("NOLIMIT")
return nolimit
def _closed(self):
self.order_mock("CLOSED")
self.close_calls += 1
if self.close_calls == 1:
return closed_before
if self.close_calls == 2:
return closed_after
closer = DummyCloser()
with patched as (m_closed, m_nolimit, m_sem):
m_nolimit.side_effect = closer._nolimit
m_closed.side_effect = closer._closed
m_sem.return_value.acquire = closer._acquire
assert (await
concurrent.ready() == ((not closed_before and not closed_after)
if not nolimit else not closed_before))
if closed_before:
assert not m_nolimit.called
assert not m_sem.called
assert (list(list(c) for c in closer.order_mock.call_args_list) == [[
('CLOSED', ), {}
]])
return
if nolimit:
assert not m_sem.called
assert (list(list(c) for c in closer.order_mock.call_args_list) == [[
('CLOSED', ), {}
], [('NOLIMIT', ), {}]])
return
assert (list(list(c)
for c in closer.order_mock.call_args_list) == [[('CLOSED', ),
{}],
[('NOLIMIT', ),
{}],
[('ACQUIRE', ),
{}],
[('CLOSED', ),
{}]])
def test_aio_concurrent_submission_lock(patches):
concurrent = aio.concurrent(["CORO"])
patched = patches("asyncio", prefix="tools.base.aio")
with patched as (m_asyncio, ):
assert concurrent.submission_lock == m_asyncio.Lock.return_value
assert (list(m_asyncio.Lock.call_args) == [(), {}])
assert "submission_lock" in concurrent.__dict__
def test_aio_concurrent_out(patches):
concurrent = aio.concurrent(["CORO"])
patched = patches("asyncio", prefix="tools.base.aio")
with patched as (m_asyncio, ):
assert concurrent.out == m_asyncio.Queue.return_value
assert (list(m_asyncio.Queue.call_args) == [(), {}])
assert "out" in concurrent.__dict__
def test_aio_concurrent_remember_task():
concurrent = aio.concurrent(["CORO"])
concurrent._running = MagicMock()
task = MagicMock()
assert not concurrent.remember_task(task)
assert (list(concurrent._running.append.call_args) == [(task, ), {}])
assert (list(task.add_done_callback.call_args) == [
(concurrent.forget_task, ), {}
])
def test_aio_concurrent_should_error(result, yield_exceptions):
concurrent = aio.concurrent(["CORO"])
concurrent.yield_exceptions = yield_exceptions
if isinstance(result, type) and issubclass(result, BaseException):
result = result()
assert (concurrent.should_error(result) == (
(isinstance(result, aio.ConcurrentIteratorError)
or isinstance(result, aio.ConcurrentError) and not yield_exceptions)))
def test_aio_concurrent_nolimit(patches, limit):
concurrent = aio.concurrent(["CORO"])
patched = patches(("concurrent.limit", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
with patched as (m_limit, ):
m_limit.return_value = limit
assert concurrent.nolimit == (limit == -1)
assert "nolimit" in concurrent.__dict__
def test_aio_concurrent_consumes_generator(patches):
concurrent = aio.concurrent(["CORO"])
patched = patches("isinstance", prefix="tools.base.aio")
with patched as (m_inst, ):
assert concurrent.consumes_generator == m_inst.return_value
assert (list(m_inst.call_args) == [(["CORO"], (types.AsyncGeneratorType,
types.GeneratorType)), {}])
assert "consumes_generator" in concurrent.__dict__
def test_aio_concurrent_default_limit(patches, cpus):
concurrent = aio.concurrent(["CORO"])
patched = patches("min", "os", prefix="tools.base.aio")
with patched as (m_min, m_os):
m_os.cpu_count.return_value = cpus
assert concurrent.default_limit == m_min.return_value
assert (list(m_min.call_args) == [(32, (cpus or 0) + 4), {}])
assert "default_limit" not in concurrent.__dict__
def test_aio_concurrent_submitting(patches, locked):
concurrent = aio.concurrent(["CORO"])
patched = patches(
("concurrent.submission_lock", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
with patched as (m_submission_lock, ):
m_submission_lock.return_value.locked.return_value = locked
assert concurrent.submitting == locked
assert "submitting" not in concurrent.__dict__
def test_aio_concurrent_running(patches, empty):
concurrent = aio.concurrent(["CORO"])
patched = patches(
("concurrent.running_queue", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
with patched as (m_running_queue, ):
m_running_queue.return_value.empty.return_value = empty
assert concurrent.running == (not empty)
assert "running" not in concurrent.__dict__
def test_aio_concurrent_closed(patches, locked):
concurrent = aio.concurrent(["CORO"])
patched = patches(
("concurrent.closing_lock", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
with patched as (m_closing_lock, ):
m_closing_lock.return_value.locked.return_value = locked
assert concurrent.closed == locked
assert "closed" not in concurrent.__dict__
def test_aio_concurrent_sem(patches):
concurrent = aio.concurrent(["CORO"])
patched = patches("asyncio",
("concurrent.limit", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
with patched as (m_asyncio, m_limit):
assert concurrent.sem == m_asyncio.Semaphore.return_value
assert (list(m_asyncio.Semaphore.call_args) == [(m_limit.return_value, ),
{}])
assert "sem" in concurrent.__dict__
def test_aio_concurrent_active(patches, running, submitting):
concurrent = aio.concurrent(["CORO"])
patched = patches(
"asyncio", ("concurrent.submitting", dict(new_callable=PropertyMock)),
("concurrent.running", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
with patched as (m_asyncio, m_submit, m_run):
m_submit.return_value = submitting
m_run.return_value = running
assert concurrent.active == (submitting or running)
assert "active" not in concurrent.__dict__
def test_aio_concurrent_forget_task(patches, closed):
concurrent = aio.concurrent(["CORO"])
patched = patches(("concurrent.closed", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
concurrent._running = MagicMock()
with patched as (m_closed, ):
m_closed.return_value = closed
assert not concurrent.forget_task("TASK")
if closed:
assert not concurrent._running.remove.called
return
assert (list(concurrent._running.remove.call_args) == [("TASK", ), {}])
def test_aio_concurrent_limit(patches, limit):
concurrent = aio.concurrent(["CORO"])
patched = patches(
("concurrent.default_limit", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
concurrent._limit = limit
with patched as (m_limit, ):
assert concurrent.limit == (limit or m_limit.return_value)
if limit:
assert not m_limit.called
assert "limit" in concurrent.__dict__
def test_aio_concurrent_dunder_aiter(patches):
concurrent = aio.concurrent(["CORO"])
patched = patches("asyncio",
"concurrent.output",
("concurrent.submit", dict(new_callable=MagicMock)),
prefix="tools.base.aio")
with patched as (m_asyncio, m_output, m_submit):
assert concurrent.__aiter__() == m_output.return_value
assert concurrent.submit_task == m_asyncio.create_task.return_value
assert (list(m_submit.call_args) == [(), {}])
assert (list(m_asyncio.create_task.call_args) == [
(m_submit.return_value, ), {}
])
async def test_aio_concurrent_close(patches, closed):
concurrent = aio.concurrent(["CORO"])
patched = patches(
("concurrent.closed", dict(new_callable=PropertyMock)),
("concurrent.closing_lock", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
with patched as (m_closed, m_lock):
m_closed.return_value = closed
m_lock.return_value.acquire = AsyncMock()
assert not await concurrent.close()
if closed:
assert not m_lock.called
else:
assert (list(m_lock.return_value.acquire.call_args) == [(), {}])
async def test_aio_concurrent_exit_on_completion(patches, active, closed):
concurrent = aio.concurrent(["CORO"])
patched = patches(("concurrent.active", dict(new_callable=PropertyMock)),
("concurrent.closed", dict(new_callable=PropertyMock)),
("concurrent.out", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
with patched as (m_active, m_closed, m_out):
m_out.return_value.put = AsyncMock()
m_active.return_value = active
m_closed.return_value = closed
assert not await concurrent.exit_on_completion()
if closed or active:
assert not m_out.called
return
assert (list(m_out.return_value.put.call_args) == [(aio._sentinel, ), {}])
def test_aio_concurrent_constructor(limit, yield_exceptions):
kwargs = {}
if limit == "XX":
limit = None
else:
kwargs["limit"] = limit
if yield_exceptions is not None:
kwargs["yield_exceptions"] = yield_exceptions
concurrent = aio.concurrent(["CORO"], **kwargs)
assert concurrent._coros == ["CORO"]
assert concurrent._limit == limit
assert (concurrent.yield_exceptions == (False if yield_exceptions is None
else yield_exceptions))
assert concurrent._running == []
assert concurrent.running_tasks is concurrent._running
assert "running_tasks" in concurrent.__dict__
async def test_aio_concurrent_coros(patches, raises, close_raises):
concurrent = aio.concurrent(["CORO"])
patched = patches(
("concurrent.iter_coros", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
results = []
return_coros = [f"CORO{i}" for i in range(0, 3)]
m_aclose = AsyncMock()
if close_raises:
m_aclose.side_effect = close_raises()
class Coros:
aclose = m_aclose
def __call__(self):
return self
async def __aiter__(self):
if raises:
raise raises("AN ERROR OCCURRED")
for coro in return_coros:
yield coro
with patched as (m_coros, ):
coros = Coros()
m_coros.return_value = coros
if raises == BaseException:
with pytest.raises(BaseException):
async for coro in concurrent.coros:
pass
else:
async for coro in concurrent.coros:
results.append(coro)
if raises == GeneratorExit:
assert (list(coros.aclose.call_args) == [(), {}])
return
assert not coros.aclose.called
assert "coros" not in concurrent.__dict__
if raises:
return
assert results == return_coros
async def test_aio_concurrent_on_task_complete(patches, closed, nolimit,
decrement):
concurrent = aio.concurrent(["CORO"])
patched = patches(
("concurrent.exit_on_completion", dict(new_callable=AsyncMock)),
("concurrent.closed", dict(new_callable=PropertyMock)),
("concurrent.out", dict(new_callable=PropertyMock)),
("concurrent.running_queue", dict(new_callable=PropertyMock)),
("concurrent.nolimit", dict(new_callable=PropertyMock)),
("concurrent.sem", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
kwargs = {}
if decrement is not None:
kwargs["decrement"] = decrement
with patched as (m_complete, m_closed, m_out, m_running_queue, m_nolimit,
m_sem):
m_nolimit.return_value = nolimit
m_closed.return_value = closed
m_out.return_value.put = AsyncMock()
assert not await concurrent.on_task_complete("RESULT", **kwargs)
if closed:
assert not m_complete.called
assert not m_nolimit.called
assert not m_sem.called
assert not m_running_queue.called
assert not m_out.return_value.put.called
return
assert (list(m_out.return_value.put.call_args) == [("RESULT", ), {}])
if nolimit:
assert not m_sem.return_value.release.called
else:
assert (list(m_sem.return_value.release.call_args) == [(), {}])
if decrement or decrement is None:
assert (list(m_running_queue.return_value.get_nowait.call_args) == [(),
{}
])
else:
assert not m_running_queue.return_value.get_nowait.called
assert (list(m_complete.call_args) == [(), {}])
async def test_aio_concurrent_create_task(patches):
concurrent = aio.concurrent(["CORO"])
patched = patches(
"asyncio",
"concurrent.remember_task",
("concurrent.task", dict(new_callable=MagicMock)),
("concurrent.running_queue", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
with patched as (m_asyncio, m_rem, m_task, m_running_queue):
assert not await concurrent.create_task("CORO")
assert (list(m_running_queue.return_value.put_nowait.call_args) == [
(None, ), {}
])
assert (list(m_task.call_args) == [("CORO", ), {}])
assert (list(m_asyncio.create_task.call_args) == [(m_task.return_value, ),
{}])
assert (list(m_rem.call_args) == [(m_asyncio.create_task.return_value, ),
{}])
async def test_aio_concurrent_iter_coros(patches, raises, consumes_async):
concurrent = aio.concurrent(["CORO"])
patched = patches(
("concurrent.consumes_async", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
coros = [f"CORO{i}" for i in range(0, 7)]
exception = BaseException("AN RAISES OCCURRED")
def iter_coros():
if raises:
raise exception
for coro in coros:
yield coro
async def async_iter_coros():
if raises:
raise exception
for coro in coros:
yield coro
concurrent._coros = (async_iter_coros()
if consumes_async else iter_coros())
results = []
with patched as (m_async, ):
m_async.return_value = consumes_async
async for result in concurrent.iter_coros():
results.append(result)
if raises:
error = results[0]
assert isinstance(error, aio.ConcurrentIteratorError)
assert error.args[0] is exception
assert results == [error]
return
assert results == coros
async def test_aio_concurrent_task(patches, raises):
concurrent = aio.concurrent(["CORO"])
patched = patches("concurrent.on_task_complete", prefix="tools.base.aio")
if raises:
exception = raises("AN ERROR OCCURRED")
async def coro():
if raises:
raise exception
return 23
with patched as (m_complete, ):
assert not await concurrent.task(coro())
result = m_complete.call_args[0][0]
if not raises:
assert result == 23
else:
assert isinstance(result, aio.ConcurrentExecutionError)
assert result.args[0] is exception
assert (list(m_complete.call_args) == [(result, ), {}])
async def test_aio_concurrent_submit(patches, coros, unready, valid_raises,
iter_errors):
concurrent = aio.concurrent(["CORO"])
patched = patches(
"isinstance",
"concurrent.validate_coro",
("concurrent.exit_on_completion", dict(new_callable=AsyncMock)),
("concurrent.create_task", dict(new_callable=AsyncMock)),
("concurrent.on_task_complete", dict(new_callable=AsyncMock)),
("concurrent.ready", dict(new_callable=AsyncMock)),
("concurrent.coros", dict(new_callable=PropertyMock)),
("concurrent.out", dict(new_callable=PropertyMock)),
("concurrent.submission_lock", dict(new_callable=PropertyMock)),
prefix="tools.base.aio")
m_order = MagicMock()
class DummyReady:
counter = 0
def ready(self):
if self.counter >= unready:
self.counter += 1
return False
self.counter += 1
return True
ready = DummyReady()
async def acquire():
m_order("ACQUIRE")
def release():
m_order("RELEASE")
corolist = [MagicMock() for coro in range(1, coros)]
async def iter_coros():
for coro in corolist:
m_order(coro)
yield coro
valid_errors = ((valid_raises == Exception) and coros > 1
and not unready == 0 and not iter_errors)
with patched as (m_inst, m_valid, m_exit, m_create, m_complete, m_ready,
m_coros, m_out, m_lock):
m_out.return_value.put = AsyncMock()
m_inst.return_value = iter_errors
m_valid.side_effect = valid_raises
m_ready.side_effect = ready.ready
m_coros.return_value = iter_coros()
m_lock.return_value.acquire.side_effect = acquire
m_lock.return_value.release.side_effect = release
if valid_errors:
with pytest.raises(Exception):
await concurrent.submit()
else:
assert not await concurrent.submit()
if valid_errors:
assert not m_lock.return_value.called
assert not m_exit.called
else:
assert (list(m_lock.return_value.release.call_args) == [(), {}])
assert (list(m_exit.call_args) == [(), {}])
if coros < 2:
assert not m_valid.called
assert not m_inst.called
assert not m_complete.called
assert not m_create.called
assert not m_ready.called
assert not m_out.return_value.put.called
return
should_close_coro = (not iter_errors and not valid_errors
and (len(corolist) > unready))
if should_close_coro:
assert corolist[unready].close.called
else:
assert not any(coro.close.called for coro in corolist)
if iter_errors:
assert (list(list(c)
for c in m_out.return_value.put.call_args_list) == [[
(corolist[0], ), {}
]])
assert (list(list(c) for c in m_inst.call_args_list) == [[
(corolist[0], aio.ConcurrentIteratorError), {}
]])
assert not m_ready.called
assert not m_valid.called
assert not m_complete.called
assert not m_create.called
return
if valid_errors:
assert (list(list(c) for c in m_inst.call_args_list) == [[
(corolist[0], aio.ConcurrentIteratorError), {}
]])
assert (list(list(c) for c in m_ready.call_args_list) == [[(), {}]])
assert (list(list(c)
for c in m_valid.call_args_list) == [[(corolist[0], ),
{}]])
assert not m_complete.called
assert not m_create.called
assert (list(list(c)
for c in m_order.call_args_list) == ([[('ACQUIRE', ), {}],
[(corolist[0], ),
{}]]))
return
assert not m_out.return_value.put.called
assert (list(list(c) for c in m_ready.call_args_list) == [[(), {}]] *
min(coros - 1, unready + 1 or 1))
assert (list(list(c) for c in m_valid.call_args_list) == [[
(corolist[i - 1], ), {}
] for i in range(1, min(coros, unready + 1))])
assert (list(list(c) for c in m_order.call_args_list) == (
[[('ACQUIRE', ), {}]] + [[(corolist[i - 1], ), {}]
for i in range(1, min(coros, unready + 2))] +
[[('RELEASE', ), {}]]))
if valid_raises:
assert len(m_complete.call_args_list) == max(min(coros - 1, unready),
0)
for c in m_complete.call_args_list:
error = list(c)[0][0]
assert isinstance(error, aio.ConcurrentError)
assert (list(c) == [(error, ), {'decrement': False}])
assert not m_create.called
return
assert not m_complete.called
assert (list(list(c) for c in m_create.call_args_list) == [[
(corolist[i - 1], ), {}
] for i in range(1, min(coros, unready + 1))])
