async def test_aio_concurrent_cancel(patches): concurrent = aio.concurrent(["CORO"]) patched = patches( ("concurrent.cancel_tasks", dict(new_callable=AsyncMock)), ("concurrent.close", dict(new_callable=AsyncMock)), ("concurrent.close_coros", dict(new_callable=AsyncMock)), ("concurrent.sem", dict(new_callable=PropertyMock)), prefix="tools.base.aio") waiter = MagicMock() class SubmitTask: def __init__(self): self.cancel = MagicMock() def __await__(self): waiter() yield concurrent.submit_task = SubmitTask() with patched as (m_cancel, m_close, m_coros, m_sem): assert not await concurrent.cancel() assert (list(m_close.call_args) == [(), {}]) assert (list(m_sem.return_value.release.call_args) == [(), {}]) assert (list(m_cancel.call_args) == [(), {}]) assert (list(m_coros.call_args) == [(), {}]) assert (list(waiter.call_args) == [(), {}])
async def test_aio_concurrent_cancel_tasks(patches, bad): concurrent = aio.concurrent(["CORO"]) patched = patches( ("concurrent.running_tasks", dict(new_callable=PropertyMock)), prefix="tools.base.aio") tasks = [] waiter = MagicMock() class Task: def __init__(self, i): self.i = i self.cancel = MagicMock() def __await__(self): waiter() if self.i == bad: raise BaseException("AN ERROR OCCURRED") for i in range(0, 7): tasks.append(Task(i)) with patched as (m_running, ): m_running.return_value = tasks assert not await concurrent.cancel_tasks() assert (list(list(c) for c in waiter.call_args_list) == [[(), {}]] * 7) for task in tasks: assert (list(task.cancel.call_args) == [(), {}])
def test_aio_concurrent_validate_coro(patches, awaitable, state): concurrent = aio.concurrent(["CORO"]) patched = patches("inspect.getcoroutinestate", prefix="tools.base.aio") # we cant patch inspect.isawaitable without fooing unittest def unawaitable(): pass async def coro(): pass awaits = (coro() if awaitable else unawaitable) with patched as (m_state, ): m_state.return_value = state if awaitable and state == inspect.CORO_CREATED: assert not concurrent.validate_coro(awaits) else: with pytest.raises(aio.ConcurrentError) as e: concurrent.validate_coro(awaits) if not awaitable: assert ( e.value.args[0] == f'Provided input was not a coroutine: {awaits}') assert not m_state.called return awaits.close() assert (list(m_state.call_args) == [(awaits, ), {}]) if state != inspect.CORO_CREATED: assert (e.value.args[0] == f'Provided coroutine has already been fired: {awaits}')
async def test_aio_concurrent_close_coros(patches, consumes_generator, bad): concurrent = aio.concurrent(["CORO"]) patched = patches( "concurrent.close", ("concurrent.iter_coros", dict(new_callable=PropertyMock)), ("concurrent.consumes_generator", dict(new_callable=PropertyMock)), prefix="tools.base.aio") coros = [] for i in range(0, 7): coro = MagicMock() if i == bad: coro.close.side_effect = BaseException("AN ERROR OCCURRED") coros.append(coro) async def iter_coros(): for coro in coros: yield coro with patched as (m_close, m_iter, m_isgen): m_isgen.return_value = consumes_generator m_iter.return_value = iter_coros assert not await concurrent.close_coros() if consumes_generator: assert not m_iter.called return assert (list(m_iter.call_args) == [(), {}]) for coro in coros: assert (list(coro.close.call_args) == [(), {}])
async def check_yapf(self) -> None: """Run flake8 on files and/or repo""" futures = aio.concurrent( self.yapf_format(python_file) for python_file in self.yapf_files) async for (python_file, (reformatted, encoding, changed)) in futures: self.yapf_result(python_file, reformatted, changed)
async def test_aio_concurrent_ready(patches, closed_before, closed_after, nolimit): concurrent = aio.concurrent(["CORO"]) patched = patches(("concurrent.closed", dict(new_callable=PropertyMock)), ("concurrent.nolimit", dict(new_callable=PropertyMock)), ("concurrent.sem", dict(new_callable=PropertyMock)), prefix="tools.base.aio") class DummyCloser: order_mock = MagicMock() close_calls = 0 async def _acquire(self): self.order_mock("ACQUIRE") def _nolimit(self): self.order_mock("NOLIMIT") return nolimit def _closed(self): self.order_mock("CLOSED") self.close_calls += 1 if self.close_calls == 1: return closed_before if self.close_calls == 2: return closed_after closer = DummyCloser() with patched as (m_closed, m_nolimit, m_sem): m_nolimit.side_effect = closer._nolimit m_closed.side_effect = closer._closed m_sem.return_value.acquire = closer._acquire assert (await concurrent.ready() == ((not closed_before and not closed_after) if not nolimit else not closed_before)) if closed_before: assert not m_nolimit.called assert not m_sem.called assert (list(list(c) for c in closer.order_mock.call_args_list) == [[ ('CLOSED', ), {} ]]) return if nolimit: assert not m_sem.called assert (list(list(c) for c in closer.order_mock.call_args_list) == [[ ('CLOSED', ), {} ], [('NOLIMIT', ), {}]]) return assert (list(list(c) for c in closer.order_mock.call_args_list) == [[('CLOSED', ), {}], [('NOLIMIT', ), {}], [('ACQUIRE', ), {}], [('CLOSED', ), {}]])
def test_aio_concurrent_submission_lock(patches): concurrent = aio.concurrent(["CORO"]) patched = patches("asyncio", prefix="tools.base.aio") with patched as (m_asyncio, ): assert concurrent.submission_lock == m_asyncio.Lock.return_value assert (list(m_asyncio.Lock.call_args) == [(), {}]) assert "submission_lock" in concurrent.__dict__
def test_aio_concurrent_out(patches): concurrent = aio.concurrent(["CORO"]) patched = patches("asyncio", prefix="tools.base.aio") with patched as (m_asyncio, ): assert concurrent.out == m_asyncio.Queue.return_value assert (list(m_asyncio.Queue.call_args) == [(), {}]) assert "out" in concurrent.__dict__
def test_aio_concurrent_remember_task(): concurrent = aio.concurrent(["CORO"]) concurrent._running = MagicMock() task = MagicMock() assert not concurrent.remember_task(task) assert (list(concurrent._running.append.call_args) == [(task, ), {}]) assert (list(task.add_done_callback.call_args) == [ (concurrent.forget_task, ), {} ])
def test_aio_concurrent_should_error(result, yield_exceptions): concurrent = aio.concurrent(["CORO"]) concurrent.yield_exceptions = yield_exceptions if isinstance(result, type) and issubclass(result, BaseException): result = result() assert (concurrent.should_error(result) == ( (isinstance(result, aio.ConcurrentIteratorError) or isinstance(result, aio.ConcurrentError) and not yield_exceptions)))
def test_aio_concurrent_nolimit(patches, limit): concurrent = aio.concurrent(["CORO"]) patched = patches(("concurrent.limit", dict(new_callable=PropertyMock)), prefix="tools.base.aio") with patched as (m_limit, ): m_limit.return_value = limit assert concurrent.nolimit == (limit == -1) assert "nolimit" in concurrent.__dict__
def test_aio_concurrent_consumes_generator(patches): concurrent = aio.concurrent(["CORO"]) patched = patches("isinstance", prefix="tools.base.aio") with patched as (m_inst, ): assert concurrent.consumes_generator == m_inst.return_value assert (list(m_inst.call_args) == [(["CORO"], (types.AsyncGeneratorType, types.GeneratorType)), {}]) assert "consumes_generator" in concurrent.__dict__
def test_aio_concurrent_default_limit(patches, cpus): concurrent = aio.concurrent(["CORO"]) patched = patches("min", "os", prefix="tools.base.aio") with patched as (m_min, m_os): m_os.cpu_count.return_value = cpus assert concurrent.default_limit == m_min.return_value assert (list(m_min.call_args) == [(32, (cpus or 0) + 4), {}]) assert "default_limit" not in concurrent.__dict__
def test_aio_concurrent_submitting(patches, locked): concurrent = aio.concurrent(["CORO"]) patched = patches( ("concurrent.submission_lock", dict(new_callable=PropertyMock)), prefix="tools.base.aio") with patched as (m_submission_lock, ): m_submission_lock.return_value.locked.return_value = locked assert concurrent.submitting == locked assert "submitting" not in concurrent.__dict__
def test_aio_concurrent_running(patches, empty): concurrent = aio.concurrent(["CORO"]) patched = patches( ("concurrent.running_queue", dict(new_callable=PropertyMock)), prefix="tools.base.aio") with patched as (m_running_queue, ): m_running_queue.return_value.empty.return_value = empty assert concurrent.running == (not empty) assert "running" not in concurrent.__dict__
def test_aio_concurrent_closed(patches, locked): concurrent = aio.concurrent(["CORO"]) patched = patches( ("concurrent.closing_lock", dict(new_callable=PropertyMock)), prefix="tools.base.aio") with patched as (m_closing_lock, ): m_closing_lock.return_value.locked.return_value = locked assert concurrent.closed == locked assert "closed" not in concurrent.__dict__
def test_aio_concurrent_sem(patches): concurrent = aio.concurrent(["CORO"]) patched = patches("asyncio", ("concurrent.limit", dict(new_callable=PropertyMock)), prefix="tools.base.aio") with patched as (m_asyncio, m_limit): assert concurrent.sem == m_asyncio.Semaphore.return_value assert (list(m_asyncio.Semaphore.call_args) == [(m_limit.return_value, ), {}]) assert "sem" in concurrent.__dict__
def test_aio_concurrent_active(patches, running, submitting): concurrent = aio.concurrent(["CORO"]) patched = patches( "asyncio", ("concurrent.submitting", dict(new_callable=PropertyMock)), ("concurrent.running", dict(new_callable=PropertyMock)), prefix="tools.base.aio") with patched as (m_asyncio, m_submit, m_run): m_submit.return_value = submitting m_run.return_value = running assert concurrent.active == (submitting or running) assert "active" not in concurrent.__dict__
def test_aio_concurrent_forget_task(patches, closed): concurrent = aio.concurrent(["CORO"]) patched = patches(("concurrent.closed", dict(new_callable=PropertyMock)), prefix="tools.base.aio") concurrent._running = MagicMock() with patched as (m_closed, ): m_closed.return_value = closed assert not concurrent.forget_task("TASK") if closed: assert not concurrent._running.remove.called return assert (list(concurrent._running.remove.call_args) == [("TASK", ), {}])
def test_aio_concurrent_limit(patches, limit): concurrent = aio.concurrent(["CORO"]) patched = patches( ("concurrent.default_limit", dict(new_callable=PropertyMock)), prefix="tools.base.aio") concurrent._limit = limit with patched as (m_limit, ): assert concurrent.limit == (limit or m_limit.return_value) if limit: assert not m_limit.called assert "limit" in concurrent.__dict__
def test_aio_concurrent_dunder_aiter(patches): concurrent = aio.concurrent(["CORO"]) patched = patches("asyncio", "concurrent.output", ("concurrent.submit", dict(new_callable=MagicMock)), prefix="tools.base.aio") with patched as (m_asyncio, m_output, m_submit): assert concurrent.__aiter__() == m_output.return_value assert concurrent.submit_task == m_asyncio.create_task.return_value assert (list(m_submit.call_args) == [(), {}]) assert (list(m_asyncio.create_task.call_args) == [ (m_submit.return_value, ), {} ])
async def test_aio_concurrent_close(patches, closed): concurrent = aio.concurrent(["CORO"]) patched = patches( ("concurrent.closed", dict(new_callable=PropertyMock)), ("concurrent.closing_lock", dict(new_callable=PropertyMock)), prefix="tools.base.aio") with patched as (m_closed, m_lock): m_closed.return_value = closed m_lock.return_value.acquire = AsyncMock() assert not await concurrent.close() if closed: assert not m_lock.called else: assert (list(m_lock.return_value.acquire.call_args) == [(), {}])
async def test_aio_concurrent_exit_on_completion(patches, active, closed): concurrent = aio.concurrent(["CORO"]) patched = patches(("concurrent.active", dict(new_callable=PropertyMock)), ("concurrent.closed", dict(new_callable=PropertyMock)), ("concurrent.out", dict(new_callable=PropertyMock)), prefix="tools.base.aio") with patched as (m_active, m_closed, m_out): m_out.return_value.put = AsyncMock() m_active.return_value = active m_closed.return_value = closed assert not await concurrent.exit_on_completion() if closed or active: assert not m_out.called return assert (list(m_out.return_value.put.call_args) == [(aio._sentinel, ), {}])
def test_aio_concurrent_constructor(limit, yield_exceptions): kwargs = {} if limit == "XX": limit = None else: kwargs["limit"] = limit if yield_exceptions is not None: kwargs["yield_exceptions"] = yield_exceptions concurrent = aio.concurrent(["CORO"], **kwargs) assert concurrent._coros == ["CORO"] assert concurrent._limit == limit assert (concurrent.yield_exceptions == (False if yield_exceptions is None else yield_exceptions)) assert concurrent._running == [] assert concurrent.running_tasks is concurrent._running assert "running_tasks" in concurrent.__dict__
async def test_aio_concurrent_coros(patches, raises, close_raises): concurrent = aio.concurrent(["CORO"]) patched = patches( ("concurrent.iter_coros", dict(new_callable=PropertyMock)), prefix="tools.base.aio") results = [] return_coros = [f"CORO{i}" for i in range(0, 3)] m_aclose = AsyncMock() if close_raises: m_aclose.side_effect = close_raises() class Coros: aclose = m_aclose def __call__(self): return self async def __aiter__(self): if raises: raise raises("AN ERROR OCCURRED") for coro in return_coros: yield coro with patched as (m_coros, ): coros = Coros() m_coros.return_value = coros if raises == BaseException: with pytest.raises(BaseException): async for coro in concurrent.coros: pass else: async for coro in concurrent.coros: results.append(coro) if raises == GeneratorExit: assert (list(coros.aclose.call_args) == [(), {}]) return assert not coros.aclose.called assert "coros" not in concurrent.__dict__ if raises: return assert results == return_coros
async def test_aio_concurrent_on_task_complete(patches, closed, nolimit, decrement): concurrent = aio.concurrent(["CORO"]) patched = patches( ("concurrent.exit_on_completion", dict(new_callable=AsyncMock)), ("concurrent.closed", dict(new_callable=PropertyMock)), ("concurrent.out", dict(new_callable=PropertyMock)), ("concurrent.running_queue", dict(new_callable=PropertyMock)), ("concurrent.nolimit", dict(new_callable=PropertyMock)), ("concurrent.sem", dict(new_callable=PropertyMock)), prefix="tools.base.aio") kwargs = {} if decrement is not None: kwargs["decrement"] = decrement with patched as (m_complete, m_closed, m_out, m_running_queue, m_nolimit, m_sem): m_nolimit.return_value = nolimit m_closed.return_value = closed m_out.return_value.put = AsyncMock() assert not await concurrent.on_task_complete("RESULT", **kwargs) if closed: assert not m_complete.called assert not m_nolimit.called assert not m_sem.called assert not m_running_queue.called assert not m_out.return_value.put.called return assert (list(m_out.return_value.put.call_args) == [("RESULT", ), {}]) if nolimit: assert not m_sem.return_value.release.called else: assert (list(m_sem.return_value.release.call_args) == [(), {}]) if decrement or decrement is None: assert (list(m_running_queue.return_value.get_nowait.call_args) == [(), {} ]) else: assert not m_running_queue.return_value.get_nowait.called assert (list(m_complete.call_args) == [(), {}])
async def test_aio_concurrent_create_task(patches): concurrent = aio.concurrent(["CORO"]) patched = patches( "asyncio", "concurrent.remember_task", ("concurrent.task", dict(new_callable=MagicMock)), ("concurrent.running_queue", dict(new_callable=PropertyMock)), prefix="tools.base.aio") with patched as (m_asyncio, m_rem, m_task, m_running_queue): assert not await concurrent.create_task("CORO") assert (list(m_running_queue.return_value.put_nowait.call_args) == [ (None, ), {} ]) assert (list(m_task.call_args) == [("CORO", ), {}]) assert (list(m_asyncio.create_task.call_args) == [(m_task.return_value, ), {}]) assert (list(m_rem.call_args) == [(m_asyncio.create_task.return_value, ), {}])
async def test_aio_concurrent_iter_coros(patches, raises, consumes_async): concurrent = aio.concurrent(["CORO"]) patched = patches( ("concurrent.consumes_async", dict(new_callable=PropertyMock)), prefix="tools.base.aio") coros = [f"CORO{i}" for i in range(0, 7)] exception = BaseException("AN RAISES OCCURRED") def iter_coros(): if raises: raise exception for coro in coros: yield coro async def async_iter_coros(): if raises: raise exception for coro in coros: yield coro concurrent._coros = (async_iter_coros() if consumes_async else iter_coros()) results = [] with patched as (m_async, ): m_async.return_value = consumes_async async for result in concurrent.iter_coros(): results.append(result) if raises: error = results[0] assert isinstance(error, aio.ConcurrentIteratorError) assert error.args[0] is exception assert results == [error] return assert results == coros
async def test_aio_concurrent_task(patches, raises): concurrent = aio.concurrent(["CORO"]) patched = patches("concurrent.on_task_complete", prefix="tools.base.aio") if raises: exception = raises("AN ERROR OCCURRED") async def coro(): if raises: raise exception return 23 with patched as (m_complete, ): assert not await concurrent.task(coro()) result = m_complete.call_args[0][0] if not raises: assert result == 23 else: assert isinstance(result, aio.ConcurrentExecutionError) assert result.args[0] is exception assert (list(m_complete.call_args) == [(result, ), {}])
async def test_aio_concurrent_submit(patches, coros, unready, valid_raises, iter_errors): concurrent = aio.concurrent(["CORO"]) patched = patches( "isinstance", "concurrent.validate_coro", ("concurrent.exit_on_completion", dict(new_callable=AsyncMock)), ("concurrent.create_task", dict(new_callable=AsyncMock)), ("concurrent.on_task_complete", dict(new_callable=AsyncMock)), ("concurrent.ready", dict(new_callable=AsyncMock)), ("concurrent.coros", dict(new_callable=PropertyMock)), ("concurrent.out", dict(new_callable=PropertyMock)), ("concurrent.submission_lock", dict(new_callable=PropertyMock)), prefix="tools.base.aio") m_order = MagicMock() class DummyReady: counter = 0 def ready(self): if self.counter >= unready: self.counter += 1 return False self.counter += 1 return True ready = DummyReady() async def acquire(): m_order("ACQUIRE") def release(): m_order("RELEASE") corolist = [MagicMock() for coro in range(1, coros)] async def iter_coros(): for coro in corolist: m_order(coro) yield coro valid_errors = ((valid_raises == Exception) and coros > 1 and not unready == 0 and not iter_errors) with patched as (m_inst, m_valid, m_exit, m_create, m_complete, m_ready, m_coros, m_out, m_lock): m_out.return_value.put = AsyncMock() m_inst.return_value = iter_errors m_valid.side_effect = valid_raises m_ready.side_effect = ready.ready m_coros.return_value = iter_coros() m_lock.return_value.acquire.side_effect = acquire m_lock.return_value.release.side_effect = release if valid_errors: with pytest.raises(Exception): await concurrent.submit() else: assert not await concurrent.submit() if valid_errors: assert not m_lock.return_value.called assert not m_exit.called else: assert (list(m_lock.return_value.release.call_args) == [(), {}]) assert (list(m_exit.call_args) == [(), {}]) if coros < 2: assert not m_valid.called assert not m_inst.called assert not m_complete.called assert not m_create.called assert not m_ready.called assert not m_out.return_value.put.called return should_close_coro = (not iter_errors and not valid_errors and (len(corolist) > unready)) if should_close_coro: assert corolist[unready].close.called else: assert not any(coro.close.called for coro in corolist) if iter_errors: assert (list(list(c) for c in m_out.return_value.put.call_args_list) == [[ (corolist[0], ), {} ]]) assert (list(list(c) for c in m_inst.call_args_list) == [[ (corolist[0], aio.ConcurrentIteratorError), {} ]]) assert not m_ready.called assert not m_valid.called assert not m_complete.called assert not m_create.called return if valid_errors: assert (list(list(c) for c in m_inst.call_args_list) == [[ (corolist[0], aio.ConcurrentIteratorError), {} ]]) assert (list(list(c) for c in m_ready.call_args_list) == [[(), {}]]) assert (list(list(c) for c in m_valid.call_args_list) == [[(corolist[0], ), {}]]) assert not m_complete.called assert not m_create.called assert (list(list(c) for c in m_order.call_args_list) == ([[('ACQUIRE', ), {}], [(corolist[0], ), {}]])) return assert not m_out.return_value.put.called assert (list(list(c) for c in m_ready.call_args_list) == [[(), {}]] * min(coros - 1, unready + 1 or 1)) assert (list(list(c) for c in m_valid.call_args_list) == [[ (corolist[i - 1], ), {} ] for i in range(1, min(coros, unready + 1))]) assert (list(list(c) for c in m_order.call_args_list) == ( [[('ACQUIRE', ), {}]] + [[(corolist[i - 1], ), {}] for i in range(1, min(coros, unready + 2))] + [[('RELEASE', ), {}]])) if valid_raises: assert len(m_complete.call_args_list) == max(min(coros - 1, unready), 0) for c in m_complete.call_args_list: error = list(c)[0][0] assert isinstance(error, aio.ConcurrentError) assert (list(c) == [(error, ), {'decrement': False}]) assert not m_create.called return assert not m_complete.called assert (list(list(c) for c in m_create.call_args_list) == [[ (corolist[i - 1], ), {} ] for i in range(1, min(coros, unready + 1))])