def test_yield_expression_result(self): @coroutine def yield_expression_coroutine(): for i in range(3): x = yield asyncio.sleep(0, result=i) self.assertEqual(x, i) asyncio.get_event_loop().run_until_complete(yield_expression_coroutine())
def test_returning_coroutine(self): @coroutine def returning_coroutine(loop=None): yield asyncio.sleep(0, loop=loop) coroutine_return('success') loop = asyncio.get_event_loop() self.assertEqual( 'success', asyncio.get_event_loop().run_until_complete( returning_coroutine(loop=loop)))
def testChildWatcher(self): true_binary = find_binary("true") self.assertNotEqual(true_binary, None) initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) try: try: asyncio.set_child_watcher(None) except NotImplementedError: pass else: self.assertTrue(False) args_tuple = ('hello', 'world') loop = asyncio.get_event_loop() future = loop.create_future() def callback(pid, returncode, *args): future.set_result((pid, returncode, args)) with asyncio.get_child_watcher() as watcher: pids = spawn([true_binary], returnpid=True) watcher.add_child_handler(pids[0], callback, *args_tuple) self.assertEqual( loop.run_until_complete(future), (pids[0], os.EX_OK, args_tuple)) finally: asyncio.set_event_loop_policy(initial_policy)
def _do_test(self, read_end, write_end): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = asyncio.get_event_loop() read_end = os.fdopen(read_end, 'rb', 0) write_end = os.fdopen(write_end, 'wb', 0) try: def reader_callback(): if not reader_callback.called.done(): reader_callback.called.set_result(None) reader_callback.called = loop.create_future() loop.add_reader(read_end.fileno(), reader_callback) # Allow the loop to check for IO events, and assert # that our future is still not done. loop.run_until_complete(asyncio.sleep(0, loop=loop)) self.assertFalse(reader_callback.called.done()) # Demonstrate that the callback is called afer the # other end of the pipe has been closed. write_end.close() loop.run_until_complete(reader_callback.called) finally: loop.remove_reader(read_end.fileno()) write_end.close() read_end.close() asyncio.set_event_loop_policy(initial_policy)
async def run(self): self.expected = getattr(self, "expected", None) or {"returncode": 0} if self.debug: fd_pipes = {} pr = None pw = None else: pr, pw = os.pipe() fd_pipes = {1: pw, 2: pw} pr = open(pr, "rb", 0) proc = AsyncFunction( scheduler=asyncio.get_event_loop(), target=self._subprocess, args=(self.args, self.cwd, self.env, self.expected, self.debug), fd_pipes=fd_pipes, ) proc.start() if pw is not None: os.close(pw) await proc.async_wait() if pr is None: stdio = None else: stdio = await _reader(pr) self.result = { "stdio": stdio, "result": proc.result, }
def test_returning_coroutine(self): @coroutine def returning_coroutine(): yield asyncio.sleep(0) coroutine_return('success') self.assertEqual('success', asyncio.get_event_loop().run_until_complete(returning_coroutine()))
def _run_test(self, test): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) try: test(asyncio.get_event_loop()) finally: asyncio.set_event_loop_policy(initial_policy)
def test_raising_coroutine(self): class TestException(Exception): pass @coroutine def raising_coroutine(loop=None): yield asyncio.sleep(0, loop=loop) raise TestException('exception') loop = asyncio.get_event_loop() self.assertRaises(TestException, loop.run_until_complete, raising_coroutine(loop=loop))
def test_raising_coroutine(self): class TestException(Exception): pass @coroutine def raising_coroutine(): yield asyncio.sleep(0) raise TestException('exception') self.assertRaises(TestException, asyncio.get_event_loop().run_until_complete, raising_coroutine())
def test_cancelled_coroutine(self): @coroutine def cancelled_coroutine(loop=None): loop = asyncio._wrap_loop(loop) while True: yield loop.create_future() loop = asyncio.get_event_loop() future = cancelled_coroutine(loop=loop) loop.call_soon(future.cancel) self.assertRaises(asyncio.CancelledError, loop.run_until_complete, future)
def check(self, **kwargs): """Perform profile dependant dependency checks @param pkg: Package in which we check (object). @param ebuild: Ebuild which we check (object). @returns: dictionary """ ebuild = kwargs.get("ebuild").get() pkg = kwargs.get("pkg").get() ebuild.unknown_pkgs, ebuild.baddepsyntax = _depend_checks( ebuild, pkg, self.portdb, self.qatracker, self.repo_metadata, self.repo_settings.qadata, ) relevant_profiles = [] for keyword, arch, groups in _gen_arches(ebuild, self.options, self.repo_settings, self.profiles): if arch not in self.profiles: # A missing profile will create an error further down # during the KEYWORDS verification. continue if self.include_arches is not None: if arch not in self.include_arches: continue for prof in self.profiles[arch]: if self.include_profiles is not None: if prof.sub_path not in self.include_profiles: continue relevant_profiles.append((keyword, groups, prof)) relevant_profiles.sort(key=sort_key) ebuild.relevant_profiles = relevant_profiles if self.options.jobs <= 1: for task in self._iter_tasks(None, None, ebuild, pkg): task, results = task for result in results: self._check_result(task, result) loop = asyncio.get_event_loop() loop.run_until_complete(self._async_check(loop, **kwargs)) return False
def testEventLoopInForkTestCase(self): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) try: loop = asyncio.get_event_loop() fork_exitcode = loop.create_future() # Make async_main fork while the loop is running, which would # trigger https://bugs.python.org/issue22087 with asyncio's # default event loop policy. loop.call_soon(async_main, fork_exitcode) assert loop.run_until_complete(fork_exitcode) == os.EX_OK finally: asyncio.set_event_loop_policy(initial_policy)
async def _check_call(self, cmd): """ Run cmd and raise RepoStorageException on failure. @param cmd: command to executre @type cmd: list """ p = SpawnProcess(args=cmd, scheduler=asyncio.get_event_loop(), **self._spawn_kwargs) p.start() if await p.async_wait() != os.EX_OK: raise RepoStorageException('command exited with status {}: {}'.\ format(p.returncode, ' '.join(cmd)))
def reader(input_file, loop=None): """ Asynchronously read a binary input file. @param input_file: binary input file @type input_file: file @param loop: event loop @type loop: EventLoop @return: bytes @rtype: asyncio.Future (or compatible) """ loop = loop or asyncio.get_event_loop() loop = getattr(loop, '_asyncio_wrapper', loop) future = loop.create_future() _Reader(future, input_file, loop) return future
def test_cancelled_future(self): """ When a coroutine raises CancelledError, the coroutine's future is cancelled. """ @coroutine def cancelled_future_coroutine(loop=None): loop = asyncio._wrap_loop(loop) while True: future = loop.create_future() loop.call_soon(future.cancel) yield future loop = asyncio.get_event_loop() future = loop.run_until_complete(asyncio.wait([cancelled_future_coroutine()]))[0].pop() self.assertTrue(future.cancelled())
def test_cancelled_future(self): """ When a coroutine raises CancelledError, the coroutine's future is cancelled. """ @coroutine def cancelled_future_coroutine(loop=None): loop = asyncio._wrap_loop(loop) while True: future = loop.create_future() loop.call_soon(future.cancel) yield future loop = asyncio.get_event_loop() future = loop.run_until_complete( asyncio.wait([cancelled_future_coroutine()]))[0].pop() self.assertTrue(future.cancelled())
def async_main(fork_exitcode, loop=None): loop = loop or asyncio.get_event_loop() # Since python2.7 does not support Process.sentinel, use Pipe to # monitor for process exit. parent_conn, child_conn = multiprocessing.Pipe() def eof_callback(proc): loop.remove_reader(parent_conn.fileno()) parent_conn.close() proc.join() fork_exitcode.set_result(proc.exitcode) proc = multiprocessing.Process(target=fork_main, args=(parent_conn, child_conn)) loop.add_reader(parent_conn.fileno(), eof_callback, proc) proc.start() child_conn.close()
def _do_test(self, read_end, write_end): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = asyncio.get_event_loop() read_end = os.fdopen(read_end, 'rb', 0) write_end = os.fdopen(write_end, 'wb', 0) try: def writer_callback(): if not writer_callback.called.done(): writer_callback.called.set_result(None) writer_callback.called = loop.create_future() _set_nonblocking(write_end.fileno()) loop.add_writer(write_end.fileno(), writer_callback) # Fill up the pipe, so that no writer callbacks should be # received until the state has changed. while True: try: os.write(write_end.fileno(), 512 * b'0') except EnvironmentError as e: if e.errno != errno.EAGAIN: raise break # Allow the loop to check for IO events, and assert # that our future is still not done. loop.run_until_complete(asyncio.sleep(0, loop=loop)) self.assertFalse(writer_callback.called.done()) # Demonstrate that the callback is called afer the # other end of the pipe has been closed. read_end.close() loop.run_until_complete(writer_callback.called) finally: loop.remove_writer(write_end.fileno()) write_end.close() read_end.close() asyncio.set_event_loop_policy(initial_policy)
async def _check_call(self, cmd, privileged=False): """ Run cmd and raise RepoStorageException on failure. @param cmd: command to executre @type cmd: list @param privileged: run with maximum privileges @type privileged: bool """ if privileged: kwargs = dict(fd_pipes=self._spawn_kwargs.get('fd_pipes')) else: kwargs = self._spawn_kwargs p = SpawnProcess(args=cmd, scheduler=asyncio.get_event_loop(), **kwargs) p.start() if await p.async_wait() != os.EX_OK: raise RepoStorageException('command exited with status {}: {}'.\ format(p.returncode, ' '.join(cmd)))
def test_catching_coroutine(self): class TestException(Exception): pass @coroutine def catching_coroutine(loop=None): loop = asyncio._wrap_loop(loop) future = loop.create_future() loop.call_soon(future.set_exception, TestException('exception')) try: yield future except TestException: self.assertTrue(True) else: self.assertTrue(False) coroutine_return('success') loop = asyncio.get_event_loop() self.assertEqual( 'success', loop.run_until_complete(catching_coroutine(loop=loop)))
def test_add_done_callback(self): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) try: loop = asyncio.get_event_loop() f1 = loop.create_future() f2 = loop.create_future() f1.add_done_callback(f2.set_result) loop.call_soon(lambda: f1.set_result(None)) loop.run_until_complete(f1) self.assertEqual(f1.done(), True) # This proves that done callbacks of f1 are executed before # loop.run_until_complete(f1) returns, which is how asyncio's # default event loop behaves. self.assertEqual(f2.done(), True) finally: asyncio.set_event_loop_policy(initial_policy)
def test_catching_coroutine(self): class TestException(Exception): pass @coroutine def catching_coroutine(loop=None): loop = asyncio._wrap_loop(loop) future = loop.create_future() loop.call_soon(future.set_exception, TestException('exception')) try: yield future except TestException: self.assertTrue(True) else: self.assertTrue(False) coroutine_return('success') loop = asyncio.get_event_loop() self.assertEqual('success', loop.run_until_complete(catching_coroutine(loop=loop)))
def test_cancelled_coroutine(self): """ Verify that a coroutine can handle (and reraise) asyncio.CancelledError in order to perform any necessary cleanup. Note that the asyncio.CancelledError will only be thrown in the coroutine if there's an opportunity (yield) before the generator raises StopIteration. """ loop = asyncio.get_event_loop() ready_for_exception = loop.create_future() exception_in_coroutine = loop.create_future() @coroutine def cancelled_coroutine(loop=None): loop = asyncio._wrap_loop(loop) while True: task = loop.create_future() try: ready_for_exception.set_result(None) yield task except BaseException as e: # Since python3.8, asyncio.CancelledError inherits # from BaseException. task.done() or task.cancel() exception_in_coroutine.set_exception(e) raise else: exception_in_coroutine.set_result(None) future = cancelled_coroutine(loop=loop) loop.run_until_complete(ready_for_exception) future.cancel() self.assertRaises(asyncio.CancelledError, loop.run_until_complete, future) self.assertRaises(asyncio.CancelledError, loop.run_until_complete, exception_in_coroutine)
def fork_main(parent_conn, child_conn): parent_conn.close() loop = asyncio.get_event_loop() # This fails with python's default event loop policy, # see https://bugs.python.org/issue22087. loop.run_until_complete(asyncio.sleep(0.1))
async def _test_content_db(self, emirrordist_cmd, env, layouts, content_db, distfiles, settings, portdb): # Simulate distfile digest change for ContentDB. emdisopts = types.SimpleNamespace(content_db=content_db, distfiles=settings["DISTDIR"]) with EmirrordistConfig(emdisopts, portdb, asyncio.get_event_loop()) as emdisconf: # Copy revisions from bar to foo. for revision_key in emdisconf.content_db["filename:{}".format( "bar")]: emdisconf.content_db.add( DistfileName("foo", digests=dict(revision_key))) # Copy revisions from foo to bar. for revision_key in emdisconf.content_db["filename:{}".format( "foo")]: emdisconf.content_db.add( DistfileName("bar", digests=dict(revision_key))) content_db_state = dict(emdisconf.content_db.items()) self.assertEqual(content_db_state, dict(emdisconf.content_db.items())) self.assertEqual( [ k[len("filename:"):] for k in content_db_state if k.startswith("filename:") ], ["bar", "foo"], ) self.assertEqual(content_db_state["filename:foo"], content_db_state["filename:bar"]) self.assertEqual(len(content_db_state["filename:foo"]), 2) for k in distfiles: try: os.unlink(os.path.join(settings["DISTDIR"], k)) except OSError: pass proc = await asyncio.create_subprocess_exec(*emirrordist_cmd, env=env) self.assertEqual(await proc.wait(), 0) for k in distfiles: with open( os.path.join(settings["DISTDIR"], layouts[0].get_path(k)), "rb") as f: self.assertEqual(f.read(), distfiles[k]) with EmirrordistConfig(emdisopts, portdb, asyncio.get_event_loop()) as emdisconf: self.assertEqual(content_db_state, dict(emdisconf.content_db.items())) # Verify that remove works as expected filename = [ filename for filename in distfiles if filename == "foo" ][0] self.assertTrue(bool(filename.digests)) emdisconf.content_db.remove(filename) # foo should still have a content revision corresponding to bar's content. self.assertEqual( [ k[len("filename:"):] for k in emdisconf.content_db if k.startswith("filename:") ], ["bar", "foo"], ) self.assertEqual(len(emdisconf.content_db["filename:foo"]), 1) self.assertEqual( len([ revision_key for revision_key in emdisconf.content_db["filename:foo"] if not filename.digests_equal( DistfileName( "foo", digests=dict(revision_key), )) ]), 1, ) # bar should still have a content revision corresponding to foo's content. self.assertEqual(len(emdisconf.content_db["filename:bar"]), 2) self.assertEqual( len([ revision_key for revision_key in emdisconf.content_db["filename:bar"] if filename.digests_equal( DistfileName( "bar", digests=dict(revision_key), )) ]), 1, ) # remove the foo which refers to bar's content bar = [filename for filename in distfiles if filename == "bar"][0] foo_remaining = DistfileName("foo", digests=bar.digests) emdisconf.content_db.remove(foo_remaining) self.assertEqual( [ k[len("filename:"):] for k in emdisconf.content_db if k.startswith("filename:") ], ["bar"], ) self.assertRaises(KeyError, emdisconf.content_db.__getitem__, "filename:foo") # bar should still have a content revision corresponding to foo's content. self.assertEqual(len(emdisconf.content_db["filename:bar"]), 2)
def test_method_coroutine(self): class Cubby: _empty = object() def __init__(self, loop): self._loop = loop self._value = self._empty self._waiters = [] def _notify(self): waiters = self._waiters self._waiters = [] for waiter in waiters: waiter.cancelled() or waiter.set_result(None) def _wait(self): waiter = self._loop.create_future() self._waiters.append(waiter) return waiter @coroutine def read(self, loop=None): while self._value is self._empty: yield self._wait() value = self._value self._value = self._empty self._notify() coroutine_return(value) @coroutine def write(self, value, loop=None): while self._value is not self._empty: yield self._wait() self._value = value self._notify() @coroutine def writer_coroutine(cubby, values, sentinel, loop=None): for value in values: yield cubby.write(value, loop=loop) yield cubby.write(sentinel, loop=loop) @coroutine def reader_coroutine(cubby, sentinel, loop=None): results = [] while True: result = yield cubby.read(loop=loop) if result == sentinel: break results.append(result) coroutine_return(results) loop = asyncio.get_event_loop() cubby = Cubby(loop) values = list(range(3)) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None, loop=loop), loop=loop) reader = asyncio.ensure_future(reader_coroutine(cubby, None, loop=loop), loop=loop) loop.run_until_complete(asyncio.wait([writer, reader], loop=loop)) self.assertEqual(reader.result(), values) # Test decoration of coroutine methods and functions for # synchronous usage, allowing coroutines to smoothly # blend with synchronous code. sync_cubby = _sync_methods(cubby, loop=loop) sync_reader = _sync_decorator(reader_coroutine, loop=loop) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None, loop=loop), loop=loop) self.assertEqual(sync_reader(cubby, None), values) self.assertTrue(writer.done()) for i in range(3): sync_cubby.write(i) self.assertEqual(sync_cubby.read(), i)
def test_method_coroutine(self): class Cubby(object): _empty = object() def __init__(self, loop): self._loop = loop self._value = self._empty self._waiters = [] def _notify(self): waiters = self._waiters self._waiters = [] for waiter in waiters: waiter.cancelled() or waiter.set_result(None) def _wait(self): waiter = self._loop.create_future() self._waiters.append(waiter) return waiter @coroutine def read(self): while self._value is self._empty: yield self._wait() value = self._value self._value = self._empty self._notify() coroutine_return(value) @coroutine def write(self, value): while self._value is not self._empty: yield self._wait() self._value = value self._notify() @coroutine def writer_coroutine(cubby, values, sentinel): for value in values: yield cubby.write(value) yield cubby.write(sentinel) @coroutine def reader_coroutine(cubby, sentinel): results = [] while True: result = yield cubby.read() if result == sentinel: break results.append(result) coroutine_return(results) loop = asyncio.get_event_loop() cubby = Cubby(loop) values = list(range(3)) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop) reader = asyncio.ensure_future(reader_coroutine(cubby, None), loop=loop) loop.run_until_complete(asyncio.wait([writer, reader])) self.assertEqual(reader.result(), values)
def test_method_coroutine(self): class Cubby(object): _empty = object() def __init__(self, loop): self._loop = loop self._value = self._empty self._waiters = [] def _notify(self): waiters = self._waiters self._waiters = [] for waiter in waiters: waiter.cancelled() or waiter.set_result(None) def _wait(self): waiter = self._loop.create_future() self._waiters.append(waiter) return waiter @coroutine def read(self): while self._value is self._empty: yield self._wait() value = self._value self._value = self._empty self._notify() coroutine_return(value) @coroutine def write(self, value): while self._value is not self._empty: yield self._wait() self._value = value self._notify() @coroutine def writer_coroutine(cubby, values, sentinel): for value in values: yield cubby.write(value) yield cubby.write(sentinel) @coroutine def reader_coroutine(cubby, sentinel): results = [] while True: result = yield cubby.read() if result == sentinel: break results.append(result) coroutine_return(results) loop = asyncio.get_event_loop() cubby = Cubby(loop) values = list(range(3)) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop) reader = asyncio.ensure_future(reader_coroutine(cubby, None), loop=loop) loop.run_until_complete(asyncio.wait([writer, reader])) self.assertEqual(reader.result(), values) # Test decoration of coroutine methods and functions for # synchronous usage, allowing coroutines to smoothly # blend with synchronous code. sync_cubby = _sync_methods(cubby, loop=loop) sync_reader = _sync_decorator(reader_coroutine, loop=loop) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop) self.assertEqual(sync_reader(cubby, None), values) self.assertTrue(writer.done()) for i in range(3): sync_cubby.write(i) self.assertEqual(sync_cubby.read(), i)