def test_soname_atom_pickle(self): loop = asyncio._wrap_loop() with ForkExecutor(loop=loop) as executor: result = loop.run_until_complete( loop.run_in_executor(executor, self._get_all_provides) ) self.assertEqual(self._ALL_PROVIDES, result)
def fork_main(parent_conn, child_conn): parent_conn.close() loop = asyncio._wrap_loop() # This fails with python's default event loop policy, # see https://bugs.python.org/issue22087. loop.run_until_complete(asyncio.sleep(0.1, loop=loop)) loop.close()
def _do_test(self, read_end, write_end): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = asyncio._wrap_loop() read_end = os.fdopen(read_end, 'rb', 0) write_end = os.fdopen(write_end, 'wb', 0) try: def reader_callback(): if not reader_callback.called.done(): reader_callback.called.set_result(None) reader_callback.called = loop.create_future() loop.add_reader(read_end.fileno(), reader_callback) # Allow the loop to check for IO events, and assert # that our future is still not done. loop.run_until_complete(asyncio.sleep(0, loop=loop)) self.assertFalse(reader_callback.called.done()) # Demonstrate that the callback is called afer the # other end of the pipe has been closed. write_end.close() loop.run_until_complete(reader_callback.called) finally: loop.remove_reader(read_end.fileno()) write_end.close() read_end.close() asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def _do_test(self, read_end, write_end): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = asyncio._wrap_loop() read_end = os.fdopen(read_end, 'rb', 0) write_end = os.fdopen(write_end, 'wb', 0) try: def reader_callback(): if not reader_callback.called.done(): reader_callback.called.set_result(None) reader_callback.called = loop.create_future() loop.add_reader(read_end.fileno(), reader_callback) # Allow the loop to check for IO events, and assert # that our future is still not done. loop.run_until_complete(asyncio.sleep(0, loop=loop)) self.assertFalse(reader_callback.called.done()) # Demonstrate that the callback is called afer the # other end of the pipe has been closed. write_end.close() loop.run_until_complete(reader_callback.called) finally: loop.remove_reader(read_end.fileno()) write_end.close() read_end.close() asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def testPipeLogger(self): loop = asyncio._wrap_loop() for x in ( 1, 2, 5, 6, 7, 8, 2**5, 2**10, 2**12, 2**13, 2**14, 2**17, 2**17 + 1, ): test_string = x * "a" output = loop.run_until_complete( self._testPipeLoggerToPipe(test_string, loop) ) self.assertEqual( test_string, output, "x = %s, len(output) = %s" % (x, len(output)) )
def __init__(self, max_workers=None, loop=None): self._max_workers = max_workers or get_cpu_count() self._loop = asyncio._wrap_loop(loop) self._submit_queue = collections.deque() self._running_tasks = {} self._shutdown = False self._shutdown_future = self._loop.create_future()
def iter_completed(futures, max_jobs=None, max_load=None, loop=None): """ This is similar to asyncio.as_completed, but takes an iterator of futures as input, and includes support for max_jobs and max_load parameters. @param futures: iterator of asyncio.Future (or compatible) @type futures: iterator @param max_jobs: max number of futures to process concurrently (default is portage.util.cpuinfo.get_cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default is portage.util.cpuinfo.get_cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop @return: iterator of futures that are done @rtype: iterator """ loop = asyncio._wrap_loop(loop) for future_done_set in async_iter_completed(futures, max_jobs=max_jobs, max_load=max_load, loop=loop): for future in loop.run_until_complete(future_done_set): yield future
def iter_completed(futures, max_jobs=None, max_load=None, loop=None): """ This is similar to asyncio.as_completed, but takes an iterator of futures as input, and includes support for max_jobs and max_load parameters. @param futures: iterator of asyncio.Future (or compatible) @type futures: iterator @param max_jobs: max number of futures to process concurrently (default is portage.util.cpuinfo.get_cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default is portage.util.cpuinfo.get_cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop @return: iterator of futures that are done @rtype: iterator """ loop = asyncio._wrap_loop(loop) for future_done_set in async_iter_completed(futures, max_jobs=max_jobs, max_load=max_load, loop=loop): for future in loop.run_until_complete(future_done_set): yield future
def __init__(self, max_workers=None, loop=None): self._max_workers = max_workers or get_cpu_count() self._loop = asyncio._wrap_loop(loop) self._submit_queue = collections.deque() self._running_tasks = {} self._shutdown = False self._shutdown_future = self._loop.create_future()
def fork_main(parent_conn, child_conn): parent_conn.close() loop = asyncio._wrap_loop() # This fails with python's default event loop policy, # see https://bugs.python.org/issue22087. loop.run_until_complete(asyncio.sleep(0.1, loop=loop)) loop.close()
def test_func(): loop = asyncio._wrap_loop() return loop.run_until_complete( self._test_mod_async(ebuilds, ebuild_inherited, eclass_defined_phases, eclass_depend, portdb, loop=loop))
def _test_getpid_fork(self): """ Verify that portage.getpid() cache is updated in a forked child process. """ loop = asyncio._wrap_loop() proc = AsyncFunction(scheduler=loop, target=portage.getpid) proc.start() proc.wait() self.assertEqual(proc.pid, proc.result)
def test_getpid_double_fork(self): """ Verify that portage.getpid() cache is updated correctly after two forks. """ loop = asyncio._wrap_loop() proc = AsyncFunction(scheduler=loop, target=self._test_getpid_fork) proc.start() self.assertEqual(proc.wait(), 0)
def async_fetch_map(self, mypkg, useflags=None, mytree=None, loop=None): """ Asynchronous form of getFetchMap. @param mypkg: cpv for an ebuild @type mypkg: String @param useflags: a collection of enabled USE flags, for evaluation of conditionals @type useflags: set, or None to enable all conditionals @param mytree: The canonical path of the tree in which the ebuild is located, or None for automatic lookup @type mypkg: String @param loop: event loop (defaults to global event loop) @type loop: EventLoop @return: A future that results in a dict which maps each file name to a set of alternative URIs. @rtype: asyncio.Future (or compatible) """ loop = asyncio._wrap_loop(loop) result = loop.create_future() def aux_get_done(aux_get_future): if result.cancelled(): return if aux_get_future.exception() is not None: if isinstance(aux_get_future.exception(), PortageKeyError): # Convert this to an InvalidDependString exception since # callers already handle it. result.set_exception(portage.exception.InvalidDependString( "getFetchMap(): aux_get() error reading " + mypkg + "; aborting.")) else: result.set_exception(future.exception()) return eapi, myuris = aux_get_future.result() if not eapi_is_supported(eapi): # Convert this to an InvalidDependString exception # since callers already handle it. result.set_exception(portage.exception.InvalidDependString( "getFetchMap(): '%s' has unsupported EAPI: '%s'" % \ (mypkg, eapi))) return try: result.set_result(_parse_uri_map(mypkg, {'EAPI':eapi,'SRC_URI':myuris}, use=useflags)) except Exception as e: result.set_exception(e) aux_get_future = self.async_aux_get( mypkg, ["EAPI", "SRC_URI"], mytree=mytree, loop=loop) result.add_done_callback(lambda result: aux_get_future.cancel() if result.cancelled() else None) aux_get_future.add_done_callback(aux_get_done) return result
def _do_test(self, read_end, write_end): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = asyncio._wrap_loop() read_end = os.fdopen(read_end, 'rb', 0) write_end = os.fdopen(write_end, 'wb', 0) try: def writer_callback(): if not writer_callback.called.done(): writer_callback.called.set_result(None) writer_callback.called = loop.create_future() _set_nonblocking(write_end.fileno()) loop.add_writer(write_end.fileno(), writer_callback) # With pypy we've seen intermittent spurious writer callbacks # here, so retry until the correct state is achieved. tries = 10 while tries: tries -= 1 # Fill up the pipe, so that no writer callbacks should be # received until the state has changed. while True: try: os.write(write_end.fileno(), 512 * b'0') except EnvironmentError as e: if e.errno != errno.EAGAIN: raise break # Allow the loop to check for IO events, and assert # that our future is still not done. loop.run_until_complete(asyncio.sleep(0, loop=loop)) if writer_callback.called.done(): writer_callback.called = loop.create_future() else: break self.assertFalse(writer_callback.called.done()) # Demonstrate that the callback is called afer the # other end of the pipe has been closed. read_end.close() loop.run_until_complete(writer_callback.called) finally: loop.remove_writer(write_end.fileno()) write_end.close() read_end.close() asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def _do_test(self, read_end, write_end): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = asyncio._wrap_loop() read_end = os.fdopen(read_end, 'rb', 0) write_end = os.fdopen(write_end, 'wb', 0) try: def writer_callback(): if not writer_callback.called.done(): writer_callback.called.set_result(None) writer_callback.called = loop.create_future() _set_nonblocking(write_end.fileno()) loop.add_writer(write_end.fileno(), writer_callback) # With pypy we've seen intermittent spurious writer callbacks # here, so retry until the correct state is achieved. tries = 10 while tries: tries -= 1 # Fill up the pipe, so that no writer callbacks should be # received until the state has changed. while True: try: os.write(write_end.fileno(), 512 * b'0') except EnvironmentError as e: if e.errno != errno.EAGAIN: raise break # Allow the loop to check for IO events, and assert # that our future is still not done. loop.run_until_complete(asyncio.sleep(0, loop=loop)) if writer_callback.called.done(): writer_callback.called = loop.create_future() else: break self.assertFalse(writer_callback.called.done()) # Demonstrate that the callback is called afer the # other end of the pipe has been closed. read_end.close() loop.run_until_complete(writer_callback.called) finally: loop.remove_writer(write_end.fileno()) write_end.close() read_end.close() asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def catching_coroutine(loop=None): loop = asyncio._wrap_loop(loop) future = loop.create_future() loop.call_soon(future.set_exception, TestException('exception')) try: yield future except TestException: self.assertTrue(True) else: self.assertTrue(False) coroutine_return('success')
def async_fetch_map(self, mypkg, useflags=None, mytree=None, loop=None): """ Asynchronous form of getFetchMap. @param mypkg: cpv for an ebuild @type mypkg: String @param useflags: a collection of enabled USE flags, for evaluation of conditionals @type useflags: set, or None to enable all conditionals @param mytree: The canonical path of the tree in which the ebuild is located, or None for automatic lookup @type mypkg: String @param loop: event loop (defaults to global event loop) @type loop: EventLoop @return: A future that results in a dict which maps each file name to a set of alternative URIs. @rtype: asyncio.Future (or compatible) """ loop = asyncio._wrap_loop(loop) result = loop.create_future() def aux_get_done(aux_get_future): if result.cancelled(): return if aux_get_future.exception() is not None: if isinstance(aux_get_future.exception(), PortageKeyError): # Convert this to an InvalidDependString exception since # callers already handle it. result.set_exception(portage.exception.InvalidDependString( "getFetchMap(): aux_get() error reading " + mypkg + "; aborting.")) else: result.set_exception(future.exception()) return eapi, myuris = aux_get_future.result() if not eapi_is_supported(eapi): # Convert this to an InvalidDependString exception # since callers already handle it. result.set_exception(portage.exception.InvalidDependString( "getFetchMap(): '%s' has unsupported EAPI: '%s'" % \ (mypkg, eapi))) return result.set_result(_parse_uri_map(mypkg, {'EAPI':eapi,'SRC_URI':myuris}, use=useflags)) aux_get_future = self.async_aux_get( mypkg, ["EAPI", "SRC_URI"], mytree=mytree, loop=loop) result.add_done_callback(lambda result: aux_get_future.cancel() if result.cancelled() else None) aux_get_future.add_done_callback(aux_get_done) return result
def catching_coroutine(loop=None): loop = asyncio._wrap_loop(loop) future = loop.create_future() loop.call_soon(future.set_exception, TestException('exception')) try: yield future except TestException: self.assertTrue(True) else: self.assertTrue(False) coroutine_return('success')
def _check_call(self, cmd): """ Run cmd and raise RepoStorageException on failure. @param cmd: command to executre @type cmd: list """ p = SpawnProcess(args=cmd, scheduler=asyncio._wrap_loop(), **self._spawn_kwargs) p.start() if (yield p.async_wait()) != os.EX_OK: raise RepoStorageException('command exited with status {}: {}'.\ format(p.returncode, ' '.join(cmd)))
def _retry(loop, try_max, try_timeout, overall_timeout, delay_func, reraise, func, *args, **kwargs): """ Retry coroutine, used to implement retry decorator. @return: func return value @rtype: asyncio.Future (or compatible) """ loop = asyncio._wrap_loop(loop) future = loop.create_future() _Retry(future, loop, try_max, try_timeout, overall_timeout, delay_func, reraise, functools.partial(func, *args, **kwargs)) return future
def _run_test(self, test): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = asyncio._wrap_loop() try: test(loop) finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def _run_test(self, test): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = asyncio._wrap_loop() try: test(loop) finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def _retry(loop, try_max, try_timeout, overall_timeout, delay_func, reraise, func, *args, **kwargs): """ Retry coroutine, used to implement retry decorator. @return: func return value @rtype: asyncio.Future (or compatible) """ loop = asyncio._wrap_loop(loop) future = loop.create_future() _Retry(future, loop, try_max, try_timeout, overall_timeout, delay_func, reraise, functools.partial(func, *args, **kwargs)) return future
def _check_call(self, cmd): """ Run cmd and raise RepoStorageException on failure. @param cmd: command to executre @type cmd: list """ p = SpawnProcess(args=cmd, scheduler=asyncio._wrap_loop(), **self._spawn_kwargs) p.start() if (yield p.async_wait()) != os.EX_OK: raise RepoStorageException('command exited with status {}: {}'.\ format(p.returncode, ' '.join(cmd)))
def test_done_callback_after_exit(self): """ Test that callbacks can be registered via the Future add_done_callback method even after the future is done, and verify that the callbacks are called. """ loop = asyncio._wrap_loop() future = loop.create_future() future.set_result(None) for i in range(3): event = loop.create_future() future.add_done_callback(lambda future: event.set_result(None)) loop.run_until_complete(event)
def reader(input_file, loop=None): """ Asynchronously read a binary input file. @param input_file: binary input file @type input_file: file @param loop: event loop @type loop: EventLoop @return: bytes @rtype: asyncio.Future (or compatible) """ loop = asyncio._wrap_loop(loop) future = loop.create_future() _Reader(future, input_file, loop) return future
def cancelled_coroutine(loop=None): loop = asyncio._wrap_loop(loop) while True: task = loop.create_future() try: ready_for_exception.set_result(None) yield task except BaseException as e: # Since python3.8, asyncio.CancelledError inherits # from BaseException. task.done() or task.cancel() exception_in_coroutine.set_exception(e) raise else: exception_in_coroutine.set_result(None)
def async_main(fork_exitcode, loop=None): loop = asyncio._wrap_loop(loop) # Since python2.7 does not support Process.sentinel, use Pipe to # monitor for process exit. parent_conn, child_conn = multiprocessing.Pipe() def eof_callback(proc): loop.remove_reader(parent_conn.fileno()) parent_conn.close() proc.join() fork_exitcode.set_result(proc.exitcode) proc = multiprocessing.Process(target=fork_main, args=(parent_conn, child_conn)) loop.add_reader(parent_conn.fileno(), eof_callback, proc) proc.start() child_conn.close()
def check(self, **kwargs): '''Perform profile dependant dependency checks @param pkg: Package in which we check (object). @param ebuild: Ebuild which we check (object). @returns: dictionary ''' ebuild = kwargs.get('ebuild').get() pkg = kwargs.get('pkg').get() ebuild.unknown_pkgs, ebuild.baddepsyntax = _depend_checks( ebuild, pkg, self.portdb, self.qatracker, self.repo_metadata, self.repo_settings.qadata) relevant_profiles = [] for keyword, arch, groups in _gen_arches(ebuild, self.options, self.repo_settings, self.profiles): if arch not in self.profiles: # A missing profile will create an error further down # during the KEYWORDS verification. continue if self.include_arches is not None: if arch not in self.include_arches: continue for prof in self.profiles[arch]: if self.include_profiles is not None: if prof.sub_path not in self.include_profiles: continue relevant_profiles.append((keyword, groups, prof)) relevant_profiles.sort(key=sort_key) ebuild.relevant_profiles = relevant_profiles if self.options.jobs <= 1: for task in self._iter_tasks(None, None, ebuild, pkg): task, results = task for result in results: self._check_result(task, result) loop = asyncio._wrap_loop() loop.run_until_complete(self._async_check(loop=loop, **kwargs)) return False
def async_main(fork_exitcode, loop=None): loop = asyncio._wrap_loop(loop) # Since python2.7 does not support Process.sentinel, use Pipe to # monitor for process exit. parent_conn, child_conn = multiprocessing.Pipe() def eof_callback(proc): loop.remove_reader(parent_conn.fileno()) parent_conn.close() proc.join() fork_exitcode.set_result(proc.exitcode) proc = multiprocessing.Process(target=fork_main, args=(parent_conn, child_conn)) loop.add_reader(parent_conn.fileno(), eof_callback, proc) proc.start() child_conn.close()
def testEventLoopInForkTestCase(self): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = None try: loop = asyncio._wrap_loop() fork_exitcode = loop.create_future() # Make async_main fork while the loop is running, which would # trigger https://bugs.python.org/issue22087 with asyncio's # default event loop policy. loop.call_soon(async_main, fork_exitcode) assert loop.run_until_complete(fork_exitcode) == os.EX_OK finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def testEventLoopInForkTestCase(self): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = None try: loop = asyncio._wrap_loop() fork_exitcode = loop.create_future() # Make async_main fork while the loop is running, which would # trigger https://bugs.python.org/issue22087 with asyncio's # default event loop policy. loop.call_soon(async_main, fork_exitcode) assert loop.run_until_complete(fork_exitcode) == os.EX_OK finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def _check_call(self, cmd, privileged=False): """ Run cmd and raise RepoStorageException on failure. @param cmd: command to executre @type cmd: list @param privileged: run with maximum privileges @type privileged: bool """ if privileged: kwargs = dict(fd_pipes=self._spawn_kwargs.get('fd_pipes')) else: kwargs = self._spawn_kwargs p = SpawnProcess(args=cmd, scheduler=asyncio._wrap_loop(), **kwargs) p.start() if (yield p.async_wait()) != os.EX_OK: raise RepoStorageException('command exited with status {}: {}'.\ format(p.returncode, ' '.join(cmd)))
def _check_call(self, cmd, privileged=False): """ Run cmd and raise RepoStorageException on failure. @param cmd: command to executre @type cmd: list @param privileged: run with maximum privileges @type privileged: bool """ if privileged: kwargs = dict(fd_pipes=self._spawn_kwargs.get('fd_pipes')) else: kwargs = self._spawn_kwargs p = SpawnProcess(args=cmd, scheduler=asyncio._wrap_loop(), **kwargs) p.start() if (yield p.async_wait()) != os.EX_OK: raise RepoStorageException('command exited with status {}: {}'.\ format(p.returncode, ' '.join(cmd)))
def test_exit_listener_after_exit(self): """ Test that callbacks can be registered via the AsynchronousTask addExitListener method even after the task is done, and verify that the callbacks are called. """ loop = asyncio._wrap_loop() task = AsynchronousTask(scheduler=loop) task.start() loop.run_until_complete(task.async_wait()) for i in range(3): event = loop.create_future() task.addStartListener(lambda task: event.set_result(None)) loop.run_until_complete(event) event = loop.create_future() task.addExitListener(lambda task: event.set_result(None)) loop.run_until_complete(event)
def _test_mod(self, auxdbmodule): ebuilds = { "cat/A-1": { "EAPI": "7" }, "cat/B-1": { "EAPI": "7" }, } playground = ResolverPlayground( ebuilds=ebuilds, user_config={ 'modules': ('portdbapi.auxdbmodule = %s' % auxdbmodule, ) }) portdb = playground.trees[playground.eroot]["porttree"].dbapi loop = asyncio._wrap_loop() loop.run_until_complete(self._test_mod_async(ebuilds, portdb))
def testChildWatcher(self): true_binary = find_binary("true") self.assertNotEqual(true_binary, None) initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = None try: try: asyncio.set_child_watcher(None) except NotImplementedError: pass else: self.assertTrue(False) args_tuple = ('hello', 'world') loop = asyncio._wrap_loop() future = loop.create_future() def callback(pid, returncode, *args): future.set_result((pid, returncode, args)) @coroutine def watch_pid(loop=None): with asyncio.get_child_watcher() as watcher: pids = spawn([true_binary], returnpid=True) watcher.add_child_handler(pids[0], callback, *args_tuple) self.assertEqual( (yield future), (pids[0], os.EX_OK, args_tuple)) loop.run_until_complete(watch_pid(loop=loop)) finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def _test_mod(self, auxdbmodule): ebuilds = { "cat/A-1": { "EAPI": "7", "MISC_CONTENT": "inherit foo", }, "cat/B-1": { "EAPI": "7", "MISC_CONTENT": "inherit foo", }, } ebuild_inherited = frozenset(["bar", "foo"]) eclass_defined_phases = "prepare" eclass_depend = "bar/foo" eclasses = { "foo": ("inherit bar", ), "bar": ( "EXPORT_FUNCTIONS src_prepare", "DEPEND=\"{}\"".format(eclass_depend), "bar_src_prepare() { default; }", ), } playground = ResolverPlayground( ebuilds=ebuilds, eclasses=eclasses, user_config={ 'modules': ('portdbapi.auxdbmodule = %s' % auxdbmodule, ) }) portdb = playground.trees[playground.eroot]["porttree"].dbapi loop = asyncio._wrap_loop() loop.run_until_complete( self._test_mod_async(ebuilds, ebuild_inherited, eclass_defined_phases, eclass_depend, portdb))
def testChildWatcher(self): true_binary = find_binary("true") self.assertNotEqual(true_binary, None) initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = None try: try: asyncio.set_child_watcher(None) except NotImplementedError: pass else: self.assertTrue(False) args_tuple = ('hello', 'world') loop = asyncio._wrap_loop() future = loop.create_future() def callback(pid, returncode, *args): future.set_result((pid, returncode, args)) with asyncio.get_child_watcher() as watcher: pids = spawn([true_binary], returnpid=True) watcher.add_child_handler(pids[0], callback, *args_tuple) self.assertEqual( loop.run_until_complete(future), (pids[0], os.EX_OK, args_tuple)) finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def test_add_done_callback(self): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = None try: loop = asyncio._wrap_loop() f1 = loop.create_future() f2 = loop.create_future() f1.add_done_callback(f2.set_result) loop.call_soon(lambda: f1.set_result(None)) loop.run_until_complete(f1) self.assertEqual(f1.done(), True) # This proves that done callbacks of f1 are executed before # loop.run_until_complete(f1) returns, which is how asyncio's # default event loop behaves. self.assertEqual(f2.done(), True) finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def test_add_done_callback(self): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = None try: loop = asyncio._wrap_loop() f1 = loop.create_future() f2 = loop.create_future() f1.add_done_callback(f2.set_result) loop.call_soon(lambda: f1.set_result(None)) loop.run_until_complete(f1) self.assertEqual(f1.done(), True) # This proves that done callbacks of f1 are executed before # loop.run_until_complete(f1) returns, which is how asyncio's # default event loop behaves. self.assertEqual(f2.done(), True) finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def _async_check(self, loop=None, **kwargs): '''Perform async profile dependant dependency checks @param arches: @param pkg: Package in which we check (object). @param ebuild: Ebuild which we check (object). @param baddepsyntax: boolean @param unknown_pkgs: set of tuples (type, atom.unevaluated_atom) @returns: dictionary ''' loop = asyncio._wrap_loop(loop) ebuild = kwargs.get('ebuild').get() pkg = kwargs.get('pkg').get() unknown_pkgs = ebuild.unknown_pkgs baddepsyntax = ebuild.baddepsyntax # Use max_workers=True to ensure immediate fork, since _iter_tasks # needs the fork to create a snapshot of current state. executor = ForkExecutor(max_workers=self.options.jobs) if self.options.jobs > 1: for future_done_set in async_iter_completed(self._iter_tasks(loop, executor, ebuild, pkg), max_jobs=self.options.jobs, max_load=self.options.load_average, loop=loop): for task in (yield future_done_set): task, results = task.result() for result in results: self._check_result(task, result) if not baddepsyntax and unknown_pkgs: type_map = {} for mytype, atom in unknown_pkgs: type_map.setdefault(mytype, set()).add(atom) for mytype, atoms in type_map.items(): self.qatracker.add_error( "dependency.unknown", "%s: %s: %s" % (ebuild.relative_path, mytype, ", ".join(sorted(atoms))))
def testSimple(self): debug = False skip_reason = self._must_skip() if skip_reason: self.portage_skip = skip_reason self.assertFalse(True, skip_reason) return copyright_header = """# Copyright 1999-%s Gentoo Authors # Distributed under the terms of the GNU General Public License v2 """ % time.gmtime().tm_year pkg_preinst_references_forbidden_var = """ pkg_preinst() { echo "This ${A} reference is not allowed. Neither is this $BROOT reference." } """ repo_configs = { "test_repo": { "layout.conf": ("update-changelog = true", ), } } profiles = ( ("x86", "default/linux/x86/test_profile", "stable"), ("x86", "default/linux/x86/test_dev", "dev"), ("x86", "default/linux/x86/test_exp", "exp"), ) profile = { "eapi": ("5", ), "package.use.stable.mask": ("dev-libs/A flag", ) } ebuilds = { "dev-libs/A-0": { "COPYRIGHT_HEADER": copyright_header, "DESCRIPTION": "Desc goes here", "EAPI": "5", "HOMEPAGE": "https://example.com", "IUSE": "flag", "KEYWORDS": "x86", "LICENSE": "GPL-2", "RDEPEND": "flag? ( dev-libs/B[flag] )", }, "dev-libs/A-1": { "COPYRIGHT_HEADER": copyright_header, "DESCRIPTION": "Desc goes here", "EAPI": "4", "HOMEPAGE": "https://example.com", "IUSE": "flag", "KEYWORDS": "~x86", "LICENSE": "GPL-2", "RDEPEND": "flag? ( dev-libs/B[flag] )", }, "dev-libs/B-1": { "COPYRIGHT_HEADER": copyright_header, "DESCRIPTION": "Desc goes here", "EAPI": "4", "HOMEPAGE": "https://example.com", "IUSE": "flag", "KEYWORDS": "~x86", "LICENSE": "GPL-2", }, "dev-libs/C-0": { "COPYRIGHT_HEADER": copyright_header, "DESCRIPTION": "Desc goes here", "EAPI": "7", "HOMEPAGE": "https://example.com", "IUSE": "flag", # must be unstable, since dev-libs/A[flag] is stable masked "KEYWORDS": "~x86", "LICENSE": "GPL-2", "RDEPEND": "flag? ( dev-libs/A[flag] )", "MISC_CONTENT": pkg_preinst_references_forbidden_var, }, } licenses = ["GPL-2"] arch_list = ["x86"] metadata_xsd = os.path.join(REPOMAN_BASE_PATH, "cnf/metadata.xsd") metadata_xml_files = ( ( "dev-libs/A", { "flags": "<flag name='flag' restrict='>=dev-libs/A-0'>Description of how USE='flag' affects this package</flag>", }, ), ( "dev-libs/B", { "flags": "<flag name='flag'>Description of how USE='flag' affects this package</flag>", }, ), ( "dev-libs/C", { "flags": "<flag name='flag'>Description of how USE='flag' affects this package</flag>", }, ), ) use_desc = (("flag", "Description of how USE='flag' affects packages"), ) playground = ResolverPlayground(ebuilds=ebuilds, profile=profile, repo_configs=repo_configs, debug=debug) loop = asyncio._wrap_loop() loop.run_until_complete( asyncio.ensure_future( self._async_test_simple( playground, metadata_xml_files, profiles, profile, licenses, arch_list, use_desc, metadata_xsd, copyright_header, debug, ), loop=loop, ))
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None): """ An asynchronous version of iter_completed. This yields futures, which when done, result in a set of input futures that are done. This serves as a wrapper around portage's internal TaskScheduler class, using standard asyncio interfaces. @param futures: iterator of asyncio.Future (or compatible) @type futures: iterator @param max_jobs: max number of futures to process concurrently (default is portage.util.cpuinfo.get_cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default is portage.util.cpuinfo.get_cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop @return: iterator of futures, which when done, result in a set of input futures that are done @rtype: iterator """ loop = asyncio._wrap_loop(loop) max_jobs = max_jobs or get_cpu_count() max_load = max_load or get_cpu_count() future_map = {} def task_generator(): for future in futures: future_map[id(future)] = future yield AsyncTaskFuture(future=future) scheduler = TaskScheduler(task_generator(), max_jobs=max_jobs, max_load=max_load, event_loop=loop) def done_callback(future_done_set, wait_result): """Propagate results from wait_result to future_done_set.""" if future_done_set.cancelled(): return done, pending = wait_result.result() for future in done: del future_map[id(future)] future_done_set.set_result(done) def cancel_callback(wait_result, future_done_set): """Cancel wait_result if future_done_set has been cancelled.""" if future_done_set.cancelled() and not wait_result.done(): wait_result.cancel() try: scheduler.start() # scheduler should ensure that future_map is non-empty until # task_generator is exhausted while future_map: wait_result = asyncio.ensure_future(asyncio.wait( list(future_map.values()), return_when=asyncio.FIRST_COMPLETED, loop=loop), loop=loop) future_done_set = loop.create_future() future_done_set.add_done_callback( functools.partial(cancel_callback, wait_result)) wait_result.add_done_callback( functools.partial(done_callback, future_done_set)) yield future_done_set finally: # cleanup in case of interruption by SIGINT, etc scheduler.cancel() scheduler.wait()
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None): """ An asynchronous version of iter_completed. This yields futures, which when done, result in a set of input futures that are done. This serves as a wrapper around portage's internal TaskScheduler class, using standard asyncio interfaces. @param futures: iterator of asyncio.Future (or compatible) @type futures: iterator @param max_jobs: max number of futures to process concurrently (default is portage.util.cpuinfo.get_cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default is portage.util.cpuinfo.get_cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop @return: iterator of futures, which when done, result in a set of input futures that are done @rtype: iterator """ loop = asyncio._wrap_loop(loop) max_jobs = max_jobs or get_cpu_count() max_load = max_load or get_cpu_count() future_map = {} def task_generator(): for future in futures: future_map[id(future)] = future yield AsyncTaskFuture(future=future) scheduler = TaskScheduler( task_generator(), max_jobs=max_jobs, max_load=max_load, event_loop=loop) def done_callback(future_done_set, wait_result): """Propagate results from wait_result to future_done_set.""" if future_done_set.cancelled(): return done, pending = wait_result.result() for future in done: del future_map[id(future)] future_done_set.set_result(done) def cancel_callback(wait_result, future_done_set): """Cancel wait_result if future_done_set has been cancelled.""" if future_done_set.cancelled() and not wait_result.done(): wait_result.cancel() try: scheduler.start() # scheduler should ensure that future_map is non-empty until # task_generator is exhausted while future_map: wait_result = asyncio.ensure_future( asyncio.wait(list(future_map.values()), return_when=asyncio.FIRST_COMPLETED, loop=loop), loop=loop) future_done_set = loop.create_future() future_done_set.add_done_callback( functools.partial(cancel_callback, wait_result)) wait_result.add_done_callback( functools.partial(done_callback, future_done_set)) yield future_done_set finally: # cleanup in case of interruption by SIGINT, etc scheduler.cancel() scheduler.wait()
def cancelled_coroutine(loop=None): loop = asyncio._wrap_loop(loop) while True: yield loop.create_future()
def cancelled_future_coroutine(loop=None): loop = asyncio._wrap_loop(loop) while True: future = loop.create_future() loop.call_soon(future.cancel) yield future
def iter_gather(futures, max_jobs=None, max_load=None, loop=None): """ This is similar to asyncio.gather, but takes an iterator of futures as input, and includes support for max_jobs and max_load parameters. @param futures: iterator of asyncio.Future (or compatible) @type futures: iterator @param max_jobs: max number of futures to process concurrently (default is portage.util.cpuinfo.get_cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default is portage.util.cpuinfo.get_cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop @return: a Future resulting in a list of done input futures, in the same order that they were yielded from the input iterator @rtype: asyncio.Future (or compatible) """ loop = asyncio._wrap_loop(loop) result = loop.create_future() futures_list = [] def future_generator(): for future in futures: futures_list.append(future) yield future completed_iter = async_iter_completed( future_generator(), max_jobs=max_jobs, max_load=max_load, loop=loop, ) def handle_result(future_done_set): if result.cancelled(): if not future_done_set.cancelled(): # All exceptions must be consumed from future_done_set, in order # to avoid triggering the event loop's exception handler. list(future.exception() for future in future_done_set.result() if not future.cancelled()) return try: handle_result.current_task = next(completed_iter) except StopIteration: result.set_result(futures_list) else: handle_result.current_task.add_done_callback(handle_result) try: handle_result.current_task = next(completed_iter) except StopIteration: handle_result.current_task = None result.set_result(futures_list) else: handle_result.current_task.add_done_callback(handle_result) def cancel_callback(result): if (result.cancelled() and handle_result.current_task is not None and not handle_result.current_task.done()): handle_result.current_task.cancel() result.add_done_callback(cancel_callback) return result
def async_xmatch(self, level, origdep, loop=None): """ Asynchronous form of xmatch. @param level: xmatch level (bestmatch-visible, match-all-cpv-only match-allmatch-visible, minimum-all, minimum-all-ignore-profile, or minimum-visible) @type level: str @param origdep: dependency to match (may omit category) @type origdep: portage.dep.Atom or str @param loop: event loop (defaults to global event loop) @type loop: EventLoop @return: match result(s) @rtype: asyncio.Future (or compatible), which results in a _pkg_str or list of _pkg_str (depends on level) """ mydep = dep_expand(origdep, mydb=self, settings=self.settings) mykey = mydep.cp #if no updates are being made to the tree, we can consult our xcache... cache_key = None if self.frozen: cache_key = (mydep, mydep.unevaluated_atom) try: coroutine_return(self.xcache[level][cache_key][:]) except KeyError: pass loop = asyncio._wrap_loop(loop) myval = None mytree = None if mydep.repo is not None: mytree = self.treemap.get(mydep.repo) if mytree is None: if level.startswith("match-"): myval = [] else: myval = "" if myval is not None: # Unknown repo, empty result. pass elif level == "match-all-cpv-only": # match *all* packages, only against the cpv, in order # to bypass unnecessary cache access for things like IUSE # and SLOT. if mydep == mykey: # Share cache with match-all/cp_list when the result is the # same. Note that this requires that mydep.repo is None and # thus mytree is also None. level = "match-all" myval = self.cp_list(mykey, mytree=mytree) else: myval = match_from_list(mydep, self.cp_list(mykey, mytree=mytree)) elif level in ("bestmatch-visible", "match-all", "match-visible", "minimum-all", "minimum-all-ignore-profile", "minimum-visible"): # Find the minimum matching visible version. This is optimized to # minimize the number of metadata accesses (improves performance # especially in cases where metadata needs to be generated). if mydep == mykey: mylist = self.cp_list(mykey, mytree=mytree) else: mylist = match_from_list(mydep, self.cp_list(mykey, mytree=mytree)) ignore_profile = level in ("minimum-all-ignore-profile",) visibility_filter = level not in ("match-all", "minimum-all", "minimum-all-ignore-profile") single_match = level not in ("match-all", "match-visible") myval = [] aux_keys = list(self._aux_cache_keys) if level == "bestmatch-visible": iterfunc = reversed else: iterfunc = iter for cpv in iterfunc(mylist): try: metadata = dict(zip(aux_keys, (yield self.async_aux_get(cpv, aux_keys, myrepo=cpv.repo, loop=loop)))) except KeyError: # ebuild not in this repo, or masked by corruption continue try: pkg_str = _pkg_str(cpv, metadata=metadata, settings=self.settings, db=self) except InvalidData: continue if visibility_filter and not self._visible(pkg_str, metadata): continue if mydep.slot is not None and \ not _match_slot(mydep, pkg_str): continue if mydep.unevaluated_atom.use is not None and \ not self._match_use(mydep, pkg_str, metadata, ignore_profile=ignore_profile): continue myval.append(pkg_str) if single_match: break if single_match: if myval: myval = myval[0] else: myval = "" else: raise AssertionError( "Invalid level argument: '%s'" % level) if self.frozen: xcache_this_level = self.xcache.get(level) if xcache_this_level is not None: xcache_this_level[cache_key] = myval if not isinstance(myval, _pkg_str): myval = myval[:] coroutine_return(myval)
def _async_manifest_fetchlist(portdb, repo_config, cp, cpv_list=None, max_jobs=None, max_load=None, loop=None): """ Asynchronous form of FetchlistDict, with max_jobs and max_load parameters in order to control async_aux_get concurrency. @param portdb: portdbapi instance @type portdb: portdbapi @param repo_config: repository configuration for a Manifest @type repo_config: RepoConfig @param cp: cp for a Manifest @type cp: str @param cpv_list: list of ebuild cpv values for a Manifest @type cpv_list: list @param max_jobs: max number of futures to process concurrently (default is portage.util.cpuinfo.get_cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default is portage.util.cpuinfo.get_cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop @return: a Future resulting in a Mapping compatible with FetchlistDict @rtype: asyncio.Future (or compatible) """ loop = asyncio._wrap_loop(loop) result = loop.create_future() cpv_list = (portdb.cp_list(cp, mytree=repo_config.location) if cpv_list is None else cpv_list) def gather_done(gather_result): # All exceptions must be consumed from gather_result before this # function returns, in order to avoid triggering the event loop's # exception handler. e = None if not gather_result.cancelled(): for future in gather_result.result(): if (future.done() and not future.cancelled() and future.exception() is not None): e = future.exception() if result.cancelled(): return elif e is None: result.set_result(dict((k, list(v.result())) for k, v in zip(cpv_list, gather_result.result()))) else: result.set_exception(e) gather_result = iter_gather( # Use a generator expression for lazy evaluation, so that iter_gather # controls the number of concurrent async_fetch_map calls. (portdb.async_fetch_map(cpv, mytree=repo_config.location, loop=loop) for cpv in cpv_list), max_jobs=max_jobs, max_load=max_load, loop=loop, ) gather_result.add_done_callback(gather_done) result.add_done_callback(lambda result: gather_result.cancel() if result.cancelled() else None) return result
def async_aux_get(self, mycpv, mylist, mytree=None, myrepo=None, loop=None): """ Asynchronous form form of aux_get. @param mycpv: cpv for an ebuild @type mycpv: str @param mylist: list of metadata keys @type mylist: list @param mytree: The canonical path of the tree in which the ebuild is located, or None for automatic lookup @type mytree: str @param myrepo: name of the repo in which the ebuild is located, or None for automatic lookup @type myrepo: str @param loop: event loop (defaults to global event loop) @type loop: EventLoop @return: list of metadata values @rtype: asyncio.Future (or compatible) """ # Don't default to self._event_loop here, since that creates a # local event loop for thread safety, and that could easily lead # to simultaneous instantiation of multiple event loops here. # Callers of this method certainly want the same event loop to # be used for all calls. loop = asyncio._wrap_loop(loop) future = loop.create_future() cache_me = False if myrepo is not None: mytree = self.treemap.get(myrepo) if mytree is None: future.set_exception(PortageKeyError(myrepo)) return future if mytree is not None and len(self.porttrees) == 1 \ and mytree == self.porttrees[0]: # mytree matches our only tree, so it's safe to # ignore mytree and cache the result mytree = None myrepo = None if mytree is None: cache_me = True if mytree is None and not self._known_keys.intersection( mylist).difference(self._aux_cache_keys): aux_cache = self._aux_cache.get(mycpv) if aux_cache is not None: future.set_result([aux_cache.get(x, "") for x in mylist]) return future cache_me = True try: cat, pkg = mycpv.split("/", 1) except ValueError: # Missing slash. Can't find ebuild so raise PortageKeyError. future.set_exception(PortageKeyError(mycpv)) return future myebuild, mylocation = self.findname2(mycpv, mytree) if not myebuild: writemsg("!!! aux_get(): %s\n" % \ _("ebuild not found for '%s'") % mycpv, noiselevel=1) future.set_exception(PortageKeyError(mycpv)) return future mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation) if mydata is not None: self._aux_get_return( future, mycpv, mylist, myebuild, ebuild_hash, mydata, mylocation, cache_me, None) return future if myebuild in self._broken_ebuilds: future.set_exception(PortageKeyError(mycpv)) return future proc = EbuildMetadataPhase(cpv=mycpv, ebuild_hash=ebuild_hash, portdb=self, repo_path=mylocation, scheduler=loop, settings=self.doebuild_settings) proc.addExitListener(functools.partial(self._aux_get_return, future, mycpv, mylist, myebuild, ebuild_hash, mydata, mylocation, cache_me)) future.add_done_callback(functools.partial(self._aux_get_cancel, proc)) proc.start() return future