def test_soname_atom_pickle(self):
loop = asyncio._wrap_loop()
with ForkExecutor(loop=loop) as executor:
result = loop.run_until_complete(
loop.run_in_executor(executor, self._get_all_provides)
)
self.assertEqual(self._ALL_PROVIDES, result)
def fork_main(parent_conn, child_conn):
parent_conn.close()
loop = asyncio._wrap_loop()
# This fails with python's default event loop policy,
# see https://bugs.python.org/issue22087.
loop.run_until_complete(asyncio.sleep(0.1, loop=loop))
loop.close()
def _do_test(self, read_end, write_end): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = asyncio._wrap_loop() read_end = os.fdopen(read_end, 'rb', 0) write_end = os.fdopen(write_end, 'wb', 0) try: def reader_callback(): if not reader_callback.called.done(): reader_callback.called.set_result(None) reader_callback.called = loop.create_future() loop.add_reader(read_end.fileno(), reader_callback) # Allow the loop to check for IO events, and assert # that our future is still not done. loop.run_until_complete(asyncio.sleep(0, loop=loop)) self.assertFalse(reader_callback.called.done()) # Demonstrate that the callback is called afer the # other end of the pipe has been closed. write_end.close() loop.run_until_complete(reader_callback.called) finally: loop.remove_reader(read_end.fileno()) write_end.close() read_end.close() asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def _do_test(self, read_end, write_end):
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
loop = asyncio._wrap_loop()
read_end = os.fdopen(read_end, 'rb', 0)
write_end = os.fdopen(write_end, 'wb', 0)
try:
def reader_callback():
if not reader_callback.called.done():
reader_callback.called.set_result(None)
reader_callback.called = loop.create_future()
loop.add_reader(read_end.fileno(), reader_callback)
# Allow the loop to check for IO events, and assert
# that our future is still not done.
loop.run_until_complete(asyncio.sleep(0, loop=loop))
self.assertFalse(reader_callback.called.done())
# Demonstrate that the callback is called afer the
# other end of the pipe has been closed.
write_end.close()
loop.run_until_complete(reader_callback.called)
finally:
loop.remove_reader(read_end.fileno())
write_end.close()
read_end.close()
asyncio.set_event_loop_policy(initial_policy)
if loop not in (None, global_event_loop()):
loop.close()
self.assertFalse(global_event_loop().is_closed())
def testPipeLogger(self):
loop = asyncio._wrap_loop()
for x in (
1,
2,
5,
6,
7,
8,
2**5,
2**10,
2**12,
2**13,
2**14,
2**17,
2**17 + 1,
):
test_string = x * "a"
output = loop.run_until_complete(
self._testPipeLoggerToPipe(test_string, loop)
)
self.assertEqual(
test_string, output, "x = %s, len(output) = %s" % (x, len(output))
)
def __init__(self, max_workers=None, loop=None):
self._max_workers = max_workers or get_cpu_count()
self._loop = asyncio._wrap_loop(loop)
self._submit_queue = collections.deque()
self._running_tasks = {}
self._shutdown = False
self._shutdown_future = self._loop.create_future()
def iter_completed(futures, max_jobs=None, max_load=None, loop=None): """ This is similar to asyncio.as_completed, but takes an iterator of futures as input, and includes support for max_jobs and max_load parameters. @param futures: iterator of asyncio.Future (or compatible) @type futures: iterator @param max_jobs: max number of futures to process concurrently (default is portage.util.cpuinfo.get_cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default is portage.util.cpuinfo.get_cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop @return: iterator of futures that are done @rtype: iterator """ loop = asyncio._wrap_loop(loop) for future_done_set in async_iter_completed(futures, max_jobs=max_jobs, max_load=max_load, loop=loop): for future in loop.run_until_complete(future_done_set): yield future
def iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
This is similar to asyncio.as_completed, but takes an iterator of
futures as input, and includes support for max_jobs and max_load
parameters.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures that are done
@rtype: iterator
"""
loop = asyncio._wrap_loop(loop)
for future_done_set in async_iter_completed(futures,
max_jobs=max_jobs,
max_load=max_load,
loop=loop):
for future in loop.run_until_complete(future_done_set):
yield future
def __init__(self, max_workers=None, loop=None):
self._max_workers = max_workers or get_cpu_count()
self._loop = asyncio._wrap_loop(loop)
self._submit_queue = collections.deque()
self._running_tasks = {}
self._shutdown = False
self._shutdown_future = self._loop.create_future()
def fork_main(parent_conn, child_conn): parent_conn.close() loop = asyncio._wrap_loop() # This fails with python's default event loop policy, # see https://bugs.python.org/issue22087. loop.run_until_complete(asyncio.sleep(0.1, loop=loop)) loop.close()
def test_func():
loop = asyncio._wrap_loop()
return loop.run_until_complete(
self._test_mod_async(ebuilds,
ebuild_inherited,
eclass_defined_phases,
eclass_depend,
portdb,
loop=loop))
def _test_getpid_fork(self):
"""
Verify that portage.getpid() cache is updated in a forked child process.
"""
loop = asyncio._wrap_loop()
proc = AsyncFunction(scheduler=loop, target=portage.getpid)
proc.start()
proc.wait()
self.assertEqual(proc.pid, proc.result)
def test_getpid_double_fork(self):
"""
Verify that portage.getpid() cache is updated correctly after
two forks.
"""
loop = asyncio._wrap_loop()
proc = AsyncFunction(scheduler=loop, target=self._test_getpid_fork)
proc.start()
self.assertEqual(proc.wait(), 0)
def async_fetch_map(self, mypkg, useflags=None, mytree=None, loop=None):
"""
Asynchronous form of getFetchMap.
@param mypkg: cpv for an ebuild
@type mypkg: String
@param useflags: a collection of enabled USE flags, for evaluation of
conditionals
@type useflags: set, or None to enable all conditionals
@param mytree: The canonical path of the tree in which the ebuild
is located, or None for automatic lookup
@type mypkg: String
@param loop: event loop (defaults to global event loop)
@type loop: EventLoop
@return: A future that results in a dict which maps each file name to
a set of alternative URIs.
@rtype: asyncio.Future (or compatible)
"""
loop = asyncio._wrap_loop(loop)
result = loop.create_future()
def aux_get_done(aux_get_future):
if result.cancelled():
return
if aux_get_future.exception() is not None:
if isinstance(aux_get_future.exception(), PortageKeyError):
# Convert this to an InvalidDependString exception since
# callers already handle it.
result.set_exception(portage.exception.InvalidDependString(
"getFetchMap(): aux_get() error reading "
+ mypkg + "; aborting."))
else:
result.set_exception(future.exception())
return
eapi, myuris = aux_get_future.result()
if not eapi_is_supported(eapi):
# Convert this to an InvalidDependString exception
# since callers already handle it.
result.set_exception(portage.exception.InvalidDependString(
"getFetchMap(): '%s' has unsupported EAPI: '%s'" % \
(mypkg, eapi)))
return
try:
result.set_result(_parse_uri_map(mypkg,
{'EAPI':eapi,'SRC_URI':myuris}, use=useflags))
except Exception as e:
result.set_exception(e)
aux_get_future = self.async_aux_get(
mypkg, ["EAPI", "SRC_URI"], mytree=mytree, loop=loop)
result.add_done_callback(lambda result:
aux_get_future.cancel() if result.cancelled() else None)
aux_get_future.add_done_callback(aux_get_done)
return result
def _do_test(self, read_end, write_end):
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
loop = asyncio._wrap_loop()
read_end = os.fdopen(read_end, 'rb', 0)
write_end = os.fdopen(write_end, 'wb', 0)
try:
def writer_callback():
if not writer_callback.called.done():
writer_callback.called.set_result(None)
writer_callback.called = loop.create_future()
_set_nonblocking(write_end.fileno())
loop.add_writer(write_end.fileno(), writer_callback)
# With pypy we've seen intermittent spurious writer callbacks
# here, so retry until the correct state is achieved.
tries = 10
while tries:
tries -= 1
# Fill up the pipe, so that no writer callbacks should be
# received until the state has changed.
while True:
try:
os.write(write_end.fileno(), 512 * b'0')
except EnvironmentError as e:
if e.errno != errno.EAGAIN:
raise
break
# Allow the loop to check for IO events, and assert
# that our future is still not done.
loop.run_until_complete(asyncio.sleep(0, loop=loop))
if writer_callback.called.done():
writer_callback.called = loop.create_future()
else:
break
self.assertFalse(writer_callback.called.done())
# Demonstrate that the callback is called afer the
# other end of the pipe has been closed.
read_end.close()
loop.run_until_complete(writer_callback.called)
finally:
loop.remove_writer(write_end.fileno())
write_end.close()
read_end.close()
asyncio.set_event_loop_policy(initial_policy)
if loop not in (None, global_event_loop()):
loop.close()
self.assertFalse(global_event_loop().is_closed())
def _do_test(self, read_end, write_end): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = asyncio._wrap_loop() read_end = os.fdopen(read_end, 'rb', 0) write_end = os.fdopen(write_end, 'wb', 0) try: def writer_callback(): if not writer_callback.called.done(): writer_callback.called.set_result(None) writer_callback.called = loop.create_future() _set_nonblocking(write_end.fileno()) loop.add_writer(write_end.fileno(), writer_callback) # With pypy we've seen intermittent spurious writer callbacks # here, so retry until the correct state is achieved. tries = 10 while tries: tries -= 1 # Fill up the pipe, so that no writer callbacks should be # received until the state has changed. while True: try: os.write(write_end.fileno(), 512 * b'0') except EnvironmentError as e: if e.errno != errno.EAGAIN: raise break # Allow the loop to check for IO events, and assert # that our future is still not done. loop.run_until_complete(asyncio.sleep(0, loop=loop)) if writer_callback.called.done(): writer_callback.called = loop.create_future() else: break self.assertFalse(writer_callback.called.done()) # Demonstrate that the callback is called afer the # other end of the pipe has been closed. read_end.close() loop.run_until_complete(writer_callback.called) finally: loop.remove_writer(write_end.fileno()) write_end.close() read_end.close() asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def catching_coroutine(loop=None):
loop = asyncio._wrap_loop(loop)
future = loop.create_future()
loop.call_soon(future.set_exception, TestException('exception'))
try:
yield future
except TestException:
self.assertTrue(True)
else:
self.assertTrue(False)
coroutine_return('success')
def async_fetch_map(self, mypkg, useflags=None, mytree=None, loop=None):
"""
Asynchronous form of getFetchMap.
@param mypkg: cpv for an ebuild
@type mypkg: String
@param useflags: a collection of enabled USE flags, for evaluation of
conditionals
@type useflags: set, or None to enable all conditionals
@param mytree: The canonical path of the tree in which the ebuild
is located, or None for automatic lookup
@type mypkg: String
@param loop: event loop (defaults to global event loop)
@type loop: EventLoop
@return: A future that results in a dict which maps each file name to
a set of alternative URIs.
@rtype: asyncio.Future (or compatible)
"""
loop = asyncio._wrap_loop(loop)
result = loop.create_future()
def aux_get_done(aux_get_future):
if result.cancelled():
return
if aux_get_future.exception() is not None:
if isinstance(aux_get_future.exception(), PortageKeyError):
# Convert this to an InvalidDependString exception since
# callers already handle it.
result.set_exception(portage.exception.InvalidDependString(
"getFetchMap(): aux_get() error reading "
+ mypkg + "; aborting."))
else:
result.set_exception(future.exception())
return
eapi, myuris = aux_get_future.result()
if not eapi_is_supported(eapi):
# Convert this to an InvalidDependString exception
# since callers already handle it.
result.set_exception(portage.exception.InvalidDependString(
"getFetchMap(): '%s' has unsupported EAPI: '%s'" % \
(mypkg, eapi)))
return
result.set_result(_parse_uri_map(mypkg,
{'EAPI':eapi,'SRC_URI':myuris}, use=useflags))
aux_get_future = self.async_aux_get(
mypkg, ["EAPI", "SRC_URI"], mytree=mytree, loop=loop)
result.add_done_callback(lambda result:
aux_get_future.cancel() if result.cancelled() else None)
aux_get_future.add_done_callback(aux_get_done)
return result
def catching_coroutine(loop=None):
loop = asyncio._wrap_loop(loop)
future = loop.create_future()
loop.call_soon(future.set_exception, TestException('exception'))
try:
yield future
except TestException:
self.assertTrue(True)
else:
self.assertTrue(False)
coroutine_return('success')
def _check_call(self, cmd):
"""
Run cmd and raise RepoStorageException on failure.
@param cmd: command to executre
@type cmd: list
"""
p = SpawnProcess(args=cmd, scheduler=asyncio._wrap_loop(), **self._spawn_kwargs)
p.start()
if (yield p.async_wait()) != os.EX_OK:
raise RepoStorageException('command exited with status {}: {}'.\
format(p.returncode, ' '.join(cmd)))
def _retry(loop, try_max, try_timeout, overall_timeout, delay_func, reraise, func, *args, **kwargs): """ Retry coroutine, used to implement retry decorator. @return: func return value @rtype: asyncio.Future (or compatible) """ loop = asyncio._wrap_loop(loop) future = loop.create_future() _Retry(future, loop, try_max, try_timeout, overall_timeout, delay_func, reraise, functools.partial(func, *args, **kwargs)) return future
def _run_test(self, test): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = asyncio._wrap_loop() try: test(loop) finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def _run_test(self, test):
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
loop = asyncio._wrap_loop()
try:
test(loop)
finally:
asyncio.set_event_loop_policy(initial_policy)
if loop not in (None, global_event_loop()):
loop.close()
self.assertFalse(global_event_loop().is_closed())
def _retry(loop, try_max, try_timeout, overall_timeout, delay_func, reraise,
func, *args, **kwargs):
"""
Retry coroutine, used to implement retry decorator.
@return: func return value
@rtype: asyncio.Future (or compatible)
"""
loop = asyncio._wrap_loop(loop)
future = loop.create_future()
_Retry(future, loop, try_max, try_timeout, overall_timeout, delay_func,
reraise, functools.partial(func, *args, **kwargs))
return future
def _check_call(self, cmd):
"""
Run cmd and raise RepoStorageException on failure.
@param cmd: command to executre
@type cmd: list
"""
p = SpawnProcess(args=cmd,
scheduler=asyncio._wrap_loop(),
**self._spawn_kwargs)
p.start()
if (yield p.async_wait()) != os.EX_OK:
raise RepoStorageException('command exited with status {}: {}'.\
format(p.returncode, ' '.join(cmd)))
def test_done_callback_after_exit(self):
"""
Test that callbacks can be registered via the Future
add_done_callback method even after the future is done, and
verify that the callbacks are called.
"""
loop = asyncio._wrap_loop()
future = loop.create_future()
future.set_result(None)
for i in range(3):
event = loop.create_future()
future.add_done_callback(lambda future: event.set_result(None))
loop.run_until_complete(event)
def reader(input_file, loop=None):
"""
Asynchronously read a binary input file.
@param input_file: binary input file
@type input_file: file
@param loop: event loop
@type loop: EventLoop
@return: bytes
@rtype: asyncio.Future (or compatible)
"""
loop = asyncio._wrap_loop(loop)
future = loop.create_future()
_Reader(future, input_file, loop)
return future
def cancelled_coroutine(loop=None):
loop = asyncio._wrap_loop(loop)
while True:
task = loop.create_future()
try:
ready_for_exception.set_result(None)
yield task
except BaseException as e:
# Since python3.8, asyncio.CancelledError inherits
# from BaseException.
task.done() or task.cancel()
exception_in_coroutine.set_exception(e)
raise
else:
exception_in_coroutine.set_result(None)
def async_main(fork_exitcode, loop=None): loop = asyncio._wrap_loop(loop) # Since python2.7 does not support Process.sentinel, use Pipe to # monitor for process exit. parent_conn, child_conn = multiprocessing.Pipe() def eof_callback(proc): loop.remove_reader(parent_conn.fileno()) parent_conn.close() proc.join() fork_exitcode.set_result(proc.exitcode) proc = multiprocessing.Process(target=fork_main, args=(parent_conn, child_conn)) loop.add_reader(parent_conn.fileno(), eof_callback, proc) proc.start() child_conn.close()
def check(self, **kwargs):
'''Perform profile dependant dependency checks
@param pkg: Package in which we check (object).
@param ebuild: Ebuild which we check (object).
@returns: dictionary
'''
ebuild = kwargs.get('ebuild').get()
pkg = kwargs.get('pkg').get()
ebuild.unknown_pkgs, ebuild.baddepsyntax = _depend_checks(
ebuild, pkg, self.portdb, self.qatracker, self.repo_metadata,
self.repo_settings.qadata)
relevant_profiles = []
for keyword, arch, groups in _gen_arches(ebuild, self.options,
self.repo_settings, self.profiles):
if arch not in self.profiles:
# A missing profile will create an error further down
# during the KEYWORDS verification.
continue
if self.include_arches is not None:
if arch not in self.include_arches:
continue
for prof in self.profiles[arch]:
if self.include_profiles is not None:
if prof.sub_path not in self.include_profiles:
continue
relevant_profiles.append((keyword, groups, prof))
relevant_profiles.sort(key=sort_key)
ebuild.relevant_profiles = relevant_profiles
if self.options.jobs <= 1:
for task in self._iter_tasks(None, None, ebuild, pkg):
task, results = task
for result in results:
self._check_result(task, result)
loop = asyncio._wrap_loop()
loop.run_until_complete(self._async_check(loop=loop, **kwargs))
return False
def async_main(fork_exitcode, loop=None):
loop = asyncio._wrap_loop(loop)
# Since python2.7 does not support Process.sentinel, use Pipe to
# monitor for process exit.
parent_conn, child_conn = multiprocessing.Pipe()
def eof_callback(proc):
loop.remove_reader(parent_conn.fileno())
parent_conn.close()
proc.join()
fork_exitcode.set_result(proc.exitcode)
proc = multiprocessing.Process(target=fork_main,
args=(parent_conn, child_conn))
loop.add_reader(parent_conn.fileno(), eof_callback, proc)
proc.start()
child_conn.close()
def testEventLoopInForkTestCase(self):
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
loop = None
try:
loop = asyncio._wrap_loop()
fork_exitcode = loop.create_future()
# Make async_main fork while the loop is running, which would
# trigger https://bugs.python.org/issue22087 with asyncio's
# default event loop policy.
loop.call_soon(async_main, fork_exitcode)
assert loop.run_until_complete(fork_exitcode) == os.EX_OK
finally:
asyncio.set_event_loop_policy(initial_policy)
if loop not in (None, global_event_loop()):
loop.close()
self.assertFalse(global_event_loop().is_closed())
def testEventLoopInForkTestCase(self): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = None try: loop = asyncio._wrap_loop() fork_exitcode = loop.create_future() # Make async_main fork while the loop is running, which would # trigger https://bugs.python.org/issue22087 with asyncio's # default event loop policy. loop.call_soon(async_main, fork_exitcode) assert loop.run_until_complete(fork_exitcode) == os.EX_OK finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def _check_call(self, cmd, privileged=False):
"""
Run cmd and raise RepoStorageException on failure.
@param cmd: command to executre
@type cmd: list
@param privileged: run with maximum privileges
@type privileged: bool
"""
if privileged:
kwargs = dict(fd_pipes=self._spawn_kwargs.get('fd_pipes'))
else:
kwargs = self._spawn_kwargs
p = SpawnProcess(args=cmd, scheduler=asyncio._wrap_loop(), **kwargs)
p.start()
if (yield p.async_wait()) != os.EX_OK:
raise RepoStorageException('command exited with status {}: {}'.\
format(p.returncode, ' '.join(cmd)))
def _check_call(self, cmd, privileged=False):
"""
Run cmd and raise RepoStorageException on failure.
@param cmd: command to executre
@type cmd: list
@param privileged: run with maximum privileges
@type privileged: bool
"""
if privileged:
kwargs = dict(fd_pipes=self._spawn_kwargs.get('fd_pipes'))
else:
kwargs = self._spawn_kwargs
p = SpawnProcess(args=cmd, scheduler=asyncio._wrap_loop(), **kwargs)
p.start()
if (yield p.async_wait()) != os.EX_OK:
raise RepoStorageException('command exited with status {}: {}'.\
format(p.returncode, ' '.join(cmd)))
def test_exit_listener_after_exit(self):
"""
Test that callbacks can be registered via the AsynchronousTask
addExitListener method even after the task is done, and
verify that the callbacks are called.
"""
loop = asyncio._wrap_loop()
task = AsynchronousTask(scheduler=loop)
task.start()
loop.run_until_complete(task.async_wait())
for i in range(3):
event = loop.create_future()
task.addStartListener(lambda task: event.set_result(None))
loop.run_until_complete(event)
event = loop.create_future()
task.addExitListener(lambda task: event.set_result(None))
loop.run_until_complete(event)
def _test_mod(self, auxdbmodule):
ebuilds = {
"cat/A-1": {
"EAPI": "7"
},
"cat/B-1": {
"EAPI": "7"
},
}
playground = ResolverPlayground(
ebuilds=ebuilds,
user_config={
'modules': ('portdbapi.auxdbmodule = %s' % auxdbmodule, )
})
portdb = playground.trees[playground.eroot]["porttree"].dbapi
loop = asyncio._wrap_loop()
loop.run_until_complete(self._test_mod_async(ebuilds, portdb))
def testChildWatcher(self):
true_binary = find_binary("true")
self.assertNotEqual(true_binary, None)
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
loop = None
try:
try:
asyncio.set_child_watcher(None)
except NotImplementedError:
pass
else:
self.assertTrue(False)
args_tuple = ('hello', 'world')
loop = asyncio._wrap_loop()
future = loop.create_future()
def callback(pid, returncode, *args):
future.set_result((pid, returncode, args))
@coroutine
def watch_pid(loop=None):
with asyncio.get_child_watcher() as watcher:
pids = spawn([true_binary], returnpid=True)
watcher.add_child_handler(pids[0], callback, *args_tuple)
self.assertEqual(
(yield future),
(pids[0], os.EX_OK, args_tuple))
loop.run_until_complete(watch_pid(loop=loop))
finally:
asyncio.set_event_loop_policy(initial_policy)
if loop not in (None, global_event_loop()):
loop.close()
self.assertFalse(global_event_loop().is_closed())
def _test_mod(self, auxdbmodule):
ebuilds = {
"cat/A-1": {
"EAPI": "7",
"MISC_CONTENT": "inherit foo",
},
"cat/B-1": {
"EAPI": "7",
"MISC_CONTENT": "inherit foo",
},
}
ebuild_inherited = frozenset(["bar", "foo"])
eclass_defined_phases = "prepare"
eclass_depend = "bar/foo"
eclasses = {
"foo": ("inherit bar", ),
"bar": (
"EXPORT_FUNCTIONS src_prepare",
"DEPEND=\"{}\"".format(eclass_depend),
"bar_src_prepare() { default; }",
),
}
playground = ResolverPlayground(
ebuilds=ebuilds,
eclasses=eclasses,
user_config={
'modules': ('portdbapi.auxdbmodule = %s' % auxdbmodule, )
})
portdb = playground.trees[playground.eroot]["porttree"].dbapi
loop = asyncio._wrap_loop()
loop.run_until_complete(
self._test_mod_async(ebuilds, ebuild_inherited,
eclass_defined_phases, eclass_depend, portdb))
def testChildWatcher(self):
true_binary = find_binary("true")
self.assertNotEqual(true_binary, None)
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
loop = None
try:
try:
asyncio.set_child_watcher(None)
except NotImplementedError:
pass
else:
self.assertTrue(False)
args_tuple = ('hello', 'world')
loop = asyncio._wrap_loop()
future = loop.create_future()
def callback(pid, returncode, *args):
future.set_result((pid, returncode, args))
with asyncio.get_child_watcher() as watcher:
pids = spawn([true_binary], returnpid=True)
watcher.add_child_handler(pids[0], callback, *args_tuple)
self.assertEqual(
loop.run_until_complete(future),
(pids[0], os.EX_OK, args_tuple))
finally:
asyncio.set_event_loop_policy(initial_policy)
if loop not in (None, global_event_loop()):
loop.close()
self.assertFalse(global_event_loop().is_closed())
def test_add_done_callback(self):
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
loop = None
try:
loop = asyncio._wrap_loop()
f1 = loop.create_future()
f2 = loop.create_future()
f1.add_done_callback(f2.set_result)
loop.call_soon(lambda: f1.set_result(None))
loop.run_until_complete(f1)
self.assertEqual(f1.done(), True)
# This proves that done callbacks of f1 are executed before
# loop.run_until_complete(f1) returns, which is how asyncio's
# default event loop behaves.
self.assertEqual(f2.done(), True)
finally:
asyncio.set_event_loop_policy(initial_policy)
if loop not in (None, global_event_loop()):
loop.close()
self.assertFalse(global_event_loop().is_closed())
def test_add_done_callback(self): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) loop = None try: loop = asyncio._wrap_loop() f1 = loop.create_future() f2 = loop.create_future() f1.add_done_callback(f2.set_result) loop.call_soon(lambda: f1.set_result(None)) loop.run_until_complete(f1) self.assertEqual(f1.done(), True) # This proves that done callbacks of f1 are executed before # loop.run_until_complete(f1) returns, which is how asyncio's # default event loop behaves. self.assertEqual(f2.done(), True) finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()): loop.close() self.assertFalse(global_event_loop().is_closed())
def _async_check(self, loop=None, **kwargs):
'''Perform async profile dependant dependency checks
@param arches:
@param pkg: Package in which we check (object).
@param ebuild: Ebuild which we check (object).
@param baddepsyntax: boolean
@param unknown_pkgs: set of tuples (type, atom.unevaluated_atom)
@returns: dictionary
'''
loop = asyncio._wrap_loop(loop)
ebuild = kwargs.get('ebuild').get()
pkg = kwargs.get('pkg').get()
unknown_pkgs = ebuild.unknown_pkgs
baddepsyntax = ebuild.baddepsyntax
# Use max_workers=True to ensure immediate fork, since _iter_tasks
# needs the fork to create a snapshot of current state.
executor = ForkExecutor(max_workers=self.options.jobs)
if self.options.jobs > 1:
for future_done_set in async_iter_completed(self._iter_tasks(loop, executor, ebuild, pkg),
max_jobs=self.options.jobs, max_load=self.options.load_average, loop=loop):
for task in (yield future_done_set):
task, results = task.result()
for result in results:
self._check_result(task, result)
if not baddepsyntax and unknown_pkgs:
type_map = {}
for mytype, atom in unknown_pkgs:
type_map.setdefault(mytype, set()).add(atom)
for mytype, atoms in type_map.items():
self.qatracker.add_error(
"dependency.unknown", "%s: %s: %s"
% (ebuild.relative_path, mytype, ", ".join(sorted(atoms))))
def testSimple(self):
debug = False
skip_reason = self._must_skip()
if skip_reason:
self.portage_skip = skip_reason
self.assertFalse(True, skip_reason)
return
copyright_header = """# Copyright 1999-%s Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
""" % time.gmtime().tm_year
pkg_preinst_references_forbidden_var = """
pkg_preinst() {
echo "This ${A} reference is not allowed. Neither is this $BROOT reference."
}
"""
repo_configs = {
"test_repo": {
"layout.conf": ("update-changelog = true", ),
}
}
profiles = (
("x86", "default/linux/x86/test_profile", "stable"),
("x86", "default/linux/x86/test_dev", "dev"),
("x86", "default/linux/x86/test_exp", "exp"),
)
profile = {
"eapi": ("5", ),
"package.use.stable.mask": ("dev-libs/A flag", )
}
ebuilds = {
"dev-libs/A-0": {
"COPYRIGHT_HEADER": copyright_header,
"DESCRIPTION": "Desc goes here",
"EAPI": "5",
"HOMEPAGE": "https://example.com",
"IUSE": "flag",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"RDEPEND": "flag? ( dev-libs/B[flag] )",
},
"dev-libs/A-1": {
"COPYRIGHT_HEADER": copyright_header,
"DESCRIPTION": "Desc goes here",
"EAPI": "4",
"HOMEPAGE": "https://example.com",
"IUSE": "flag",
"KEYWORDS": "~x86",
"LICENSE": "GPL-2",
"RDEPEND": "flag? ( dev-libs/B[flag] )",
},
"dev-libs/B-1": {
"COPYRIGHT_HEADER": copyright_header,
"DESCRIPTION": "Desc goes here",
"EAPI": "4",
"HOMEPAGE": "https://example.com",
"IUSE": "flag",
"KEYWORDS": "~x86",
"LICENSE": "GPL-2",
},
"dev-libs/C-0": {
"COPYRIGHT_HEADER": copyright_header,
"DESCRIPTION": "Desc goes here",
"EAPI": "7",
"HOMEPAGE": "https://example.com",
"IUSE": "flag",
# must be unstable, since dev-libs/A[flag] is stable masked
"KEYWORDS": "~x86",
"LICENSE": "GPL-2",
"RDEPEND": "flag? ( dev-libs/A[flag] )",
"MISC_CONTENT": pkg_preinst_references_forbidden_var,
},
}
licenses = ["GPL-2"]
arch_list = ["x86"]
metadata_xsd = os.path.join(REPOMAN_BASE_PATH, "cnf/metadata.xsd")
metadata_xml_files = (
(
"dev-libs/A",
{
"flags":
"<flag name='flag' restrict='>=dev-libs/A-0'>Description of how USE='flag' affects this package</flag>",
},
),
(
"dev-libs/B",
{
"flags":
"<flag name='flag'>Description of how USE='flag' affects this package</flag>",
},
),
(
"dev-libs/C",
{
"flags":
"<flag name='flag'>Description of how USE='flag' affects this package</flag>",
},
),
)
use_desc = (("flag",
"Description of how USE='flag' affects packages"), )
playground = ResolverPlayground(ebuilds=ebuilds,
profile=profile,
repo_configs=repo_configs,
debug=debug)
loop = asyncio._wrap_loop()
loop.run_until_complete(
asyncio.ensure_future(
self._async_test_simple(
playground,
metadata_xml_files,
profiles,
profile,
licenses,
arch_list,
use_desc,
metadata_xsd,
copyright_header,
debug,
),
loop=loop,
))
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
An asynchronous version of iter_completed. This yields futures, which
when done, result in a set of input futures that are done. This serves
as a wrapper around portage's internal TaskScheduler class, using
standard asyncio interfaces.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures, which when done, result in a set of
input futures that are done
@rtype: iterator
"""
loop = asyncio._wrap_loop(loop)
max_jobs = max_jobs or get_cpu_count()
max_load = max_load or get_cpu_count()
future_map = {}
def task_generator():
for future in futures:
future_map[id(future)] = future
yield AsyncTaskFuture(future=future)
scheduler = TaskScheduler(task_generator(),
max_jobs=max_jobs,
max_load=max_load,
event_loop=loop)
def done_callback(future_done_set, wait_result):
"""Propagate results from wait_result to future_done_set."""
if future_done_set.cancelled():
return
done, pending = wait_result.result()
for future in done:
del future_map[id(future)]
future_done_set.set_result(done)
def cancel_callback(wait_result, future_done_set):
"""Cancel wait_result if future_done_set has been cancelled."""
if future_done_set.cancelled() and not wait_result.done():
wait_result.cancel()
try:
scheduler.start()
# scheduler should ensure that future_map is non-empty until
# task_generator is exhausted
while future_map:
wait_result = asyncio.ensure_future(asyncio.wait(
list(future_map.values()),
return_when=asyncio.FIRST_COMPLETED,
loop=loop),
loop=loop)
future_done_set = loop.create_future()
future_done_set.add_done_callback(
functools.partial(cancel_callback, wait_result))
wait_result.add_done_callback(
functools.partial(done_callback, future_done_set))
yield future_done_set
finally:
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
An asynchronous version of iter_completed. This yields futures, which
when done, result in a set of input futures that are done. This serves
as a wrapper around portage's internal TaskScheduler class, using
standard asyncio interfaces.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures, which when done, result in a set of
input futures that are done
@rtype: iterator
"""
loop = asyncio._wrap_loop(loop)
max_jobs = max_jobs or get_cpu_count()
max_load = max_load or get_cpu_count()
future_map = {}
def task_generator():
for future in futures:
future_map[id(future)] = future
yield AsyncTaskFuture(future=future)
scheduler = TaskScheduler(
task_generator(),
max_jobs=max_jobs,
max_load=max_load,
event_loop=loop)
def done_callback(future_done_set, wait_result):
"""Propagate results from wait_result to future_done_set."""
if future_done_set.cancelled():
return
done, pending = wait_result.result()
for future in done:
del future_map[id(future)]
future_done_set.set_result(done)
def cancel_callback(wait_result, future_done_set):
"""Cancel wait_result if future_done_set has been cancelled."""
if future_done_set.cancelled() and not wait_result.done():
wait_result.cancel()
try:
scheduler.start()
# scheduler should ensure that future_map is non-empty until
# task_generator is exhausted
while future_map:
wait_result = asyncio.ensure_future(
asyncio.wait(list(future_map.values()),
return_when=asyncio.FIRST_COMPLETED, loop=loop), loop=loop)
future_done_set = loop.create_future()
future_done_set.add_done_callback(
functools.partial(cancel_callback, wait_result))
wait_result.add_done_callback(
functools.partial(done_callback, future_done_set))
yield future_done_set
finally:
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()
def cancelled_coroutine(loop=None): loop = asyncio._wrap_loop(loop) while True: yield loop.create_future()
def cancelled_future_coroutine(loop=None): loop = asyncio._wrap_loop(loop) while True: future = loop.create_future() loop.call_soon(future.cancel) yield future
def iter_gather(futures, max_jobs=None, max_load=None, loop=None): """ This is similar to asyncio.gather, but takes an iterator of futures as input, and includes support for max_jobs and max_load parameters. @param futures: iterator of asyncio.Future (or compatible) @type futures: iterator @param max_jobs: max number of futures to process concurrently (default is portage.util.cpuinfo.get_cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default is portage.util.cpuinfo.get_cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop @return: a Future resulting in a list of done input futures, in the same order that they were yielded from the input iterator @rtype: asyncio.Future (or compatible) """ loop = asyncio._wrap_loop(loop) result = loop.create_future() futures_list = [] def future_generator(): for future in futures: futures_list.append(future) yield future completed_iter = async_iter_completed( future_generator(), max_jobs=max_jobs, max_load=max_load, loop=loop, ) def handle_result(future_done_set): if result.cancelled(): if not future_done_set.cancelled(): # All exceptions must be consumed from future_done_set, in order # to avoid triggering the event loop's exception handler. list(future.exception() for future in future_done_set.result() if not future.cancelled()) return try: handle_result.current_task = next(completed_iter) except StopIteration: result.set_result(futures_list) else: handle_result.current_task.add_done_callback(handle_result) try: handle_result.current_task = next(completed_iter) except StopIteration: handle_result.current_task = None result.set_result(futures_list) else: handle_result.current_task.add_done_callback(handle_result) def cancel_callback(result): if (result.cancelled() and handle_result.current_task is not None and not handle_result.current_task.done()): handle_result.current_task.cancel() result.add_done_callback(cancel_callback) return result
def async_xmatch(self, level, origdep, loop=None):
"""
Asynchronous form of xmatch.
@param level: xmatch level (bestmatch-visible, match-all-cpv-only
match-allmatch-visible, minimum-all, minimum-all-ignore-profile,
or minimum-visible)
@type level: str
@param origdep: dependency to match (may omit category)
@type origdep: portage.dep.Atom or str
@param loop: event loop (defaults to global event loop)
@type loop: EventLoop
@return: match result(s)
@rtype: asyncio.Future (or compatible), which results in a _pkg_str
or list of _pkg_str (depends on level)
"""
mydep = dep_expand(origdep, mydb=self, settings=self.settings)
mykey = mydep.cp
#if no updates are being made to the tree, we can consult our xcache...
cache_key = None
if self.frozen:
cache_key = (mydep, mydep.unevaluated_atom)
try:
coroutine_return(self.xcache[level][cache_key][:])
except KeyError:
pass
loop = asyncio._wrap_loop(loop)
myval = None
mytree = None
if mydep.repo is not None:
mytree = self.treemap.get(mydep.repo)
if mytree is None:
if level.startswith("match-"):
myval = []
else:
myval = ""
if myval is not None:
# Unknown repo, empty result.
pass
elif level == "match-all-cpv-only":
# match *all* packages, only against the cpv, in order
# to bypass unnecessary cache access for things like IUSE
# and SLOT.
if mydep == mykey:
# Share cache with match-all/cp_list when the result is the
# same. Note that this requires that mydep.repo is None and
# thus mytree is also None.
level = "match-all"
myval = self.cp_list(mykey, mytree=mytree)
else:
myval = match_from_list(mydep,
self.cp_list(mykey, mytree=mytree))
elif level in ("bestmatch-visible", "match-all",
"match-visible", "minimum-all", "minimum-all-ignore-profile",
"minimum-visible"):
# Find the minimum matching visible version. This is optimized to
# minimize the number of metadata accesses (improves performance
# especially in cases where metadata needs to be generated).
if mydep == mykey:
mylist = self.cp_list(mykey, mytree=mytree)
else:
mylist = match_from_list(mydep,
self.cp_list(mykey, mytree=mytree))
ignore_profile = level in ("minimum-all-ignore-profile",)
visibility_filter = level not in ("match-all",
"minimum-all", "minimum-all-ignore-profile")
single_match = level not in ("match-all", "match-visible")
myval = []
aux_keys = list(self._aux_cache_keys)
if level == "bestmatch-visible":
iterfunc = reversed
else:
iterfunc = iter
for cpv in iterfunc(mylist):
try:
metadata = dict(zip(aux_keys, (yield self.async_aux_get(cpv,
aux_keys, myrepo=cpv.repo, loop=loop))))
except KeyError:
# ebuild not in this repo, or masked by corruption
continue
try:
pkg_str = _pkg_str(cpv, metadata=metadata,
settings=self.settings, db=self)
except InvalidData:
continue
if visibility_filter and not self._visible(pkg_str, metadata):
continue
if mydep.slot is not None and \
not _match_slot(mydep, pkg_str):
continue
if mydep.unevaluated_atom.use is not None and \
not self._match_use(mydep, pkg_str, metadata,
ignore_profile=ignore_profile):
continue
myval.append(pkg_str)
if single_match:
break
if single_match:
if myval:
myval = myval[0]
else:
myval = ""
else:
raise AssertionError(
"Invalid level argument: '%s'" % level)
if self.frozen:
xcache_this_level = self.xcache.get(level)
if xcache_this_level is not None:
xcache_this_level[cache_key] = myval
if not isinstance(myval, _pkg_str):
myval = myval[:]
coroutine_return(myval)
def _async_manifest_fetchlist(portdb, repo_config, cp, cpv_list=None, max_jobs=None, max_load=None, loop=None): """ Asynchronous form of FetchlistDict, with max_jobs and max_load parameters in order to control async_aux_get concurrency. @param portdb: portdbapi instance @type portdb: portdbapi @param repo_config: repository configuration for a Manifest @type repo_config: RepoConfig @param cp: cp for a Manifest @type cp: str @param cpv_list: list of ebuild cpv values for a Manifest @type cpv_list: list @param max_jobs: max number of futures to process concurrently (default is portage.util.cpuinfo.get_cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default is portage.util.cpuinfo.get_cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop @return: a Future resulting in a Mapping compatible with FetchlistDict @rtype: asyncio.Future (or compatible) """ loop = asyncio._wrap_loop(loop) result = loop.create_future() cpv_list = (portdb.cp_list(cp, mytree=repo_config.location) if cpv_list is None else cpv_list) def gather_done(gather_result): # All exceptions must be consumed from gather_result before this # function returns, in order to avoid triggering the event loop's # exception handler. e = None if not gather_result.cancelled(): for future in gather_result.result(): if (future.done() and not future.cancelled() and future.exception() is not None): e = future.exception() if result.cancelled(): return elif e is None: result.set_result(dict((k, list(v.result())) for k, v in zip(cpv_list, gather_result.result()))) else: result.set_exception(e) gather_result = iter_gather( # Use a generator expression for lazy evaluation, so that iter_gather # controls the number of concurrent async_fetch_map calls. (portdb.async_fetch_map(cpv, mytree=repo_config.location, loop=loop) for cpv in cpv_list), max_jobs=max_jobs, max_load=max_load, loop=loop, ) gather_result.add_done_callback(gather_done) result.add_done_callback(lambda result: gather_result.cancel() if result.cancelled() else None) return result
def async_aux_get(self, mycpv, mylist, mytree=None, myrepo=None, loop=None):
"""
Asynchronous form form of aux_get.
@param mycpv: cpv for an ebuild
@type mycpv: str
@param mylist: list of metadata keys
@type mylist: list
@param mytree: The canonical path of the tree in which the ebuild
is located, or None for automatic lookup
@type mytree: str
@param myrepo: name of the repo in which the ebuild is located,
or None for automatic lookup
@type myrepo: str
@param loop: event loop (defaults to global event loop)
@type loop: EventLoop
@return: list of metadata values
@rtype: asyncio.Future (or compatible)
"""
# Don't default to self._event_loop here, since that creates a
# local event loop for thread safety, and that could easily lead
# to simultaneous instantiation of multiple event loops here.
# Callers of this method certainly want the same event loop to
# be used for all calls.
loop = asyncio._wrap_loop(loop)
future = loop.create_future()
cache_me = False
if myrepo is not None:
mytree = self.treemap.get(myrepo)
if mytree is None:
future.set_exception(PortageKeyError(myrepo))
return future
if mytree is not None and len(self.porttrees) == 1 \
and mytree == self.porttrees[0]:
# mytree matches our only tree, so it's safe to
# ignore mytree and cache the result
mytree = None
myrepo = None
if mytree is None:
cache_me = True
if mytree is None and not self._known_keys.intersection(
mylist).difference(self._aux_cache_keys):
aux_cache = self._aux_cache.get(mycpv)
if aux_cache is not None:
future.set_result([aux_cache.get(x, "") for x in mylist])
return future
cache_me = True
try:
cat, pkg = mycpv.split("/", 1)
except ValueError:
# Missing slash. Can't find ebuild so raise PortageKeyError.
future.set_exception(PortageKeyError(mycpv))
return future
myebuild, mylocation = self.findname2(mycpv, mytree)
if not myebuild:
writemsg("!!! aux_get(): %s\n" % \
_("ebuild not found for '%s'") % mycpv, noiselevel=1)
future.set_exception(PortageKeyError(mycpv))
return future
mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation)
if mydata is not None:
self._aux_get_return(
future, mycpv, mylist, myebuild, ebuild_hash,
mydata, mylocation, cache_me, None)
return future
if myebuild in self._broken_ebuilds:
future.set_exception(PortageKeyError(mycpv))
return future
proc = EbuildMetadataPhase(cpv=mycpv,
ebuild_hash=ebuild_hash, portdb=self,
repo_path=mylocation, scheduler=loop,
settings=self.doebuild_settings)
proc.addExitListener(functools.partial(self._aux_get_return,
future, mycpv, mylist, myebuild, ebuild_hash, mydata, mylocation,
cache_me))
future.add_done_callback(functools.partial(self._aux_get_cancel, proc))
proc.start()
return future
