def test(loop):
output = None
try:
proc = loop.run_until_complete(
asyncio.create_subprocess_exec(cat_binary,
stdin=stdin_pr,
stdout=stdout_pw,
stderr=stdout_pw))
# These belong exclusively to the subprocess now.
stdout_pw.close()
stdin_pr.close()
output = asyncio.ensure_future(reader(stdout_pr, loop=loop),
loop=loop)
with ForkExecutor(loop=loop) as executor:
writer = asyncio.ensure_future(loop.run_in_executor(
executor, stdin_pw.write, stdin_data),
loop=loop)
# This belongs exclusively to the writer now.
stdin_pw.close()
loop.run_until_complete(writer)
self.assertEqual(loop.run_until_complete(proc.wait()),
os.EX_OK)
self.assertEqual(loop.run_until_complete(output), stdin_data)
finally:
if output is not None and not output.done():
output.cancel()
for f in files:
f.close()
def _wait_loop(self, timeout=None): loop = self.scheduler tasks = [self.async_wait()] if timeout is not None: tasks.append(asyncio.ensure_future( asyncio.sleep(timeout, loop=loop), loop=loop)) try: loop.run_until_complete(asyncio.ensure_future( asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, loop=loop), loop=loop)) finally: for task in tasks: task.cancel()
def _wait_loop(self, timeout=None): loop = self.scheduler tasks = [self.async_wait()] if timeout is not None: tasks.append(asyncio.ensure_future( asyncio.sleep(timeout, loop=loop), loop=loop)) try: loop.run_until_complete(asyncio.ensure_future( asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, loop=loop), loop=loop)) finally: for task in tasks: task.cancel()
async def _testPipeLoggerToPipe(self, test_string, loop):
"""
Test PipeLogger writing to a pipe connected to a PipeReader.
This verifies that PipeLogger does not deadlock when writing
to a pipe that's drained by a PipeReader running in the same
process (requires non-blocking write).
"""
input_fd, writer_pipe = os.pipe()
_set_nonblocking(writer_pipe)
writer_pipe = os.fdopen(writer_pipe, 'wb', 0)
writer = asyncio.ensure_future(
_writer(writer_pipe, test_string.encode('ascii')))
writer.add_done_callback(lambda writer: writer_pipe.close())
pr, pw = os.pipe()
consumer = PipeLogger(background=True,
input_fd=input_fd,
log_file_path=os.fdopen(pw, 'wb', 0),
scheduler=loop)
consumer.start()
# Before starting the reader, wait here for a moment, in order
# to exercise PipeLogger's handling of EAGAIN during write.
await asyncio.wait([writer], timeout=0.01)
reader = _reader(pr)
await writer
content = await reader
await consumer.async_wait()
self.assertEqual(consumer.returncode, os.EX_OK)
return content.decode('ascii', 'replace')
def test(loop):
output = None
try:
with open(os.devnull, 'rb', 0) as devnull:
proc = loop.run_until_complete(
asyncio.create_subprocess_exec(echo_binary,
*args_tuple,
stdin=devnull,
stdout=stdout_pw,
stderr=stdout_pw))
# This belongs exclusively to the subprocess now.
stdout_pw.close()
output = asyncio.ensure_future(reader(stdout_pr, loop=loop),
loop=loop)
self.assertEqual(loop.run_until_complete(proc.wait()),
os.EX_OK)
self.assertEqual(
tuple(loop.run_until_complete(output).split()), args_tuple)
finally:
if output is not None and not output.done():
output.cancel()
for f in files:
f.close()
def schedule(self):
if self._scheduling:
# Ignore any recursive schedule() calls triggered via
# self._task_exit().
return
self._scheduling = True
try:
while self._task_queue and (self.max_jobs is True or len(
self.running_tasks) < self.max_jobs):
task = self._task_queue.popleft()
cancelled = getattr(task, "cancelled", None)
if not cancelled:
self.running_tasks.add(task)
future = asyncio.ensure_future(self._task_coroutine(task),
loop=task.scheduler)
future.add_done_callback(
lambda future: future.cancelled() or future.result())
# This callback will be invoked as soon as the task
# exits (before the future's done callback is called),
# and this is required in order for bool(self) to have
# an updated value for Scheduler._schedule to base
# assumptions upon. Delayed updates to bool(self) is
# what caused Scheduler to hang as in bug 709746.
task.addExitListener(self._task_exit)
finally:
self._scheduling = False
def _commands_exit(self, task):
if self._default_exit(task) != os.EX_OK:
self._async_wait()
return
if self.phase == "install":
out = io.StringIO()
_post_src_install_soname_symlinks(self.settings, out)
msg = out.getvalue()
if msg:
self.scheduler.output(
msg, log_path=self.settings.get("PORTAGE_LOG_FILE"))
if "qa-unresolved-soname-deps" in self.settings.features:
# This operates on REQUIRES metadata generated by the above function call.
future = asyncio.ensure_future(self._soname_deps_qa(),
loop=self.scheduler)
# If an unexpected exception occurs, then this will raise it.
future.add_done_callback(
lambda future: future.cancelled() or future.result())
self._start_task(AsyncTaskFuture(future=future),
self._default_final_exit)
else:
self._default_final_exit(task)
else:
self._default_final_exit(task)
def _spawn(self, args, fd_pipes=None, **kwargs):
"""
Override SpawnProcess._spawn to fork a subprocess that calls
self._run(). This uses multiprocessing.Process in order to leverage
any pre-fork and post-fork interpreter housekeeping that it provides,
promoting a healthy state for the forked interpreter.
"""
# Since multiprocessing.Process closes sys.__stdin__, create a
# temporary duplicate of fd_pipes[0] so that sys.__stdin__ can
# be restored in the subprocess, in case this is needed for
# things like PROPERTIES=interactive support.
stdin_dup = None
try:
stdin_fd = fd_pipes.get(0)
if stdin_fd is not None and stdin_fd == portage._get_stdin(
).fileno():
stdin_dup = os.dup(stdin_fd)
fcntl.fcntl(stdin_dup, fcntl.F_SETFD,
fcntl.fcntl(stdin_fd, fcntl.F_GETFD))
fd_pipes[0] = stdin_dup
self._proc = multiprocessing.Process(target=self._bootstrap,
args=(fd_pipes, ))
self._proc.start()
finally:
if stdin_dup is not None:
os.close(stdin_dup)
self._proc_join_task = asyncio.ensure_future(self._proc_join(
self._proc, loop=self.scheduler),
loop=self.scheduler)
self._proc_join_task.add_done_callback(
functools.partial(self._proc_join_done, self._proc))
return [self._proc.pid]
def _schedule(self): while (not self._shutdown and self._submit_queue and len(self._running_tasks) < self._max_workers): future, proc = self._submit_queue.popleft() proc.scheduler = self._loop self._running_tasks[id(proc)] = proc future.add_done_callback(functools.partial(self._cancel_cb, proc)) proc_future = asyncio.ensure_future(self._proc_coroutine(proc), loop=self._loop) proc_future.add_done_callback(functools.partial(self._proc_coroutine_done, future, proc))
def _subprocess_transport_callback(self, transp, protocol, result, waiter):
if waiter.exception() is None:
result.set_result((transp, protocol))
else:
transp.close()
wait_transp = asyncio.ensure_future(transp._wait(), loop=self)
wait_transp.add_done_callback(
functools.partial(self._subprocess_transport_failure, result,
waiter.exception()))
def _subprocess_transport_callback(self, transp, protocol, result, waiter): if waiter.exception() is None: result.set_result((transp, protocol)) else: transp.close() wait_transp = asyncio.ensure_future(transp._wait(), loop=self) wait_transp.add_done_callback( functools.partial(self._subprocess_transport_failure, result, waiter.exception()))
def _async_start(self):
pipe_logger = None
filter_proc = None
try:
log_input = None
if self.log_path is not None:
log_filter_file = self.log_filter_file
if log_filter_file is not None:
split_value = shlex_split(log_filter_file)
log_filter_file = split_value if split_value else None
if log_filter_file:
filter_input, stdin = os.pipe()
log_input, filter_output = os.pipe()
try:
filter_proc = yield asyncio.create_subprocess_exec(
*log_filter_file,
env=self.env,
stdin=filter_input,
stdout=filter_output,
stderr=filter_output,
loop=self.scheduler)
except EnvironmentError:
# Maybe the command is missing or broken somehow...
os.close(filter_input)
os.close(stdin)
os.close(log_input)
os.close(filter_output)
else:
self._stdin = os.fdopen(stdin, 'wb', 0)
os.close(filter_input)
os.close(filter_output)
if self._stdin is None:
# Since log_filter_file is unspecified or refers to a file
# that is missing or broken somehow, create a pipe that
# logs directly to pipe_logger.
log_input, stdin = os.pipe()
self._stdin = os.fdopen(stdin, 'wb', 0)
# Set background=True so that pipe_logger does not log to stdout.
pipe_logger = PipeLogger(background=True,
scheduler=self.scheduler,
input_fd=log_input,
log_file_path=self.log_path)
yield pipe_logger.async_start()
except asyncio.CancelledError:
if pipe_logger is not None and pipe_logger.poll() is None:
pipe_logger.cancel()
if filter_proc is not None and filter_proc.returncode is None:
filter_proc.terminate()
raise
self._main_task = asyncio.ensure_future(self._main(
pipe_logger, filter_proc=filter_proc),
loop=self.scheduler)
self._main_task.add_done_callback(self._main_exit)
def execute_wrapper():
kill_switch = parent_loop.create_future()
parent_future = asyncio.ensure_future(parent_loop.run_in_executor(
self._executor, wrapper, kill_switch),
loop=parent_loop)
parent_future.add_done_callback(
lambda parent_future: None
if kill_switch.done() else kill_switch.set_result(None))
return parent_future
def execute_wrapper(): kill_switch = parent_loop.create_future() parent_future = asyncio.ensure_future( parent_loop.run_in_executor(self._executor, wrapper, kill_switch), loop=parent_loop) parent_future.add_done_callback( lambda parent_future: None if kill_switch.done() else kill_switch.set_result(None)) return parent_future
def _start(self):
filter_proc = None
log_input = None
if self.log_path is not None:
log_filter_file = self.log_filter_file
if log_filter_file is not None:
split_value = shlex_split(log_filter_file)
log_filter_file = split_value if split_value else None
if log_filter_file:
filter_input, stdin = os.pipe()
log_input, filter_output = os.pipe()
try:
filter_proc = PopenProcess(
proc=subprocess.Popen(
log_filter_file,
env=self.env,
stdin=filter_input,
stdout=filter_output,
stderr=filter_output,
),
scheduler=self.scheduler,
)
filter_proc.start()
except EnvironmentError:
# Maybe the command is missing or broken somehow...
os.close(filter_input)
os.close(stdin)
os.close(log_input)
os.close(filter_output)
else:
self._stdin = os.fdopen(stdin, "wb", 0)
os.close(filter_input)
os.close(filter_output)
if self._stdin is None:
# Since log_filter_file is unspecified or refers to a file
# that is missing or broken somehow, create a pipe that
# logs directly to pipe_logger.
log_input, stdin = os.pipe()
self._stdin = os.fdopen(stdin, "wb", 0)
# Set background=True so that pipe_logger does not log to stdout.
pipe_logger = PipeLogger(
background=True,
scheduler=self.scheduler,
input_fd=log_input,
log_file_path=self.log_path,
)
pipe_logger.start()
self._main_task_cancel = functools.partial(self._main_cancel,
filter_proc, pipe_logger)
self._main_task = asyncio.ensure_future(self._main(
filter_proc, pipe_logger),
loop=self.scheduler)
self._main_task.add_done_callback(self._main_exit)
def execute_wrapper():
kill_switch = threading.Event()
parent_future = asyncio.ensure_future(parent_loop.run_in_executor(
self._executor, wrapper, kill_switch),
loop=parent_loop)
def kill_callback(parent_future):
if not kill_switch.is_set():
kill_switch.set()
parent_future.add_done_callback(kill_callback)
return parent_future
def _start(self):
log_file_path = self.log_file_path
if hasattr(log_file_path, 'write'):
self._log_file_nb = True
self._log_file = log_file_path
_set_nonblocking(self._log_file.fileno())
elif log_file_path is not None:
try:
self._log_file = open(
_unicode_encode(log_file_path,
encoding=_encodings["fs"],
errors="strict"),
mode="ab",
)
if log_file_path.endswith(".gz"):
self._log_file_real = self._log_file
self._log_file = gzip.GzipFile(filename="",
mode="ab",
fileobj=self._log_file)
portage.util.apply_secpass_permissions(
log_file_path,
uid=portage.portage_uid,
gid=portage.portage_gid,
mode=0o660,
)
except FileNotFoundError:
if self._was_cancelled():
self._async_wait()
return
raise
if isinstance(self.input_fd, int):
self.input_fd = os.fdopen(self.input_fd, 'rb', 0)
fd = self.input_fd.fileno()
fcntl.fcntl(fd, fcntl.F_SETFL,
fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
self._io_loop_task = asyncio.ensure_future(self._io_loop(
self.input_fd),
loop=self.scheduler)
self._io_loop_task.add_done_callback(self._io_loop_done)
self._registered = True
def _start_task(self, task, exit_handler):
"""
Register exit handler for the given task, set it
as self._current_task, and call task.async_start().
Subclasses can use this as a generic way to start
a task.
"""
try:
task.scheduler = self.scheduler
except AttributeError:
pass
task.addExitListener(exit_handler)
self._current_task = task
result = asyncio.ensure_future(task.async_start(), loop=self.scheduler)
result.add_done_callback(self._current_task_start_cb)
def _start(self):
log_file_path = self.log_file_path
if hasattr(log_file_path, 'write'):
self._log_file_nb = True
self._log_file = log_file_path
_set_nonblocking(self._log_file.fileno())
elif log_file_path is not None:
self._log_file = open(_unicode_encode(log_file_path,
encoding=_encodings['fs'], errors='strict'), mode='ab')
if log_file_path.endswith('.gz'):
self._log_file_real = self._log_file
self._log_file = gzip.GzipFile(filename='', mode='ab',
fileobj=self._log_file)
portage.util.apply_secpass_permissions(log_file_path,
uid=portage.portage_uid, gid=portage.portage_gid,
mode=0o660)
if isinstance(self.input_fd, int):
self.input_fd = os.fdopen(self.input_fd, 'rb', 0)
fd = self.input_fd.fileno()
fcntl.fcntl(fd, fcntl.F_SETFL,
fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
# FD_CLOEXEC is enabled by default in Python >=3.4.
if sys.hexversion < 0x3040000:
try:
fcntl.FD_CLOEXEC
except AttributeError:
pass
else:
fcntl.fcntl(fd, fcntl.F_SETFD,
fcntl.fcntl(fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
self._io_loop_task = asyncio.ensure_future(self._io_loop(self.input_fd), loop=self.scheduler)
self._io_loop_task.add_done_callback(self._io_loop_done)
self._registered = True
def _start(self):
if self.fd_pipes is None:
self.fd_pipes = {}
else:
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
master_fd, slave_fd = self._pipe(fd_pipes)
can_log = self._can_log(slave_fd)
if can_log:
log_file_path = self.logfile
else:
log_file_path = None
null_input = None
if not self.background or 0 in fd_pipes:
# Subclasses such as AbstractEbuildProcess may have already passed
# in a null file descriptor in fd_pipes, so use that when given.
pass
else:
# TODO: Use job control functions like tcsetpgrp() to control
# access to stdin. Until then, use /dev/null so that any
# attempts to read from stdin will immediately return EOF
# instead of blocking indefinitely.
null_input = os.open('/dev/null', os.O_RDWR)
fd_pipes[0] = null_input
fd_pipes.setdefault(0, portage._get_stdin().fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stderr__.fileno())
# flush any pending output
stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
for fd in fd_pipes.values():
if fd in stdout_filenos:
sys.__stdout__.flush()
sys.__stderr__.flush()
break
fd_pipes_orig = fd_pipes.copy()
if log_file_path is not None or self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
else:
# Create a dummy pipe that PipeLogger uses to efficiently
# monitor for process exit by listening for the EOF event.
# Re-use of the allocated fd number for the key in fd_pipes
# guarantees that the keys will not collide for similarly
# allocated pipes which are used by callers such as
# FileDigester and MergeProcess. See the _setup_pipes
# docstring for more benefits of this allocation approach.
self._dummy_pipe_fd = slave_fd
fd_pipes[slave_fd] = slave_fd
kwargs = {}
for k in self._spawn_kwarg_names:
v = getattr(self, k)
if v is not None:
kwargs[k] = v
kwargs["fd_pipes"] = fd_pipes
kwargs["returnpid"] = True
kwargs.pop("logfile", None)
retval = self._spawn(self.args, **kwargs)
os.close(slave_fd)
if null_input is not None:
os.close(null_input)
if isinstance(retval, int):
# spawn failed
self.returncode = retval
self._async_wait()
return
self.pid = retval[0]
stdout_fd = None
if can_log and not self.background:
stdout_fd = os.dup(fd_pipes_orig[1])
build_logger = BuildLogger(env=self.env,
log_path=log_file_path,
log_filter_file=self.log_filter_file,
scheduler=self.scheduler)
build_logger.start()
pipe_logger = PipeLogger(background=self.background,
scheduler=self.scheduler,
input_fd=master_fd,
log_file_path=build_logger.stdin,
stdout_fd=stdout_fd)
pipe_logger.start()
self._registered = True
self._main_task = asyncio.ensure_future(self._main(
build_logger, pipe_logger),
loop=self.scheduler)
self._main_task.add_done_callback(self._main_exit)
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
An asynchronous version of iter_completed. This yields futures, which
when done, result in a set of input futures that are done. This serves
as a wrapper around portage's internal TaskScheduler class, using
standard asyncio interfaces.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures, which when done, result in a set of
input futures that are done
@rtype: iterator
"""
loop = asyncio._wrap_loop(loop)
max_jobs = max_jobs or get_cpu_count()
max_load = max_load or get_cpu_count()
future_map = {}
def task_generator():
for future in futures:
future_map[id(future)] = future
yield AsyncTaskFuture(future=future)
scheduler = TaskScheduler(
task_generator(),
max_jobs=max_jobs,
max_load=max_load,
event_loop=loop)
def done_callback(future_done_set, wait_result):
"""Propagate results from wait_result to future_done_set."""
if future_done_set.cancelled():
return
done, pending = wait_result.result()
for future in done:
del future_map[id(future)]
future_done_set.set_result(done)
def cancel_callback(wait_result, future_done_set):
"""Cancel wait_result if future_done_set has been cancelled."""
if future_done_set.cancelled() and not wait_result.done():
wait_result.cancel()
try:
scheduler.start()
# scheduler should ensure that future_map is non-empty until
# task_generator is exhausted
while future_map:
wait_result = asyncio.ensure_future(
asyncio.wait(list(future_map.values()),
return_when=asyncio.FIRST_COMPLETED, loop=loop), loop=loop)
future_done_set = loop.create_future()
future_done_set.add_done_callback(
functools.partial(cancel_callback, wait_result))
wait_result.add_done_callback(
functools.partial(done_callback, future_done_set))
yield future_done_set
finally:
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()
def testSimple(self):
debug = False
install_something = """
S="${WORKDIR}"
pkg_pretend() {
einfo "called pkg_pretend for $CATEGORY/$PF"
}
src_install() {
einfo "installing something..."
insinto /usr/lib/${P}
echo "blah blah blah" > "${T}"/regular-file
doins "${T}"/regular-file
dosym regular-file /usr/lib/${P}/symlink || die
# Test CONFIG_PROTECT
insinto /etc
newins "${T}"/regular-file ${PN}-${SLOT%/*}
# Test code for bug #381629, using a copyright symbol encoded with latin-1.
# We use $(printf "\\xa9") rather than $'\\xa9', since printf apparently
# works in any case, while $'\\xa9' transforms to \\xef\\xbf\\xbd under
# some conditions. TODO: Find out why it transforms to \\xef\\xbf\\xbd when
# running tests for Python 3.2 (even though it's bash that is ultimately
# responsible for performing the transformation).
local latin_1_dir=/usr/lib/${P}/latin-1-$(printf "\\xa9")-directory
insinto "${latin_1_dir}"
echo "blah blah blah" > "${T}"/latin-1-$(printf "\\xa9")-regular-file || die
doins "${T}"/latin-1-$(printf "\\xa9")-regular-file
dosym latin-1-$(printf "\\xa9")-regular-file ${latin_1_dir}/latin-1-$(printf "\\xa9")-symlink || die
call_has_and_best_version
}
pkg_config() {
einfo "called pkg_config for $CATEGORY/$PF"
}
pkg_info() {
einfo "called pkg_info for $CATEGORY/$PF"
}
pkg_preinst() {
if ! ___eapi_best_version_and_has_version_support_-b_-d_-r; then
# The BROOT variable is unset during pkg_* phases for EAPI 7,
# therefore best/has_version -b is expected to fail if we attempt
# to call it for EAPI 7 here.
call_has_and_best_version
fi
}
call_has_and_best_version() {
local root_arg
if ___eapi_best_version_and_has_version_support_-b_-d_-r; then
root_arg="-b"
else
root_arg="--host-root"
fi
einfo "called ${EBUILD_PHASE_FUNC} for $CATEGORY/$PF"
einfo "EPREFIX=${EPREFIX}"
einfo "PORTAGE_OVERRIDE_EPREFIX=${PORTAGE_OVERRIDE_EPREFIX}"
einfo "ROOT=${ROOT}"
einfo "EROOT=${EROOT}"
einfo "SYSROOT=${SYSROOT}"
einfo "ESYSROOT=${ESYSROOT}"
einfo "BROOT=${BROOT}"
# Test that has_version and best_version work correctly with
# prefix (involves internal ROOT -> EROOT calculation in order
# to support ROOT override via the environment with EAPIs 3
# and later which support prefix).
if has_version $CATEGORY/$PN:$SLOT ; then
einfo "has_version detects an installed instance of $CATEGORY/$PN:$SLOT"
einfo "best_version reports that the installed instance is $(best_version $CATEGORY/$PN:$SLOT)"
else
einfo "has_version does not detect an installed instance of $CATEGORY/$PN:$SLOT"
fi
if [[ ${EPREFIX} != ${PORTAGE_OVERRIDE_EPREFIX} ]] ; then
if has_version ${root_arg} $CATEGORY/$PN:$SLOT ; then
einfo "has_version ${root_arg} detects an installed instance of $CATEGORY/$PN:$SLOT"
einfo "best_version ${root_arg} reports that the installed instance is $(best_version ${root_arg} $CATEGORY/$PN:$SLOT)"
else
einfo "has_version ${root_arg} does not detect an installed instance of $CATEGORY/$PN:$SLOT"
fi
fi
}
"""
ebuilds = {
"dev-libs/A-1": {
"EAPI": "5",
"IUSE": "+flag",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"MISC_CONTENT": install_something,
"RDEPEND": "flag? ( dev-libs/B[flag] )",
},
"dev-libs/B-1": {
"EAPI": "5",
"IUSE": "+flag",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"MISC_CONTENT": install_something,
},
"dev-libs/C-1": {
"EAPI": "7",
"KEYWORDS": "~x86",
"RDEPEND": "dev-libs/D[flag]",
"MISC_CONTENT": install_something,
},
"dev-libs/D-1": {
"EAPI": "7",
"KEYWORDS": "~x86",
"IUSE": "flag",
"MISC_CONTENT": install_something,
},
"virtual/foo-0": {
"EAPI": "5",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
},
}
installed = {
"dev-libs/A-1": {
"EAPI": "5",
"IUSE": "+flag",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"RDEPEND": "flag? ( dev-libs/B[flag] )",
"USE": "flag",
},
"dev-libs/B-1": {
"EAPI": "5",
"IUSE": "+flag",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"USE": "flag",
},
"dev-libs/depclean-me-1": {
"EAPI": "5",
"IUSE": "",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"USE": "",
},
"app-misc/depclean-me-1": {
"EAPI": "5",
"IUSE": "",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"RDEPEND": "dev-libs/depclean-me",
"USE": "",
},
}
metadata_xml_files = (
(
"dev-libs/A",
{
"flags":
"<flag name='flag'>Description of how USE='flag' affects this package</flag>",
},
),
(
"dev-libs/B",
{
"flags":
"<flag name='flag'>Description of how USE='flag' affects this package</flag>",
},
),
)
for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
with self.subTest(binpkg_format=binpkg_format):
print(colorize("HILITE", binpkg_format), end=" ... ")
sys.stdout.flush()
playground = ResolverPlayground(
ebuilds=ebuilds,
installed=installed,
debug=debug,
user_config={
"make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format, ),
},
)
loop = asyncio._wrap_loop()
loop.run_until_complete(
asyncio.ensure_future(
self._async_test_simple(playground,
metadata_xml_files,
loop=loop),
loop=loop,
))
def _start(self):
self.future = asyncio.ensure_future(
self.scheduler.run_in_executor(ForkExecutor(loop=self.scheduler),
self._run))
super(FileCopier, self)._start()
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
An asynchronous version of iter_completed. This yields futures, which
when done, result in a set of input futures that are done. This serves
as a wrapper around portage's internal TaskScheduler class, using
standard asyncio interfaces.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures, which when done, result in a set of
input futures that are done
@rtype: iterator
"""
loop = asyncio._wrap_loop(loop)
max_jobs = max_jobs or get_cpu_count()
max_load = max_load or get_cpu_count()
future_map = {}
def task_generator():
for future in futures:
future_map[id(future)] = future
yield AsyncTaskFuture(future=future)
scheduler = TaskScheduler(task_generator(),
max_jobs=max_jobs,
max_load=max_load,
event_loop=loop)
def done_callback(future_done_set, wait_result):
"""Propagate results from wait_result to future_done_set."""
if future_done_set.cancelled():
return
done, pending = wait_result.result()
for future in done:
del future_map[id(future)]
future_done_set.set_result(done)
def cancel_callback(wait_result, future_done_set):
"""Cancel wait_result if future_done_set has been cancelled."""
if future_done_set.cancelled() and not wait_result.done():
wait_result.cancel()
@coroutine
def fetch_wait_result(scheduler, first, loop=None):
if first:
yield scheduler.async_start()
# If the current coroutine awakens just after a call to
# done_callback but before scheduler has been notified of
# corresponding done future(s), then wait here until scheduler
# is notified (which will cause future_map to populate).
while not future_map and scheduler.poll() is None:
yield asyncio.sleep(0, loop=loop)
if not future_map:
if scheduler.poll() is not None:
coroutine_return((set(), set()))
else:
raise AssertionError('expected non-empty future_map')
wait_result = yield asyncio.wait(list(future_map.values()),
return_when=asyncio.FIRST_COMPLETED,
loop=loop)
coroutine_return(wait_result)
first = True
try:
while True:
wait_result = asyncio.ensure_future(fetch_wait_result(scheduler,
first,
loop=loop),
loop=loop)
first = False
future_done_set = loop.create_future()
future_done_set.add_done_callback(
functools.partial(cancel_callback, wait_result))
wait_result.add_done_callback(
functools.partial(done_callback, future_done_set))
yield future_done_set
if not future_map and scheduler.poll() is not None:
break
finally:
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()
def start_coroutine(future):
result = asyncio.ensure_future(coroutine_func(),
loop=parent_loop)
pending[id(result)] = result
result.add_done_callback(done_callback)
future.set_result(result)
def _start(self):
future = asyncio.ensure_future(self._async_start(),
loop=self.scheduler)
self._start_task(AsyncTaskFuture(future=future),
self._async_start_exit)
def test_method_coroutine(self):
class Cubby:
_empty = object()
def __init__(self, loop):
self._loop = loop
self._value = self._empty
self._waiters = []
def _notify(self):
waiters = self._waiters
self._waiters = []
for waiter in waiters:
waiter.cancelled() or waiter.set_result(None)
def _wait(self):
waiter = self._loop.create_future()
self._waiters.append(waiter)
return waiter
@coroutine
def read(self, loop=None):
while self._value is self._empty:
yield self._wait()
value = self._value
self._value = self._empty
self._notify()
coroutine_return(value)
@coroutine
def write(self, value, loop=None):
while self._value is not self._empty:
yield self._wait()
self._value = value
self._notify()
@coroutine
def writer_coroutine(cubby, values, sentinel, loop=None):
for value in values:
yield cubby.write(value, loop=loop)
yield cubby.write(sentinel, loop=loop)
@coroutine
def reader_coroutine(cubby, sentinel, loop=None):
results = []
while True:
result = yield cubby.read(loop=loop)
if result == sentinel:
break
results.append(result)
coroutine_return(results)
loop = asyncio.get_event_loop()
cubby = Cubby(loop)
values = list(range(3))
writer = asyncio.ensure_future(writer_coroutine(cubby,
values,
None,
loop=loop),
loop=loop)
reader = asyncio.ensure_future(reader_coroutine(cubby, None,
loop=loop),
loop=loop)
loop.run_until_complete(asyncio.wait([writer, reader], loop=loop))
self.assertEqual(reader.result(), values)
# Test decoration of coroutine methods and functions for
# synchronous usage, allowing coroutines to smoothly
# blend with synchronous code.
sync_cubby = _sync_methods(cubby, loop=loop)
sync_reader = _sync_decorator(reader_coroutine, loop=loop)
writer = asyncio.ensure_future(writer_coroutine(cubby,
values,
None,
loop=loop),
loop=loop)
self.assertEqual(sync_reader(cubby, None), values)
self.assertTrue(writer.done())
for i in range(3):
sync_cubby.write(i)
self.assertEqual(sync_cubby.read(), i)
def _iter_tasks(self, loop, executor, ebuild, pkg):
for keyword, groups, prof in ebuild.relevant_profiles:
is_stable_profile = prof.status == "stable"
is_dev_profile = prof.status == "dev" and \
self.options.include_dev
is_exp_profile = prof.status == "exp" and \
self.options.include_exp_profiles == 'y'
if not (is_stable_profile or is_dev_profile or is_exp_profile):
continue
dep_settings = self.caches['arch'].get(prof.sub_path)
if dep_settings is None:
dep_settings = portage.config(
config_profile_path=prof.abs_path,
config_incrementals=self.repoman_incrementals,
config_root=self.repo_settings.config_root,
local_config=False,
_unmatched_removal=self.options.unmatched_removal,
env=self.env, repositories=self.repo_settings.repoman_settings.repositories)
if not prof.abs_path:
self._populate_implicit_iuse(dep_settings,
self.repo_settings.repo_config.eclass_db.porttrees)
dep_settings.categories = self.repo_settings.repoman_settings.categories
if self.options.without_mask:
dep_settings._mask_manager_obj = \
copy.deepcopy(dep_settings._mask_manager)
dep_settings._mask_manager._pmaskdict.clear()
self.caches['arch'][prof.sub_path] = dep_settings
xmatch_cache_key = (prof.sub_path, tuple(groups))
xcache = self.caches['arch_xmatch'].get(xmatch_cache_key)
if xcache is None:
self.portdb.melt()
self.portdb.freeze()
xcache = self.portdb.xcache
xcache.update(self.caches['shared_xmatch'])
self.caches['arch_xmatch'][xmatch_cache_key] = xcache
self.repo_settings.trees[self.repo_settings.root]["porttree"].settings = dep_settings
self.portdb.settings = dep_settings
self.portdb.xcache = xcache
dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
# just in case, prevent config.reset() from nuking these.
dep_settings.backup_changes("ACCEPT_KEYWORDS")
# This attribute is used in dbapi._match_use() to apply
# use.stable.{mask,force} settings based on the stable
# status of the parent package. This is required in order
# for USE deps of unstable packages to be resolved correctly,
# since otherwise use.stable.{mask,force} settings of
# dependencies may conflict (see bug #456342).
dep_settings._parent_stable = dep_settings._isStable(pkg)
# Handle package.use*.{force,mask) calculation, for use
# in dep_check.
dep_settings.useforce = dep_settings._use_manager.getUseForce(
pkg, stable=dep_settings._parent_stable)
dep_settings.usemask = dep_settings._use_manager.getUseMask(
pkg, stable=dep_settings._parent_stable)
task = types.SimpleNamespace(ebuild=ebuild, prof=prof, keyword=keyword)
target = functools.partial(self._task_subprocess, task, pkg, dep_settings)
if self.options.jobs <= 1:
yield (task, target())
else:
task.future = asyncio.ensure_future(loop.run_in_executor(executor, target), loop=loop)
yield self._task(task, loop=loop)
def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None):
"""
An asynchronous version of iter_completed. This yields futures, which
when done, result in a set of input futures that are done. This serves
as a wrapper around portage's internal TaskScheduler class, using
standard asyncio interfaces.
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@return: iterator of futures, which when done, result in a set of
input futures that are done
@rtype: iterator
"""
loop = asyncio._wrap_loop(loop)
max_jobs = max_jobs or get_cpu_count()
max_load = max_load or get_cpu_count()
future_map = {}
def task_generator():
for future in futures:
future_map[id(future)] = future
yield AsyncTaskFuture(future=future)
scheduler = TaskScheduler(task_generator(),
max_jobs=max_jobs,
max_load=max_load,
event_loop=loop)
def done_callback(future_done_set, wait_result):
"""Propagate results from wait_result to future_done_set."""
if future_done_set.cancelled():
return
done, pending = wait_result.result()
for future in done:
del future_map[id(future)]
future_done_set.set_result(done)
def cancel_callback(wait_result, future_done_set):
"""Cancel wait_result if future_done_set has been cancelled."""
if future_done_set.cancelled() and not wait_result.done():
wait_result.cancel()
try:
scheduler.start()
# scheduler should ensure that future_map is non-empty until
# task_generator is exhausted
while future_map:
wait_result = asyncio.ensure_future(asyncio.wait(
list(future_map.values()),
return_when=asyncio.FIRST_COMPLETED,
loop=loop),
loop=loop)
future_done_set = loop.create_future()
future_done_set.add_done_callback(
functools.partial(cancel_callback, wait_result))
wait_result.add_done_callback(
functools.partial(done_callback, future_done_set))
yield future_done_set
finally:
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()
def test_method_coroutine(self): class Cubby(object): _empty = object() def __init__(self, loop): self._loop = loop self._value = self._empty self._waiters = [] def _notify(self): waiters = self._waiters self._waiters = [] for waiter in waiters: waiter.cancelled() or waiter.set_result(None) def _wait(self): waiter = self._loop.create_future() self._waiters.append(waiter) return waiter @coroutine def read(self): while self._value is self._empty: yield self._wait() value = self._value self._value = self._empty self._notify() coroutine_return(value) @coroutine def write(self, value): while self._value is not self._empty: yield self._wait() self._value = value self._notify() @coroutine def writer_coroutine(cubby, values, sentinel): for value in values: yield cubby.write(value) yield cubby.write(sentinel) @coroutine def reader_coroutine(cubby, sentinel): results = [] while True: result = yield cubby.read() if result == sentinel: break results.append(result) coroutine_return(results) loop = asyncio.get_event_loop() cubby = Cubby(loop) values = list(range(3)) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop) reader = asyncio.ensure_future(reader_coroutine(cubby, None), loop=loop) loop.run_until_complete(asyncio.wait([writer, reader])) self.assertEqual(reader.result(), values)
def _start(self):
self.future = asyncio.ensure_future(self.future, self.scheduler)
self.future.add_done_callback(self._done_callback)
def testSimple(self):
debug = False
skip_reason = self._must_skip()
if skip_reason:
self.portage_skip = skip_reason
self.assertFalse(True, skip_reason)
return
copyright_header = """# Copyright 1999-%s Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
""" % time.gmtime().tm_year
pkg_preinst_references_forbidden_var = """
pkg_preinst() {
echo "This ${A} reference is not allowed. Neither is this $BROOT reference."
}
"""
repo_configs = {
"test_repo": {
"layout.conf": ("update-changelog = true", ),
}
}
profiles = (
("x86", "default/linux/x86/test_profile", "stable"),
("x86", "default/linux/x86/test_dev", "dev"),
("x86", "default/linux/x86/test_exp", "exp"),
)
profile = {
"eapi": ("5", ),
"package.use.stable.mask": ("dev-libs/A flag", )
}
ebuilds = {
"dev-libs/A-0": {
"COPYRIGHT_HEADER": copyright_header,
"DESCRIPTION": "Desc goes here",
"EAPI": "5",
"HOMEPAGE": "https://example.com",
"IUSE": "flag",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"RDEPEND": "flag? ( dev-libs/B[flag] )",
},
"dev-libs/A-1": {
"COPYRIGHT_HEADER": copyright_header,
"DESCRIPTION": "Desc goes here",
"EAPI": "4",
"HOMEPAGE": "https://example.com",
"IUSE": "flag",
"KEYWORDS": "~x86",
"LICENSE": "GPL-2",
"RDEPEND": "flag? ( dev-libs/B[flag] )",
},
"dev-libs/B-1": {
"COPYRIGHT_HEADER": copyright_header,
"DESCRIPTION": "Desc goes here",
"EAPI": "4",
"HOMEPAGE": "https://example.com",
"IUSE": "flag",
"KEYWORDS": "~x86",
"LICENSE": "GPL-2",
},
"dev-libs/C-0": {
"COPYRIGHT_HEADER": copyright_header,
"DESCRIPTION": "Desc goes here",
"EAPI": "7",
"HOMEPAGE": "https://example.com",
"IUSE": "flag",
# must be unstable, since dev-libs/A[flag] is stable masked
"KEYWORDS": "~x86",
"LICENSE": "GPL-2",
"RDEPEND": "flag? ( dev-libs/A[flag] )",
"MISC_CONTENT": pkg_preinst_references_forbidden_var,
},
}
licenses = ["GPL-2"]
arch_list = ["x86"]
metadata_xsd = os.path.join(REPOMAN_BASE_PATH, "cnf/metadata.xsd")
metadata_xml_files = (
(
"dev-libs/A",
{
"flags":
"<flag name='flag' restrict='>=dev-libs/A-0'>Description of how USE='flag' affects this package</flag>",
},
),
(
"dev-libs/B",
{
"flags":
"<flag name='flag'>Description of how USE='flag' affects this package</flag>",
},
),
(
"dev-libs/C",
{
"flags":
"<flag name='flag'>Description of how USE='flag' affects this package</flag>",
},
),
)
use_desc = (("flag",
"Description of how USE='flag' affects packages"), )
playground = ResolverPlayground(ebuilds=ebuilds,
profile=profile,
repo_configs=repo_configs,
debug=debug)
loop = asyncio._wrap_loop()
loop.run_until_complete(
asyncio.ensure_future(
self._async_test_simple(
playground,
metadata_xml_files,
profiles,
profile,
licenses,
arch_list,
use_desc,
metadata_xsd,
copyright_header,
debug,
),
loop=loop,
))
def test_method_coroutine(self): class Cubby(object): _empty = object() def __init__(self, loop): self._loop = loop self._value = self._empty self._waiters = [] def _notify(self): waiters = self._waiters self._waiters = [] for waiter in waiters: waiter.cancelled() or waiter.set_result(None) def _wait(self): waiter = self._loop.create_future() self._waiters.append(waiter) return waiter @coroutine def read(self): while self._value is self._empty: yield self._wait() value = self._value self._value = self._empty self._notify() coroutine_return(value) @coroutine def write(self, value): while self._value is not self._empty: yield self._wait() self._value = value self._notify() @coroutine def writer_coroutine(cubby, values, sentinel): for value in values: yield cubby.write(value) yield cubby.write(sentinel) @coroutine def reader_coroutine(cubby, sentinel): results = [] while True: result = yield cubby.read() if result == sentinel: break results.append(result) coroutine_return(results) loop = asyncio.get_event_loop() cubby = Cubby(loop) values = list(range(3)) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop) reader = asyncio.ensure_future(reader_coroutine(cubby, None), loop=loop) loop.run_until_complete(asyncio.wait([writer, reader])) self.assertEqual(reader.result(), values) # Test decoration of coroutine methods and functions for # synchronous usage, allowing coroutines to smoothly # blend with synchronous code. sync_cubby = _sync_methods(cubby, loop=loop) sync_reader = _sync_decorator(reader_coroutine, loop=loop) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop) self.assertEqual(sync_reader(cubby, None), values) self.assertTrue(writer.done()) for i in range(3): sync_cubby.write(i) self.assertEqual(sync_cubby.read(), i)
