async def _testPipeLoggerToPipe(self, test_string, loop):
"""
Test PipeLogger writing to a pipe connected to a PipeReader.
This verifies that PipeLogger does not deadlock when writing
to a pipe that's drained by a PipeReader running in the same
process (requires non-blocking write).
"""
input_fd, writer_pipe = os.pipe()
_set_nonblocking(writer_pipe)
writer_pipe = os.fdopen(writer_pipe, 'wb', 0)
writer = asyncio.ensure_future(
_writer(writer_pipe, test_string.encode('ascii')))
writer.add_done_callback(lambda writer: writer_pipe.close())
pr, pw = os.pipe()
consumer = PipeLogger(background=True,
input_fd=input_fd,
log_file_path=os.fdopen(pw, 'wb', 0),
scheduler=loop)
consumer.start()
# Before starting the reader, wait here for a moment, in order
# to exercise PipeLogger's handling of EAGAIN during write.
await asyncio.wait([writer], timeout=0.01)
reader = _reader(pr)
await writer
content = await reader
await consumer.async_wait()
self.assertEqual(consumer.returncode, os.EX_OK)
return content.decode('ascii', 'replace')
def _start(self):
filter_proc = None
log_input = None
if self.log_path is not None:
log_filter_file = self.log_filter_file
if log_filter_file is not None:
split_value = shlex_split(log_filter_file)
log_filter_file = split_value if split_value else None
if log_filter_file:
filter_input, stdin = os.pipe()
log_input, filter_output = os.pipe()
try:
filter_proc = PopenProcess(
proc=subprocess.Popen(
log_filter_file,
env=self.env,
stdin=filter_input,
stdout=filter_output,
stderr=filter_output,
),
scheduler=self.scheduler,
)
filter_proc.start()
except EnvironmentError:
# Maybe the command is missing or broken somehow...
os.close(filter_input)
os.close(stdin)
os.close(log_input)
os.close(filter_output)
else:
self._stdin = os.fdopen(stdin, "wb", 0)
os.close(filter_input)
os.close(filter_output)
if self._stdin is None:
# Since log_filter_file is unspecified or refers to a file
# that is missing or broken somehow, create a pipe that
# logs directly to pipe_logger.
log_input, stdin = os.pipe()
self._stdin = os.fdopen(stdin, "wb", 0)
# Set background=True so that pipe_logger does not log to stdout.
pipe_logger = PipeLogger(
background=True,
scheduler=self.scheduler,
input_fd=log_input,
log_file_path=self.log_path,
)
pipe_logger.start()
self._main_task_cancel = functools.partial(self._main_cancel,
filter_proc, pipe_logger)
self._main_task = asyncio.ensure_future(self._main(
filter_proc, pipe_logger),
loop=self.scheduler)
self._main_task.add_done_callback(self._main_exit)
def _testPipeLogger(self, test_string):
producer = PopenProcess(proc=subprocess.Popen(
["bash", "-c", self._echo_cmd % test_string],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT),
scheduler=global_event_loop())
fd, log_file_path = tempfile.mkstemp()
try:
consumer = PipeLogger(background=True,
input_fd=producer.proc.stdout,
log_file_path=log_file_path)
producer.pipe_reader = consumer
producer.start()
producer.wait()
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)
with open(log_file_path, 'rb') as f:
content = f.read()
finally:
os.close(fd)
os.unlink(log_file_path)
return content.decode('ascii', 'replace')
def _testPipeLogger(self, test_string):
producer = PopenProcess(proc=subprocess.Popen(
["bash", "-c", self._echo_cmd % test_string],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT),
scheduler=global_event_loop())
fd, log_file_path = tempfile.mkstemp()
try:
consumer = PipeLogger(background=True,
input_fd=os.dup(
producer.proc.stdout.fileno()),
log_file_path=log_file_path)
# Close the stdout pipe, since we duplicated it, and it
# must be closed in order to avoid a ResourceWarning.
producer.proc.stdout.close()
producer.pipe_reader = consumer
producer.start()
producer.wait()
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)
with open(log_file_path, 'rb') as f:
content = f.read()
finally:
os.close(fd)
os.unlink(log_file_path)
return content.decode('ascii', 'replace')
def _testPipeLoggerToPipe(self, test_string, loop=None):
"""
Test PipeLogger writing to a pipe connected to a PipeReader.
This verifies that PipeLogger does not deadlock when writing
to a pipe that's drained by a PipeReader running in the same
process (requires non-blocking write).
"""
producer = PopenProcess(proc=subprocess.Popen(
["bash", "-c", self._echo_cmd % test_string],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT),
scheduler=loop)
pr, pw = os.pipe()
consumer = producer.pipe_reader = PipeLogger(
background=True,
input_fd=producer.proc.stdout,
log_file_path=os.fdopen(pw, 'wb', 0))
reader = _reader(pr, loop=loop)
yield producer.async_start()
content = yield reader
yield producer.async_wait()
yield consumer.async_wait()
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)
coroutine_return(content.decode('ascii', 'replace'))
def _fetch_uri(self, uri):
if self.config.options.dry_run:
# Simply report success.
logging.info("dry-run: fetch '%s' from '%s'" %
(self.distfile, uri))
self._success()
self.returncode = os.EX_OK
self._async_wait()
return
if self.config.options.temp_dir:
self._fetch_tmp_dir_info = "temp-dir"
distdir = self.config.options.temp_dir
else:
self._fetch_tmp_dir_info = "distfiles"
distdir = self.config.options.distfiles
tmp_basename = self.distfile + "._emirrordist_fetch_.%s" % portage.getpid(
)
variables = {"DISTDIR": distdir, "URI": uri, "FILE": tmp_basename}
self._fetch_tmp_file = os.path.join(distdir, tmp_basename)
try:
os.unlink(self._fetch_tmp_file)
except OSError:
pass
args = portage.util.shlex_split(default_fetchcommand)
args = [portage.util.varexpand(x, mydict=variables) for x in args]
args = [
_unicode_encode(x, encoding=_encodings["fs"], errors="strict")
for x in args
]
null_fd = os.open(os.devnull, os.O_RDONLY)
fetcher = PopenProcess(
background=self.background,
proc=subprocess.Popen(args,
stdin=null_fd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT),
scheduler=self.scheduler,
)
os.close(null_fd)
fetcher.pipe_reader = PipeLogger(
background=self.background,
input_fd=fetcher.proc.stdout,
log_file_path=self._log_path,
scheduler=self.scheduler,
)
self._start_task(fetcher, self._fetcher_exit)
def _async_start(self):
pipe_logger = None
filter_proc = None
try:
log_input = None
if self.log_path is not None:
log_filter_file = self.log_filter_file
if log_filter_file is not None:
split_value = shlex_split(log_filter_file)
log_filter_file = split_value if split_value else None
if log_filter_file:
filter_input, stdin = os.pipe()
log_input, filter_output = os.pipe()
try:
filter_proc = yield asyncio.create_subprocess_exec(
*log_filter_file,
env=self.env,
stdin=filter_input,
stdout=filter_output,
stderr=filter_output,
loop=self.scheduler)
except EnvironmentError:
# Maybe the command is missing or broken somehow...
os.close(filter_input)
os.close(stdin)
os.close(log_input)
os.close(filter_output)
else:
self._stdin = os.fdopen(stdin, 'wb', 0)
os.close(filter_input)
os.close(filter_output)
if self._stdin is None:
# Since log_filter_file is unspecified or refers to a file
# that is missing or broken somehow, create a pipe that
# logs directly to pipe_logger.
log_input, stdin = os.pipe()
self._stdin = os.fdopen(stdin, 'wb', 0)
# Set background=True so that pipe_logger does not log to stdout.
pipe_logger = PipeLogger(background=True,
scheduler=self.scheduler,
input_fd=log_input,
log_file_path=self.log_path)
yield pipe_logger.async_start()
except asyncio.CancelledError:
if pipe_logger is not None and pipe_logger.poll() is None:
pipe_logger.cancel()
if filter_proc is not None and filter_proc.returncode is None:
filter_proc.terminate()
raise
self._main_task = asyncio.ensure_future(self._main(
pipe_logger, filter_proc=filter_proc),
loop=self.scheduler)
self._main_task.add_done_callback(self._main_exit)
def _start_gpg_proc(self):
gpg_vars = self.gpg_vars
if gpg_vars is None:
gpg_vars = {}
else:
gpg_vars = gpg_vars.copy()
gpg_vars["FILE"] = self._manifest_path
gpg_cmd = varexpand(self.gpg_cmd, mydict=gpg_vars)
gpg_cmd = shlex_split(gpg_cmd)
gpg_proc = PopenProcess(proc=subprocess.Popen(gpg_cmd,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
# PipeLogger echos output and efficiently monitors for process
# exit by listening for the stdout EOF event.
gpg_proc.pipe_reader = PipeLogger(background=self.background,
input_fd=gpg_proc.proc.stdout, scheduler=self.scheduler)
self._start_task(gpg_proc, self._gpg_proc_exit)
def _fetch_uri(self, uri):
if self.config.options.dry_run:
# Simply report success.
logging.info("dry-run: fetch '%s' from '%s'" %
(self.distfile, uri))
self._success()
self.returncode = os.EX_OK
self.wait()
return
if self.config.options.temp_dir:
self._fetch_tmp_dir_info = 'temp-dir'
distdir = self.config.options.temp_dir
else:
self._fetch_tmp_dir_info = 'distfiles'
distdir = self.config.options.distfiles
tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid()
variables = {"DISTDIR": distdir, "URI": uri, "FILE": tmp_basename}
self._fetch_tmp_file = os.path.join(distdir, tmp_basename)
try:
os.unlink(self._fetch_tmp_file)
except OSError:
pass
args = portage.util.shlex_split(default_fetchcommand)
args = [portage.util.varexpand(x, mydict=variables) for x in args]
if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
not os.path.isabs(args[0]):
# Python 3.1 _execvp throws TypeError for non-absolute executable
# path passed as bytes (see http://bugs.python.org/issue8513).
fullname = portage.process.find_binary(args[0])
if fullname is None:
raise portage.exception.CommandNotFound(args[0])
args[0] = fullname
args = [
_unicode_encode(x, encoding=_encodings['fs'], errors='strict')
for x in args
]
null_fd = os.open(os.devnull, os.O_RDONLY)
fetcher = PopenProcess(background=self.background,
proc=subprocess.Popen(args,
stdin=null_fd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT),
scheduler=self.scheduler)
os.close(null_fd)
fetcher.pipe_reader = PipeLogger(background=self.background,
input_fd=fetcher.proc.stdout,
log_file_path=self._log_path,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetcher_exit)
def _start(self):
if self.fd_pipes is None:
self.fd_pipes = {}
fd_pipes = self.fd_pipes
master_fd, slave_fd = self._pipe(fd_pipes)
can_log = self._can_log(slave_fd)
if can_log:
log_file_path = self.logfile
else:
log_file_path = None
null_input = None
if not self.background or 0 in fd_pipes:
# Subclasses such as AbstractEbuildProcess may have already passed
# in a null file descriptor in fd_pipes, so use that when given.
pass
else:
# TODO: Use job control functions like tcsetpgrp() to control
# access to stdin. Until then, use /dev/null so that any
# attempts to read from stdin will immediately return EOF
# instead of blocking indefinitely.
null_input = os.open('/dev/null', os.O_RDWR)
fd_pipes[0] = null_input
fd_pipes.setdefault(0, sys.__stdin__.fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stderr__.fileno())
# flush any pending output
stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
for fd in fd_pipes.values():
if fd in stdout_filenos:
sys.__stdout__.flush()
sys.__stderr__.flush()
break
fd_pipes_orig = fd_pipes.copy()
if log_file_path is not None:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
else:
# Create a dummy pipe so the scheduler can monitor
# the process from inside a poll() loop.
fd_pipes[self._dummy_pipe_fd] = slave_fd
if self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
kwargs = {}
for k in self._spawn_kwarg_names:
v = getattr(self, k)
if v is not None:
kwargs[k] = v
kwargs["fd_pipes"] = fd_pipes
kwargs["returnpid"] = True
kwargs.pop("logfile", None)
retval = self._spawn(self.args, **kwargs)
os.close(slave_fd)
if null_input is not None:
os.close(null_input)
if isinstance(retval, int):
# spawn failed
self._unregister()
self._set_returncode((self.pid, retval))
self.wait()
return
self.pid = retval[0]
portage.process.spawned_pids.remove(self.pid)
stdout_fd = None
if can_log and not self.background:
stdout_fd = os.dup(fd_pipes_orig[1])
self._pipe_logger = PipeLogger(background=self.background,
scheduler=self.scheduler, input_fd=master_fd,
log_file_path=log_file_path,
stdout_fd=stdout_fd)
self._pipe_logger.addExitListener(self._pipe_logger_exit)
self._pipe_logger.start()
self._registered = True
class SpawnProcess(SubProcess):
"""
Constructor keyword args are passed into portage.process.spawn().
The required "args" keyword argument will be passed as the first
spawn() argument.
"""
_spawn_kwarg_names = ("env", "opt_name", "fd_pipes",
"uid", "gid", "groups", "umask", "logfile",
"path_lookup", "pre_exec")
__slots__ = ("args",) + \
_spawn_kwarg_names + ("_pipe_logger", "_selinux_type",)
def _start(self):
if self.fd_pipes is None:
self.fd_pipes = {}
fd_pipes = self.fd_pipes
master_fd, slave_fd = self._pipe(fd_pipes)
can_log = self._can_log(slave_fd)
if can_log:
log_file_path = self.logfile
else:
log_file_path = None
null_input = None
if not self.background or 0 in fd_pipes:
# Subclasses such as AbstractEbuildProcess may have already passed
# in a null file descriptor in fd_pipes, so use that when given.
pass
else:
# TODO: Use job control functions like tcsetpgrp() to control
# access to stdin. Until then, use /dev/null so that any
# attempts to read from stdin will immediately return EOF
# instead of blocking indefinitely.
null_input = os.open('/dev/null', os.O_RDWR)
fd_pipes[0] = null_input
fd_pipes.setdefault(0, sys.__stdin__.fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stderr__.fileno())
# flush any pending output
stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
for fd in fd_pipes.values():
if fd in stdout_filenos:
sys.__stdout__.flush()
sys.__stderr__.flush()
break
fd_pipes_orig = fd_pipes.copy()
if log_file_path is not None:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
else:
# Create a dummy pipe so the scheduler can monitor
# the process from inside a poll() loop.
fd_pipes[self._dummy_pipe_fd] = slave_fd
if self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
kwargs = {}
for k in self._spawn_kwarg_names:
v = getattr(self, k)
if v is not None:
kwargs[k] = v
kwargs["fd_pipes"] = fd_pipes
kwargs["returnpid"] = True
kwargs.pop("logfile", None)
retval = self._spawn(self.args, **kwargs)
os.close(slave_fd)
if null_input is not None:
os.close(null_input)
if isinstance(retval, int):
# spawn failed
self._unregister()
self._set_returncode((self.pid, retval))
self.wait()
return
self.pid = retval[0]
portage.process.spawned_pids.remove(self.pid)
stdout_fd = None
if can_log and not self.background:
stdout_fd = os.dup(fd_pipes_orig[1])
self._pipe_logger = PipeLogger(background=self.background,
scheduler=self.scheduler, input_fd=master_fd,
log_file_path=log_file_path,
stdout_fd=stdout_fd)
self._pipe_logger.addExitListener(self._pipe_logger_exit)
self._pipe_logger.start()
self._registered = True
def _can_log(self, slave_fd):
return True
def _pipe(self, fd_pipes):
"""
@type fd_pipes: dict
@param fd_pipes: pipes from which to copy terminal size if desired.
"""
return os.pipe()
def _spawn(self, args, **kwargs):
spawn_func = portage.process.spawn
if self._selinux_type is not None:
spawn_func = portage.selinux.spawn_wrapper(spawn_func,
self._selinux_type)
# bash is an allowed entrypoint, while most binaries are not
if args[0] != BASH_BINARY:
args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
return spawn_func(args, **kwargs)
def _pipe_logger_exit(self, pipe_logger):
self._pipe_logger = None
self._unregister()
self.wait()
def _waitpid_loop(self):
SubProcess._waitpid_loop(self)
pipe_logger = self._pipe_logger
if pipe_logger is not None:
self._pipe_logger = None
pipe_logger.removeExitListener(self._pipe_logger_exit)
pipe_logger.cancel()
pipe_logger.wait()
def _start(self):
if self.fd_pipes is None:
self.fd_pipes = {}
else:
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
master_fd, slave_fd = self._pipe(fd_pipes)
can_log = self._can_log(slave_fd)
if can_log:
log_file_path = self.logfile
else:
log_file_path = None
null_input = None
if not self.background or 0 in fd_pipes:
# Subclasses such as AbstractEbuildProcess may have already passed
# in a null file descriptor in fd_pipes, so use that when given.
pass
else:
# TODO: Use job control functions like tcsetpgrp() to control
# access to stdin. Until then, use /dev/null so that any
# attempts to read from stdin will immediately return EOF
# instead of blocking indefinitely.
null_input = os.open('/dev/null', os.O_RDWR)
fd_pipes[0] = null_input
fd_pipes.setdefault(0, portage._get_stdin().fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stderr__.fileno())
# flush any pending output
stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
for fd in fd_pipes.values():
if fd in stdout_filenos:
sys.__stdout__.flush()
sys.__stderr__.flush()
break
fd_pipes_orig = fd_pipes.copy()
if log_file_path is not None or self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
else:
# Create a dummy pipe that PipeLogger uses to efficiently
# monitor for process exit by listening for the EOF event.
# Re-use of the allocated fd number for the key in fd_pipes
# guarantees that the keys will not collide for similarly
# allocated pipes which are used by callers such as
# FileDigester and MergeProcess. See the _setup_pipes
# docstring for more benefits of this allocation approach.
self._dummy_pipe_fd = slave_fd
fd_pipes[slave_fd] = slave_fd
kwargs = {}
for k in self._spawn_kwarg_names:
v = getattr(self, k)
if v is not None:
kwargs[k] = v
kwargs["fd_pipes"] = fd_pipes
kwargs["returnpid"] = True
kwargs.pop("logfile", None)
retval = self._spawn(self.args, **kwargs)
os.close(slave_fd)
if null_input is not None:
os.close(null_input)
if isinstance(retval, int):
# spawn failed
self.returncode = retval
self._async_wait()
return
self.pid = retval[0]
stdout_fd = None
if can_log and not self.background:
stdout_fd = os.dup(fd_pipes_orig[1])
build_logger = BuildLogger(env=self.env,
log_path=log_file_path,
log_filter_file=self.log_filter_file,
scheduler=self.scheduler)
build_logger.start()
pipe_logger = PipeLogger(background=self.background,
scheduler=self.scheduler,
input_fd=master_fd,
log_file_path=build_logger.stdin,
stdout_fd=stdout_fd)
pipe_logger.start()
self._registered = True
self._main_task = asyncio.ensure_future(self._main(
build_logger, pipe_logger),
loop=self.scheduler)
self._main_task.add_done_callback(self._main_exit)
def _start(self):
if self.fd_pipes is None:
self.fd_pipes = {}
else:
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
master_fd, slave_fd = self._pipe(fd_pipes)
can_log = self._can_log(slave_fd)
if can_log:
log_file_path = self.logfile
else:
log_file_path = None
null_input = None
if not self.background or 0 in fd_pipes:
# Subclasses such as AbstractEbuildProcess may have already passed
# in a null file descriptor in fd_pipes, so use that when given.
pass
else:
# TODO: Use job control functions like tcsetpgrp() to control
# access to stdin. Until then, use /dev/null so that any
# attempts to read from stdin will immediately return EOF
# instead of blocking indefinitely.
null_input = os.open('/dev/null', os.O_RDWR)
fd_pipes[0] = null_input
fd_pipes.setdefault(0, portage._get_stdin().fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stderr__.fileno())
# flush any pending output
stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
for fd in fd_pipes.values():
if fd in stdout_filenos:
sys.__stdout__.flush()
sys.__stderr__.flush()
break
fd_pipes_orig = fd_pipes.copy()
if log_file_path is not None or self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
else:
# Create a dummy pipe that PipeLogger uses to efficiently
# monitor for process exit by listening for the EOF event.
# Re-use of the allocated fd number for the key in fd_pipes
# guarantees that the keys will not collide for similarly
# allocated pipes which are used by callers such as
# FileDigester and MergeProcess. See the _setup_pipes
# docstring for more benefits of this allocation approach.
self._dummy_pipe_fd = slave_fd
fd_pipes[slave_fd] = slave_fd
kwargs = {}
for k in self._spawn_kwarg_names:
v = getattr(self, k)
if v is not None:
kwargs[k] = v
kwargs["fd_pipes"] = fd_pipes
kwargs["returnpid"] = True
kwargs.pop("logfile", None)
retval = self._spawn(self.args, **kwargs)
os.close(slave_fd)
if null_input is not None:
os.close(null_input)
if isinstance(retval, int):
# spawn failed
self._unregister()
self._set_returncode((self.pid, retval))
self._async_wait()
return
self.pid = retval[0]
stdout_fd = None
if can_log and not self.background:
stdout_fd = os.dup(fd_pipes_orig[1])
# FD_CLOEXEC is enabled by default in Python >=3.4.
if sys.hexversion < 0x3040000 and fcntl is not None:
try:
fcntl.FD_CLOEXEC
except AttributeError:
pass
else:
fcntl.fcntl(stdout_fd, fcntl.F_SETFD,
fcntl.fcntl(stdout_fd,
fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
self._pipe_logger = PipeLogger(background=self.background,
scheduler=self.scheduler, input_fd=master_fd,
log_file_path=log_file_path,
stdout_fd=stdout_fd)
self._pipe_logger.addExitListener(self._pipe_logger_exit)
self._pipe_logger.start()
self._registered = True
class SpawnProcess(SubProcess):
"""
Constructor keyword args are passed into portage.process.spawn().
The required "args" keyword argument will be passed as the first
spawn() argument.
"""
_spawn_kwarg_names = ("env", "opt_name", "fd_pipes",
"uid", "gid", "groups", "umask", "logfile",
"path_lookup", "pre_exec", "close_fds", "cgroup",
"unshare_ipc", "unshare_net")
__slots__ = ("args",) + \
_spawn_kwarg_names + ("_pipe_logger", "_selinux_type",)
def _start(self):
if self.fd_pipes is None:
self.fd_pipes = {}
else:
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
master_fd, slave_fd = self._pipe(fd_pipes)
can_log = self._can_log(slave_fd)
if can_log:
log_file_path = self.logfile
else:
log_file_path = None
null_input = None
if not self.background or 0 in fd_pipes:
# Subclasses such as AbstractEbuildProcess may have already passed
# in a null file descriptor in fd_pipes, so use that when given.
pass
else:
# TODO: Use job control functions like tcsetpgrp() to control
# access to stdin. Until then, use /dev/null so that any
# attempts to read from stdin will immediately return EOF
# instead of blocking indefinitely.
null_input = os.open('/dev/null', os.O_RDWR)
fd_pipes[0] = null_input
fd_pipes.setdefault(0, portage._get_stdin().fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stderr__.fileno())
# flush any pending output
stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
for fd in fd_pipes.values():
if fd in stdout_filenos:
sys.__stdout__.flush()
sys.__stderr__.flush()
break
fd_pipes_orig = fd_pipes.copy()
if log_file_path is not None or self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
else:
# Create a dummy pipe that PipeLogger uses to efficiently
# monitor for process exit by listening for the EOF event.
# Re-use of the allocated fd number for the key in fd_pipes
# guarantees that the keys will not collide for similarly
# allocated pipes which are used by callers such as
# FileDigester and MergeProcess. See the _setup_pipes
# docstring for more benefits of this allocation approach.
self._dummy_pipe_fd = slave_fd
fd_pipes[slave_fd] = slave_fd
kwargs = {}
for k in self._spawn_kwarg_names:
v = getattr(self, k)
if v is not None:
kwargs[k] = v
kwargs["fd_pipes"] = fd_pipes
kwargs["returnpid"] = True
kwargs.pop("logfile", None)
retval = self._spawn(self.args, **kwargs)
os.close(slave_fd)
if null_input is not None:
os.close(null_input)
if isinstance(retval, int):
# spawn failed
self._unregister()
self._set_returncode((self.pid, retval))
self._async_wait()
return
self.pid = retval[0]
stdout_fd = None
if can_log and not self.background:
stdout_fd = os.dup(fd_pipes_orig[1])
# FD_CLOEXEC is enabled by default in Python >=3.4.
if sys.hexversion < 0x3040000 and fcntl is not None:
try:
fcntl.FD_CLOEXEC
except AttributeError:
pass
else:
fcntl.fcntl(stdout_fd, fcntl.F_SETFD,
fcntl.fcntl(stdout_fd,
fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
self._pipe_logger = PipeLogger(background=self.background,
scheduler=self.scheduler, input_fd=master_fd,
log_file_path=log_file_path,
stdout_fd=stdout_fd)
self._pipe_logger.addExitListener(self._pipe_logger_exit)
self._pipe_logger.start()
self._registered = True
def _can_log(self, slave_fd):
return True
def _pipe(self, fd_pipes):
"""
@type fd_pipes: dict
@param fd_pipes: pipes from which to copy terminal size if desired.
"""
return os.pipe()
def _spawn(self, args, **kwargs):
spawn_func = portage.process.spawn
if self._selinux_type is not None:
spawn_func = portage.selinux.spawn_wrapper(spawn_func,
self._selinux_type)
# bash is an allowed entrypoint, while most binaries are not
if args[0] != BASH_BINARY:
args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
return spawn_func(args, **kwargs)
def _pipe_logger_exit(self, pipe_logger):
self._pipe_logger = None
self._unregister()
self.wait()
def _waitpid_loop(self):
SubProcess._waitpid_loop(self)
pipe_logger = self._pipe_logger
if pipe_logger is not None:
self._pipe_logger = None
pipe_logger.removeExitListener(self._pipe_logger_exit)
pipe_logger.cancel()
pipe_logger.wait()
def _set_returncode(self, wait_retval):
SubProcess._set_returncode(self, wait_retval)
if self.cgroup:
def get_pids(cgroup):
try:
with open(os.path.join(cgroup, 'cgroup.procs'), 'r') as f:
return [int(p) for p in f.read().split()]
except OSError:
# cgroup removed already?
return []
def kill_all(pids, sig):
for p in pids:
try:
os.kill(p, sig)
except OSError as e:
if e.errno == errno.EPERM:
# Reported with hardened kernel (bug #358211).
writemsg_level(
"!!! kill: (%i) - Operation not permitted\n" %
(p,), level=logging.ERROR,
noiselevel=-1)
elif e.errno != errno.ESRCH:
raise
# step 1: kill all orphans
pids = get_pids(self.cgroup)
if pids:
kill_all(pids, signal.SIGKILL)
# step 2: remove the cgroup
try:
os.rmdir(self.cgroup)
except OSError:
# it may be removed already, or busy
# we can't do anything good about it
pass
class SpawnProcess(SubProcess):
"""
Constructor keyword args are passed into portage.process.spawn().
The required "args" keyword argument will be passed as the first
spawn() argument.
"""
_spawn_kwarg_names = ("env", "opt_name", "fd_pipes", "uid", "gid",
"groups", "umask", "logfile", "path_lookup",
"pre_exec", "close_fds", "cgroup", "unshare_ipc",
"unshare_mount", "unshare_pid", "unshare_net")
__slots__ = ("args",) + \
_spawn_kwarg_names + ("_pipe_logger", "_selinux_type",)
# Max number of attempts to kill the processes listed in cgroup.procs,
# given that processes may fork before they can be killed.
_CGROUP_CLEANUP_RETRY_MAX = 8
def _start(self):
if self.fd_pipes is None:
self.fd_pipes = {}
else:
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
master_fd, slave_fd = self._pipe(fd_pipes)
can_log = self._can_log(slave_fd)
if can_log:
log_file_path = self.logfile
else:
log_file_path = None
null_input = None
if not self.background or 0 in fd_pipes:
# Subclasses such as AbstractEbuildProcess may have already passed
# in a null file descriptor in fd_pipes, so use that when given.
pass
else:
# TODO: Use job control functions like tcsetpgrp() to control
# access to stdin. Until then, use /dev/null so that any
# attempts to read from stdin will immediately return EOF
# instead of blocking indefinitely.
null_input = os.open('/dev/null', os.O_RDWR)
fd_pipes[0] = null_input
fd_pipes.setdefault(0, portage._get_stdin().fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stderr__.fileno())
# flush any pending output
stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
for fd in fd_pipes.values():
if fd in stdout_filenos:
sys.__stdout__.flush()
sys.__stderr__.flush()
break
fd_pipes_orig = fd_pipes.copy()
if log_file_path is not None or self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
else:
# Create a dummy pipe that PipeLogger uses to efficiently
# monitor for process exit by listening for the EOF event.
# Re-use of the allocated fd number for the key in fd_pipes
# guarantees that the keys will not collide for similarly
# allocated pipes which are used by callers such as
# FileDigester and MergeProcess. See the _setup_pipes
# docstring for more benefits of this allocation approach.
self._dummy_pipe_fd = slave_fd
fd_pipes[slave_fd] = slave_fd
kwargs = {}
for k in self._spawn_kwarg_names:
v = getattr(self, k)
if v is not None:
kwargs[k] = v
kwargs["fd_pipes"] = fd_pipes
kwargs["returnpid"] = True
kwargs.pop("logfile", None)
retval = self._spawn(self.args, **kwargs)
os.close(slave_fd)
if null_input is not None:
os.close(null_input)
if isinstance(retval, int):
# spawn failed
self.returncode = retval
self._async_wait()
return
self.pid = retval[0]
stdout_fd = None
if can_log and not self.background:
stdout_fd = os.dup(fd_pipes_orig[1])
# FD_CLOEXEC is enabled by default in Python >=3.4.
if sys.hexversion < 0x3040000 and fcntl is not None:
try:
fcntl.FD_CLOEXEC
except AttributeError:
pass
else:
fcntl.fcntl(
stdout_fd, fcntl.F_SETFD,
fcntl.fcntl(stdout_fd, fcntl.F_GETFD)
| fcntl.FD_CLOEXEC)
self._pipe_logger = PipeLogger(background=self.background,
scheduler=self.scheduler,
input_fd=master_fd,
log_file_path=log_file_path,
stdout_fd=stdout_fd)
self._pipe_logger.addExitListener(self._pipe_logger_exit)
self._pipe_logger.start()
self._registered = True
def _can_log(self, slave_fd):
return True
def _pipe(self, fd_pipes):
"""
@type fd_pipes: dict
@param fd_pipes: pipes from which to copy terminal size if desired.
"""
return os.pipe()
def _spawn(self, args, **kwargs):
spawn_func = portage.process.spawn
if self._selinux_type is not None:
spawn_func = portage.selinux.spawn_wrapper(spawn_func,
self._selinux_type)
# bash is an allowed entrypoint, while most binaries are not
if args[0] != BASH_BINARY:
args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
return spawn_func(args, **kwargs)
def _pipe_logger_exit(self, pipe_logger):
self._pipe_logger = None
self._async_waitpid()
def _unregister(self):
SubProcess._unregister(self)
if self.cgroup is not None:
self._cgroup_cleanup()
self.cgroup = None
if self._pipe_logger is not None:
self._pipe_logger.cancel()
self._pipe_logger = None
def _cancel(self):
SubProcess._cancel(self)
self._cgroup_cleanup()
def _cgroup_cleanup(self):
if self.cgroup:
def get_pids(cgroup):
try:
with open(os.path.join(cgroup, 'cgroup.procs'), 'r') as f:
return [int(p) for p in f.read().split()]
except EnvironmentError:
# removed by cgroup-release-agent
return []
def kill_all(pids, sig):
for p in pids:
try:
os.kill(p, sig)
except OSError as e:
if e.errno == errno.EPERM:
# Reported with hardened kernel (bug #358211).
writemsg_level(
"!!! kill: (%i) - Operation not permitted\n" %
(p, ),
level=logging.ERROR,
noiselevel=-1)
elif e.errno != errno.ESRCH:
raise
# step 1: kill all orphans (loop in case of new forks)
remaining = self._CGROUP_CLEANUP_RETRY_MAX
while remaining:
remaining -= 1
pids = get_pids(self.cgroup)
if pids:
kill_all(pids, signal.SIGKILL)
else:
break
if pids:
msg = []
msg.append(
_("Failed to kill pid(s) in '%(cgroup)s': %(pids)s") %
dict(cgroup=os.path.join(self.cgroup, 'cgroup.procs'),
pids=' '.join(str(pid) for pid in pids)))
self._elog('eerror', msg)
# step 2: remove the cgroup
try:
os.rmdir(self.cgroup)
except OSError:
# it may be removed already, or busy
# we can't do anything good about it
pass
def _elog(self, elog_funcname, lines):
elog_func = getattr(EOutput(), elog_funcname)
for line in lines:
elog_func(line)