def _testLockNonblock(self):
tempdir = tempfile.mkdtemp()
try:
path = os.path.join(tempdir, 'lock_me')
lock1 = portage.locks.lockfile(path)
pid = os.fork()
if pid == 0:
portage.locks._close_fds()
# Disable close_fds since we don't exec
# (see _setup_pipes docstring).
portage.process._setup_pipes({0:0, 1:1, 2:2}, close_fds=False)
rval = 2
try:
try:
lock2 = portage.locks.lockfile(path, flags=os.O_NONBLOCK)
except portage.exception.TryAgain:
rval = os.EX_OK
else:
rval = 1
portage.locks.unlockfile(lock2)
except SystemExit:
raise
except:
traceback.print_exc()
finally:
os._exit(rval)
self.assertEqual(pid > 0, True)
pid, status = os.waitpid(pid, 0)
self.assertEqual(os.WIFEXITED(status), True)
self.assertEqual(os.WEXITSTATUS(status), os.EX_OK)
portage.locks.unlockfile(lock1)
finally:
shutil.rmtree(tempdir)
def _testLockNonblock(self):
tempdir = tempfile.mkdtemp()
try:
path = os.path.join(tempdir, 'lock_me')
lock1 = portage.locks.lockfile(path)
pid = os.fork()
if pid == 0:
portage.locks._close_fds()
# Disable close_fds since we don't exec
# (see _setup_pipes docstring).
portage.process._setup_pipes({0:0, 1:1, 2:2}, close_fds=False)
rval = 2
try:
try:
lock2 = portage.locks.lockfile(path, flags=os.O_NONBLOCK)
except portage.exception.TryAgain:
rval = os.EX_OK
else:
rval = 1
portage.locks.unlockfile(lock2)
except SystemExit:
raise
except:
traceback.print_exc()
finally:
os._exit(rval)
self.assertEqual(pid > 0, True)
pid, status = os.waitpid(pid, 0)
self.assertEqual(os.WIFEXITED(status), True)
self.assertEqual(os.WEXITSTATUS(status), os.EX_OK)
portage.locks.unlockfile(lock1)
finally:
shutil.rmtree(tempdir)
def _create_pty_or_pipe(copy_term_size=None):
"""
Try to create a pty and if then fails then create a normal
pipe instead.
@param copy_term_size: If a tty file descriptor is given
then the term size will be copied to the pty.
@type copy_term_size: int
@rtype: tuple
@return: A tuple of (is_pty, master_fd, slave_fd) where
is_pty is True if a pty was successfully allocated, and
False if a normal pipe was allocated.
"""
got_pty = False
global _disable_openpty, _fbsd_test_pty
if _fbsd_test_pty and not _disable_openpty:
# Test for python openpty breakage after freebsd7 to freebsd8
# upgrade, which results in a 'Function not implemented' error
# and the process being killed.
pid = os.fork()
if pid == 0:
pty.openpty()
os._exit(os.EX_OK)
pid, status = os.waitpid(pid, 0)
if (status & 0xff) == 140:
_disable_openpty = True
_fbsd_test_pty = False
if _disable_openpty:
master_fd, slave_fd = os.pipe()
else:
try:
master_fd, slave_fd = pty.openpty()
got_pty = True
except EnvironmentError as e:
_disable_openpty = True
writemsg("openpty failed: '%s'\n" % str(e),
noiselevel=-1)
del e
master_fd, slave_fd = os.pipe()
if got_pty:
# Disable post-processing of output since otherwise weird
# things like \n -> \r\n transformations may occur.
mode = termios.tcgetattr(slave_fd)
mode[1] &= ~termios.OPOST
termios.tcsetattr(slave_fd, termios.TCSANOW, mode)
if got_pty and \
copy_term_size is not None and \
os.isatty(copy_term_size):
rows, columns = get_term_size()
set_term_size(rows, columns, slave_fd)
return (got_pty, master_fd, slave_fd)
def _create_pty_or_pipe(copy_term_size=None):
"""
Try to create a pty and if then fails then create a normal
pipe instead.
@param copy_term_size: If a tty file descriptor is given
then the term size will be copied to the pty.
@type copy_term_size: int
@rtype: tuple
@returns: A tuple of (is_pty, master_fd, slave_fd) where
is_pty is True if a pty was successfully allocated, and
False if a normal pipe was allocated.
"""
got_pty = False
global _disable_openpty, _fbsd_test_pty
if _fbsd_test_pty and not _disable_openpty:
# Test for python openpty breakage after freebsd7 to freebsd8
# upgrade, which results in a 'Function not implemented' error
# and the process being killed.
pid = os.fork()
if pid == 0:
pty.openpty()
os._exit(os.EX_OK)
pid, status = os.waitpid(pid, 0)
if (status & 0xff) == 140:
_disable_openpty = True
_fbsd_test_pty = False
if _disable_openpty:
master_fd, slave_fd = os.pipe()
else:
try:
master_fd, slave_fd = pty.openpty()
got_pty = True
except EnvironmentError as e:
_disable_openpty = True
writemsg("openpty failed: '%s'\n" % str(e),
noiselevel=-1)
del e
master_fd, slave_fd = os.pipe()
if got_pty:
# Disable post-processing of output since otherwise weird
# things like \n -> \r\n transformations may occur.
mode = termios.tcgetattr(slave_fd)
mode[1] &= ~termios.OPOST
termios.tcsetattr(slave_fd, termios.TCSANOW, mode)
if got_pty and \
copy_term_size is not None and \
os.isatty(copy_term_size):
rows, columns = get_term_size()
set_term_size(rows, columns, slave_fd)
return (got_pty, master_fd, slave_fd)
def _spawn(self, args, fd_pipes=None, **kwargs): """ Fork a subprocess, apply local settings, and call fetch(). """ parent_pid = os.getpid() pid = None try: pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError( "fork returned non-integer: %s" % (repr(pid),)) return [pid] rval = 1 try: # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Unregister SIGCHLD handler and wakeup_fd for the parent # process's event loop (bug 655656). signal.signal(signal.SIGCHLD, signal.SIG_DFL) try: wakeup_fd = signal.set_wakeup_fd(-1) if wakeup_fd > 0: os.close(wakeup_fd) except (ValueError, OSError): pass portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) rval = self._run() except SystemExit: raise except: traceback.print_exc() # os._exit() skips stderr flush! sys.stderr.flush() finally: os._exit(rval) finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1)
def _spawn(self, args, fd_pipes=None, **kwargs):
"""
Fork a subprocess, apply local settings, and call fetch().
"""
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError(
"fork returned non-integer: %s" % (repr(pid), ))
portage.process.spawned_pids.append(pid)
return [pid]
# TODO: Find out why PyPy 1.8 with close_fds=True triggers
# "[Errno 9] Bad file descriptor" in subprocesses. It could
# be due to garbage collection of file objects that were not
# closed before going out of scope, since PyPy's garbage
# collector does not support the refcounting semantics that
# CPython does.
close_fds = platform.python_implementation() != 'PyPy'
portage.process._setup_pipes(fd_pipes, close_fds=close_fds)
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Force consistent color output, in case we are capturing fetch
# output through a normal pipe due to unavailability of ptys.
portage.output.havecolor = self._settings.get('NOCOLOR') \
not in ('yes', 'true')
rval = 1
allow_missing = self._get_manifest().allow_missing
try:
if fetch(
self._uri_map,
self._settings,
fetchonly=self.fetchonly,
digests=copy.deepcopy(self._get_digests()),
allow_missing_digests=allow_missing):
rval = os.EX_OK
except SystemExit:
raise
except:
traceback.print_exc()
finally:
# Call os._exit() from finally block, in order to suppress any
# finally blocks from earlier in the call stack. See bug #345289.
os._exit(rval)
def _spawn(self, args, fd_pipes=None, **kwargs):
"""
Fork a subprocess, apply local settings, and call fetch().
"""
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" %
(repr(pid), ))
portage.process.spawned_pids.append(pid)
return [pid]
# TODO: Find out why PyPy 1.8 with close_fds=True triggers
# "[Errno 9] Bad file descriptor" in subprocesses. It could
# be due to garbage collection of file objects that were not
# closed before going out of scope, since PyPy's garbage
# collector does not support the refcounting semantics that
# CPython does.
close_fds = platform.python_implementation() != 'PyPy'
portage.process._setup_pipes(fd_pipes, close_fds=close_fds)
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Force consistent color output, in case we are capturing fetch
# output through a normal pipe due to unavailability of ptys.
portage.output.havecolor = self._settings.get('NOCOLOR') \
not in ('yes', 'true')
rval = 1
allow_missing = self._get_manifest().allow_missing
try:
if fetch(self._uri_map,
self._settings,
fetchonly=self.fetchonly,
digests=copy.deepcopy(self._get_digests()),
allow_missing_digests=allow_missing):
rval = os.EX_OK
except SystemExit:
raise
except:
traceback.print_exc()
finally:
# Call os._exit() from finally block, in order to suppress any
# finally blocks from earlier in the call stack. See bug #345289.
os._exit(rval)
def _spawn(self, args, fd_pipes=None, **kwargs): """ Fork a subprocess, apply local settings, and call fetch(). """ parent_pid = os.getpid() pid = None try: pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError( "fork returned non-integer: %s" % (repr(pid),)) portage.process.spawned_pids.append(pid) return [pid] rval = 1 try: # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) rval = self._run() except SystemExit: raise except: traceback.print_exc() finally: os._exit(rval) finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1)
def _spawn(self, args, fd_pipes=None, **kwargs):
"""
Fork a subprocess, apply local settings, and call fetch().
"""
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" %
(repr(pid), ))
portage.process.spawned_pids.append(pid)
return [pid]
portage.locks._close_fds()
# Disable close_fds since we don't exec (see _setup_pipes docstring).
portage.process._setup_pipes(fd_pipes, close_fds=False)
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Force consistent color output, in case we are capturing fetch
# output through a normal pipe due to unavailability of ptys.
portage.output.havecolor = self._settings.get('NOCOLOR') \
not in ('yes', 'true')
rval = 1
allow_missing = self._get_manifest().allow_missing
try:
if fetch(self._uri_map,
self._settings,
fetchonly=self.fetchonly,
digests=copy.deepcopy(self._get_digests()),
allow_missing_digests=allow_missing):
rval = os.EX_OK
except SystemExit:
raise
except:
traceback.print_exc()
finally:
# Call os._exit() from finally block, in order to suppress any
# finally blocks from earlier in the call stack. See bug #345289.
os._exit(rval)
def _spawn(self, args, fd_pipes=None, **kwargs):
"""
Fork a subprocess, apply local settings, and call fetch().
"""
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError(
"fork returned non-integer: %s" % (repr(pid),))
portage.process.spawned_pids.append(pid)
return [pid]
portage.locks._close_fds()
# Disable close_fds since we don't exec (see _setup_pipes docstring).
portage.process._setup_pipes(fd_pipes, close_fds=False)
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Force consistent color output, in case we are capturing fetch
# output through a normal pipe due to unavailability of ptys.
portage.output.havecolor = self._settings.get('NOCOLOR') \
not in ('yes', 'true')
rval = 1
allow_missing = self._get_manifest().allow_missing
try:
if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly,
digests=copy.deepcopy(self._get_digests()),
allow_missing_digests=allow_missing):
rval = os.EX_OK
except SystemExit:
raise
except:
traceback.print_exc()
finally:
# Call os._exit() from finally block, in order to suppress any
# finally blocks from earlier in the call stack. See bug #345289.
os._exit(rval)
def __init__(self, environment):
self.conf = environment
pidfile = open(self.conf.get('pid_file'), 'w')
if (self.conf.get('no_daemon') == False):
print "forking to background"
if (os.fork() == 0):
os.setpgid(0,0);
pidfile.write(str(os.getpid()));
pidfile.close();
fd = os.open("/dev/null", os.O_WRONLY);
os.dup2(fd,1);
os.close(fd);
#self.main_loop()
else:
sys.exit()
else:
print "Keeping in foreground"
pidfile.write(str(os.getpid()));
pidfile.close();
def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
uid=None, gid=None, groups=None, umask=None, logfile=None,
path_lookup=True, pre_exec=None):
"""
Spawns a given command.
@param mycommand: the command to execute
@type mycommand: String or List (Popen style list)
@param env: A dict of Key=Value pairs for env variables
@type env: Dictionary
@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
@type opt_name: String
@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
@type fd_pipes: Dictionary
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
@type returnpid: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@param gid: Group ID to spawn as; useful for dropping privilages
@type gid: Integer
@param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts.
@type groups: List
@param umask: An integer representing the umask for the process (see man chmod for umask details)
@type umask: Integer
@param logfile: name of a file to use for logging purposes
@type logfile: String
@param path_lookup: If the binary is not fully specified then look for it in PATH
@type path_lookup: Boolean
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
logfile requires stdout and stderr to be assigned to this process (ie not pointed
somewhere else.)
"""
# mycommand is either a str or a list
if isinstance(mycommand, basestring):
mycommand = mycommand.split()
if sys.hexversion < 0x3000000:
# Avoid a potential UnicodeEncodeError from os.execve().
env_bytes = {}
for k, v in env.items():
env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \
_unicode_encode(v, encoding=_encodings['content'])
env = env_bytes
del env_bytes
# If an absolute path to an executable file isn't given
# search for it unless we've been told not to.
binary = mycommand[0]
if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \
(not os.path.isabs(binary) or not os.path.isfile(binary)
or not os.access(binary, os.X_OK)):
binary = path_lookup and find_binary(binary) or None
if not binary:
raise CommandNotFound(mycommand[0])
# If we haven't been told what file descriptors to use
# default to propagating our stdin, stdout and stderr.
if fd_pipes is None:
fd_pipes = {
0:sys.__stdin__.fileno(),
1:sys.__stdout__.fileno(),
2:sys.__stderr__.fileno(),
}
# mypids will hold the pids of all processes created.
mypids = []
if logfile:
# Using a log file requires that stdout and stderr
# are assigned to the process we're running.
if 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
# Create a pipe
(pr, pw) = os.pipe()
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
mypids.extend(spawn(('tee', '-i', '-a', logfile),
returnpid=True, fd_pipes={0:pr,
1:fd_pipes[1], 2:fd_pipes[2]}))
# We don't need the read end of the pipe, so close it.
os.close(pr)
# Assign the write end of the pipe to our stdout and stderr.
fd_pipes[1] = pw
fd_pipes[2] = pw
pid = os.fork()
if pid == 0:
try:
_exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, pre_exec)
except SystemExit:
raise
except Exception as e:
# We need to catch _any_ exception so that it doesn't
# propagate out of this function and cause exiting
# with anything other than os._exit()
sys.stderr.write("%s:\n %s\n" % (e, " ".join(mycommand)))
traceback.print_exc()
sys.stderr.flush()
os._exit(1)
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" % (repr(pid),))
# Add the pid to our local and the global pid lists.
mypids.append(pid)
spawned_pids.append(pid)
# If we started a tee process the write side of the pipe is no
# longer needed, so close it.
if logfile:
os.close(pw)
# If the caller wants to handle cleaning up the processes, we tell
# it about all processes that were created.
if returnpid:
return mypids
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
# in the pipe are well behaved, it will die when the process
# it is reading from dies.
pid = mypids.pop(0)
# and wait for it.
retval = os.waitpid(pid, 0)[1]
# When it's done, we can remove it from the
# global pid list as well.
spawned_pids.remove(pid)
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if os.waitpid(pid, os.WNOHANG)[0] == 0:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
spawned_pids.remove(pid)
# If it got a signal, return the signal that was sent.
if (retval & 0xff):
return ((retval & 0xff) << 8)
# Otherwise, return its exit code.
return (retval >> 8)
# Everything succeeded
return 0
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
dblink.merge(). TODO: Share code with ForkProcess.
"""
elog_reader_fd, elog_writer_fd = os.pipe()
fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
# of file descriptors in the subprocess can prevent
# access to open database connections such as that
# used by the sqlite metadata cache module.
blockers = self.blockers()
mylink = portage.dblink(self.mycat,
self.mypkg,
settings=self.settings,
treetype=self.treetype,
vartree=self.vartree,
blockers=blockers,
pipe=elog_writer_fd)
fd_pipes[elog_writer_fd] = elog_writer_fd
self._elog_reg_id = self.scheduler.io_add_watch(
elog_reader_fd, self._registered_events, self._elog_output_handler)
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
# extremely unlikely chance that the COUNTER values will not be
# ordered correctly unless we lock the vdb here.
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
counter = None
if not self.unmerge:
counter = self.vartree.dbapi.counter_tick()
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" %
(repr(pid), ))
os.close(elog_writer_fd)
self._elog_reader_fd = elog_reader_fd
self._buf = ""
self._elog_keys = set()
# invalidate relevant vardbapi caches
if self.vartree.dbapi._categories is not None:
self.vartree.dbapi._categories = None
self.vartree.dbapi._pkgs_changed = True
self.vartree.dbapi._clear_pkg_cache(mylink)
portage.process.spawned_pids.append(pid)
return [pid]
os.close(elog_reader_fd)
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
portage.locks._close_fds()
# We don't exec, so use close_fds=False
# (see _setup_pipes docstring).
portage.process._setup_pipes(fd_pipes, close_fds=False)
portage.output.havecolor = self.settings.get('NOCOLOR') \
not in ('yes', 'true')
# Avoid wastful updates of the vdb cache.
self.vartree.dbapi._flush_cache_enabled = False
# In this subprocess we don't want PORTAGE_BACKGROUND to
# suppress stdout/stderr output since they are pipes. We
# also don't want to open PORTAGE_LOG_FILE, since it will
# already be opened by the parent process, so we set the
# "subprocess" value for use in conditional logging code
# involving PORTAGE_LOG_FILE.
if not self.unmerge:
# unmerge phases have separate logs
if self.settings.get("PORTAGE_BACKGROUND") == "1":
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
else:
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
self.settings["PORTAGE_BACKGROUND"] = "subprocess"
self.settings.backup_changes("PORTAGE_BACKGROUND")
rval = 1
try:
if self.unmerge:
if not mylink.exists():
rval = os.EX_OK
elif mylink.unmerge(
ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
mylink.lockdb()
try:
mylink.delete()
finally:
mylink.unlockdb()
rval = os.EX_OK
else:
rval = mylink.merge(self.pkgloc,
self.infloc,
myebuild=self.myebuild,
mydbapi=self.mydbapi,
prev_mtimes=self.prev_mtimes,
counter=counter)
except SystemExit:
raise
except:
traceback.print_exc()
finally:
os._exit(rval)
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
dblink.merge(). TODO: Share code with ForkProcess.
"""
elog_reader_fd, elog_writer_fd = os.pipe()
fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
# of file descriptors in the subprocess can prevent
# access to open database connections such as that
# used by the sqlite metadata cache module.
blockers = self.blockers()
mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings,
treetype=self.treetype, vartree=self.vartree,
blockers=blockers, pipe=elog_writer_fd)
fd_pipes[elog_writer_fd] = elog_writer_fd
self._elog_reg_id = self.scheduler.io_add_watch(elog_reader_fd,
self._registered_events, self._elog_output_handler)
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
# extremely unlikely chance that the COUNTER values will not be
# ordered correctly unless we lock the vdb here.
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
counter = None
if not self.unmerge:
counter = self.vartree.dbapi.counter_tick()
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError(
"fork returned non-integer: %s" % (repr(pid),))
os.close(elog_writer_fd)
self._elog_reader_fd = elog_reader_fd
self._buf = ""
self._elog_keys = set()
# invalidate relevant vardbapi caches
if self.vartree.dbapi._categories is not None:
self.vartree.dbapi._categories = None
self.vartree.dbapi._pkgs_changed = True
self.vartree.dbapi._clear_pkg_cache(mylink)
portage.process.spawned_pids.append(pid)
return [pid]
os.close(elog_reader_fd)
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
portage.locks._close_fds()
# We don't exec, so use close_fds=False
# (see _setup_pipes docstring).
portage.process._setup_pipes(fd_pipes, close_fds=False)
portage.output.havecolor = self.settings.get('NOCOLOR') \
not in ('yes', 'true')
# Avoid wastful updates of the vdb cache.
self.vartree.dbapi._flush_cache_enabled = False
# In this subprocess we don't want PORTAGE_BACKGROUND to
# suppress stdout/stderr output since they are pipes. We
# also don't want to open PORTAGE_LOG_FILE, since it will
# already be opened by the parent process, so we set the
# "subprocess" value for use in conditional logging code
# involving PORTAGE_LOG_FILE.
if not self.unmerge:
# unmerge phases have separate logs
if self.settings.get("PORTAGE_BACKGROUND") == "1":
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
else:
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
self.settings["PORTAGE_BACKGROUND"] = "subprocess"
self.settings.backup_changes("PORTAGE_BACKGROUND")
rval = 1
try:
if self.unmerge:
if not mylink.exists():
rval = os.EX_OK
elif mylink.unmerge(
ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
mylink.lockdb()
try:
mylink.delete()
finally:
mylink.unlockdb()
rval = os.EX_OK
else:
rval = mylink.merge(self.pkgloc, self.infloc,
myebuild=self.myebuild, mydbapi=self.mydbapi,
prev_mtimes=self.prev_mtimes, counter=counter)
except SystemExit:
raise
except:
traceback.print_exc()
finally:
os._exit(rval)
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
def _exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, cwd,
pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid,
unshare_flags, cgroup):
"""
Execute a given binary with options
@param binary: Name of program to execute
@type binary: String
@param mycommand: Options for program
@type mycommand: String
@param opt_name: Name of process (defaults to binary)
@type opt_name: String
@param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 }
@type fd_pipes: Dictionary
@param env: Key,Value mapping for Environmental Variables
@type env: Dictionary
@param gid: Group ID to run the process under
@type gid: Integer
@param groups: Groups the Process should be in.
@type groups: List
@param uid: User ID to run the process under
@type uid: Integer
@param umask: an int representing a unix umask (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param unshare_flags: Flags for the unshare(2) function
@type unshare_flags: Integer
@param cgroup: CGroup path to bind the process to
@type cgroup: String
@rtype: None
@return: Never returns (calls os.execve)
"""
# If the process we're creating hasn't been given a name
# assign it the name of the executable.
if not opt_name:
if binary is portage._python_interpreter:
# NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0]
# does not contain the full path of the binary.
opt_name = binary
else:
opt_name = os.path.basename(binary)
# Set up the command's argument list.
myargs = [opt_name]
myargs.extend(mycommand[1:])
# Avoid a potential UnicodeEncodeError from os.execve().
myargs = [_unicode_encode(x, encoding=_encodings['fs'],
errors='strict') for x in myargs]
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Unregister SIGCHLD handler and wakeup_fd for the parent
# process's event loop (bug 655656).
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
try:
wakeup_fd = signal.set_wakeup_fd(-1)
if wakeup_fd > 0:
os.close(wakeup_fd)
except (ValueError, OSError):
pass
# Quiet killing of subprocesses by SIGPIPE (see bug #309001).
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# Avoid issues triggered by inheritance of SIGQUIT handler from
# the parent process (see bug #289486).
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
_setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
# Add to cgroup
# it's better to do it from the child since we can guarantee
# it is done before we start forking children
if cgroup:
with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
f.write('%d\n' % os.getpid())
# Unshare (while still uid==0)
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
try:
# Since a failed unshare call could corrupt process
# state, first validate that the call can succeed.
# The parent process should call _unshare_validate
# before it forks, so that all child processes can
# reuse _unshare_validate results that have been
# cached by the parent process.
errno_value = _unshare_validate(unshare_flags)
if errno_value == 0 and libc.unshare(unshare_flags) != 0:
errno_value = ctypes.get_errno()
if errno_value != 0:
involved_features = []
if unshare_ipc:
involved_features.append('ipc-sandbox')
if unshare_mount:
involved_features.append('mount-sandbox')
if unshare_net:
involved_features.append('network-sandbox')
if unshare_pid:
involved_features.append('pid-sandbox')
writemsg("Unable to unshare: %s (for FEATURES=\"%s\")\n" % (
errno.errorcode.get(errno_value, '?'), ' '.join(involved_features)),
noiselevel=-1)
else:
if unshare_pid:
main_child_pid = os.fork()
if main_child_pid == 0:
# pid namespace requires us to become init
binary, myargs = portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'),
_unicode_encode('' if uid is None else str(uid)),
_unicode_encode('' if gid is None else str(gid)),
_unicode_encode('' if groups is None else ','.join(str(group) for group in groups)),
_unicode_encode('' if umask is None else str(umask)),
_unicode_encode(','.join(str(fd) for fd in fd_pipes)),
binary] + myargs
uid = None
gid = None
groups = None
umask = None
else:
# Execute a supervisor process which will forward
# signals to init and forward exit status to the
# parent process. The supervisor process runs in
# the global pid namespace, so skip /proc remount
# and other setup that's intended only for the
# init process.
binary, myargs = portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'), str(main_child_pid)]
os.execve(binary, myargs, env)
if unshare_mount:
# mark the whole filesystem as slave to avoid
# mounts escaping the namespace
s = subprocess.Popen(['mount',
'--make-rslave', '/'])
mount_ret = s.wait()
if mount_ret != 0:
# TODO: should it be fatal maybe?
writemsg("Unable to mark mounts slave: %d\n" % (mount_ret,),
noiselevel=-1)
if unshare_pid:
# we need at least /proc being slave
s = subprocess.Popen(['mount',
'--make-slave', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
# can't proceed with shared /proc
writemsg("Unable to mark /proc slave: %d\n" % (mount_ret,),
noiselevel=-1)
os._exit(1)
# mount new /proc for our namespace
s = subprocess.Popen(['mount',
'-n', '-t', 'proc', 'proc', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
writemsg("Unable to mount new /proc: %d\n" % (mount_ret,),
noiselevel=-1)
os._exit(1)
if unshare_net:
# use 'localhost' to avoid hostname resolution problems
try:
socket.sethostname('localhost')
except Exception as e:
writemsg("Unable to set hostname: %s (for FEATURES=\"network-sandbox\")\n" % (
e,),
noiselevel=-1)
_configure_loopback_interface()
except AttributeError:
# unshare() not supported by libc
pass
# Set requested process permissions.
if gid:
# Cast proxies to int, in case it matters.
os.setgid(int(gid))
if groups:
os.setgroups(groups)
if uid:
# Cast proxies to int, in case it matters.
os.setuid(int(uid))
if umask:
os.umask(umask)
if cwd is not None:
os.chdir(cwd)
if pre_exec:
pre_exec()
# And switch to the new process.
os.execve(binary, myargs, env)
def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
uid=None, gid=None, groups=None, umask=None, cwd=None, logfile=None,
path_lookup=True, pre_exec=None,
close_fds=(sys.version_info < (3, 4)), unshare_net=False,
unshare_ipc=False, unshare_mount=False, unshare_pid=False,
cgroup=None):
"""
Spawns a given command.
@param mycommand: the command to execute
@type mycommand: String or List (Popen style list)
@param env: If env is not None, it must be a mapping that defines the environment
variables for the new process; these are used instead of the default behavior
of inheriting the current process's environment.
@type env: None or Mapping
@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
@type opt_name: String
@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
(default is {0:stdin, 1:stdout, 2:stderr})
@type fd_pipes: Dictionary
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
@type returnpid: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@param gid: Group ID to spawn as; useful for dropping privilages
@type gid: Integer
@param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts.
@type groups: List
@param umask: An integer representing the umask for the process (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param logfile: name of a file to use for logging purposes
@type logfile: String
@param path_lookup: If the binary is not fully specified then look for it in PATH
@type path_lookup: Boolean
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param close_fds: If True, then close all file descriptors except those
referenced by fd_pipes (default is True for python3.3 and earlier, and False for
python3.4 and later due to non-inheritable file descriptor behavior from PEP 446).
@type close_fds: Boolean
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
logfile requires stdout and stderr to be assigned to this process (ie not pointed
somewhere else.)
"""
# mycommand is either a str or a list
if isinstance(mycommand, basestring):
mycommand = mycommand.split()
env = os.environ if env is None else env
if sys.hexversion < 0x3000000:
# Avoid a potential UnicodeEncodeError from os.execve().
env_bytes = {}
for k, v in env.items():
env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \
_unicode_encode(v, encoding=_encodings['content'])
env = env_bytes
del env_bytes
# If an absolute path to an executable file isn't given
# search for it unless we've been told not to.
binary = mycommand[0]
if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \
(not os.path.isabs(binary) or not os.path.isfile(binary)
or not os.access(binary, os.X_OK)):
binary = path_lookup and find_binary(binary) or None
if not binary:
raise CommandNotFound(mycommand[0])
# If we haven't been told what file descriptors to use
# default to propagating our stdin, stdout and stderr.
if fd_pipes is None:
fd_pipes = {
0:portage._get_stdin().fileno(),
1:sys.__stdout__.fileno(),
2:sys.__stderr__.fileno(),
}
# mypids will hold the pids of all processes created.
mypids = []
if logfile:
# Using a log file requires that stdout and stderr
# are assigned to the process we're running.
if 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
# Create a pipe
(pr, pw) = os.pipe()
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
mypids.extend(spawn(('tee', '-i', '-a', logfile),
returnpid=True, fd_pipes={0:pr,
1:fd_pipes[1], 2:fd_pipes[2]}))
# We don't need the read end of the pipe, so close it.
os.close(pr)
# Assign the write end of the pipe to our stdout and stderr.
fd_pipes[1] = pw
fd_pipes[2] = pw
# Cache _has_ipv6() result for use in child processes.
_has_ipv6()
# This caches the libc library lookup and _unshare_validator results
# in the current process, so that results are cached for use in
# child processes.
unshare_flags = 0
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
# from /usr/include/bits/sched.h
CLONE_NEWNS = 0x00020000
CLONE_NEWUTS = 0x04000000
CLONE_NEWIPC = 0x08000000
CLONE_NEWPID = 0x20000000
CLONE_NEWNET = 0x40000000
if unshare_net:
# UTS namespace to override hostname
unshare_flags |= CLONE_NEWNET | CLONE_NEWUTS
if unshare_ipc:
unshare_flags |= CLONE_NEWIPC
if unshare_mount:
# NEWNS = mount namespace
unshare_flags |= CLONE_NEWNS
if unshare_pid:
# we also need mount namespace for slave /proc
unshare_flags |= CLONE_NEWPID | CLONE_NEWNS
_unshare_validate(unshare_flags)
# Force instantiation of portage.data.userpriv_groups before the
# fork, so that the result is cached in the main process.
bool(groups)
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid == 0:
try:
_exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, cwd, pre_exec, close_fds,
unshare_net, unshare_ipc, unshare_mount, unshare_pid,
unshare_flags, cgroup)
except SystemExit:
raise
except Exception as e:
# We need to catch _any_ exception so that it doesn't
# propagate out of this function and cause exiting
# with anything other than os._exit()
writemsg("%s:\n %s\n" % (e, " ".join(mycommand)),
noiselevel=-1)
traceback.print_exc()
sys.stderr.flush()
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" % (repr(pid),))
# Add the pid to our local and the global pid lists.
mypids.append(pid)
# If we started a tee process the write side of the pipe is no
# longer needed, so close it.
if logfile:
os.close(pw)
# If the caller wants to handle cleaning up the processes, we tell
# it about all processes that were created.
if returnpid:
return mypids
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
# in the pipe are well behaved, it will die when the process
# it is reading from dies.
pid = mypids.pop(0)
# and wait for it.
retval = os.waitpid(pid, 0)[1]
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if os.waitpid(pid, os.WNOHANG)[0] == 0:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
# If it got a signal, return the signal that was sent.
if (retval & 0xff):
return ((retval & 0xff) << 8)
# Otherwise, return its exit code.
return (retval >> 8)
# Everything succeeded
return 0
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount,
unshare_pid, cgroup):
"""
Execute a given binary with options
@param binary: Name of program to execute
@type binary: String
@param mycommand: Options for program
@type mycommand: String
@param opt_name: Name of process (defaults to binary)
@type opt_name: String
@param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 }
@type fd_pipes: Dictionary
@param env: Key,Value mapping for Environmental Variables
@type env: Dictionary
@param gid: Group ID to run the process under
@type gid: Integer
@param groups: Groups the Process should be in.
@type groups: Integer
@param uid: User ID to run the process under
@type uid: Integer
@param umask: an int representing a unix umask (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
@rtype: None
@return: Never returns (calls os.execve)
"""
# If the process we're creating hasn't been given a name
# assign it the name of the executable.
if not opt_name:
if binary is portage._python_interpreter:
# NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0]
# does not contain the full path of the binary.
opt_name = binary
else:
opt_name = os.path.basename(binary)
# Set up the command's argument list.
myargs = [opt_name]
myargs.extend(mycommand[1:])
# Avoid a potential UnicodeEncodeError from os.execve().
myargs = [
_unicode_encode(x, encoding=_encodings['fs'], errors='strict')
for x in myargs
]
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Unregister SIGCHLD handler and wakeup_fd for the parent
# process's event loop (bug 655656).
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
try:
wakeup_fd = signal.set_wakeup_fd(-1)
if wakeup_fd > 0:
os.close(wakeup_fd)
except (ValueError, OSError):
pass
# Quiet killing of subprocesses by SIGPIPE (see bug #309001).
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# Avoid issues triggered by inheritance of SIGQUIT handler from
# the parent process (see bug #289486).
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
_setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
# Add to cgroup
# it's better to do it from the child since we can guarantee
# it is done before we start forking children
if cgroup:
with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
f.write('%d\n' % os.getpid())
# Unshare (while still uid==0)
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
# from /usr/include/bits/sched.h
CLONE_NEWNS = 0x00020000
CLONE_NEWIPC = 0x08000000
CLONE_NEWPID = 0x20000000
CLONE_NEWNET = 0x40000000
flags = 0
if unshare_net:
flags |= CLONE_NEWNET
if unshare_ipc:
flags |= CLONE_NEWIPC
if unshare_mount:
# NEWNS = mount namespace
flags |= CLONE_NEWNS
if unshare_pid:
# we also need mount namespace for slave /proc
flags |= CLONE_NEWPID | CLONE_NEWNS
try:
if libc.unshare(flags) != 0:
writemsg(
"Unable to unshare: %s\n" %
(errno.errorcode.get(ctypes.get_errno(), '?')),
noiselevel=-1)
else:
if unshare_pid:
# pid namespace requires us to become init
fork_ret = os.fork()
if fork_ret != 0:
os.execv(portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'),
'%s' % fork_ret,
])
if unshare_mount:
# mark the whole filesystem as slave to avoid
# mounts escaping the namespace
s = subprocess.Popen(
['mount', '--make-rslave', '/'])
mount_ret = s.wait()
if mount_ret != 0:
# TODO: should it be fatal maybe?
writemsg("Unable to mark mounts slave: %d\n" %
(mount_ret, ),
noiselevel=-1)
if unshare_pid:
# we need at least /proc being slave
s = subprocess.Popen(
['mount', '--make-slave', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
# can't proceed with shared /proc
writemsg("Unable to mark /proc slave: %d\n" %
(mount_ret, ),
noiselevel=-1)
os._exit(1)
# mount new /proc for our namespace
s = subprocess.Popen(
['mount', '-t', 'proc', 'proc', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
writemsg("Unable to mount new /proc: %d\n" %
(mount_ret, ),
noiselevel=-1)
os._exit(1)
if unshare_net:
# 'up' the loopback
IFF_UP = 0x1
ifreq = struct.pack('16sh', b'lo', IFF_UP)
SIOCSIFFLAGS = 0x8914
sock = socket.socket(socket.AF_INET,
socket.SOCK_DGRAM, 0)
try:
fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq)
except IOError as e:
writemsg(
"Unable to enable loopback interface: %s\n"
% (errno.errorcode.get(e.errno, '?')),
noiselevel=-1)
sock.close()
except AttributeError:
# unshare() not supported by libc
pass
# Set requested process permissions.
if gid:
# Cast proxies to int, in case it matters.
os.setgid(int(gid))
if groups:
os.setgroups(groups)
if uid:
# Cast proxies to int, in case it matters.
os.setuid(int(uid))
if umask:
os.umask(umask)
if cwd is not None:
os.chdir(cwd)
if pre_exec:
pre_exec()
# And switch to the new process.
os.execve(binary, myargs, env)
def _exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, cwd,
pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid,
unshare_flags, cgroup):
"""
Execute a given binary with options
@param binary: Name of program to execute
@type binary: String
@param mycommand: Options for program
@type mycommand: String
@param opt_name: Name of process (defaults to binary)
@type opt_name: String
@param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 }
@type fd_pipes: Dictionary
@param env: Key,Value mapping for Environmental Variables
@type env: Dictionary
@param gid: Group ID to run the process under
@type gid: Integer
@param groups: Groups the Process should be in.
@type groups: List
@param uid: User ID to run the process under
@type uid: Integer
@param umask: an int representing a unix umask (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param unshare_flags: Flags for the unshare(2) function
@type unshare_flags: Integer
@param cgroup: CGroup path to bind the process to
@type cgroup: String
@rtype: None
@return: Never returns (calls os.execve)
"""
# If the process we're creating hasn't been given a name
# assign it the name of the executable.
if not opt_name:
if binary is portage._python_interpreter:
# NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0]
# does not contain the full path of the binary.
opt_name = binary
else:
opt_name = os.path.basename(binary)
# Set up the command's argument list.
myargs = [opt_name]
myargs.extend(mycommand[1:])
# Avoid a potential UnicodeEncodeError from os.execve().
myargs = [_unicode_encode(x, encoding=_encodings['fs'],
errors='strict') for x in myargs]
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Unregister SIGCHLD handler and wakeup_fd for the parent
# process's event loop (bug 655656).
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
try:
wakeup_fd = signal.set_wakeup_fd(-1)
if wakeup_fd > 0:
os.close(wakeup_fd)
except (ValueError, OSError):
pass
# Quiet killing of subprocesses by SIGPIPE (see bug #309001).
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# Avoid issues triggered by inheritance of SIGQUIT handler from
# the parent process (see bug #289486).
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
_setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
# Add to cgroup
# it's better to do it from the child since we can guarantee
# it is done before we start forking children
if cgroup:
with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
f.write('%d\n' % os.getpid())
# Unshare (while still uid==0)
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
try:
# Since a failed unshare call could corrupt process
# state, first validate that the call can succeed.
# The parent process should call _unshare_validate
# before it forks, so that all child processes can
# reuse _unshare_validate results that have been
# cached by the parent process.
errno_value = _unshare_validate(unshare_flags)
if errno_value == 0 and libc.unshare(unshare_flags) != 0:
errno_value = ctypes.get_errno()
if errno_value != 0:
writemsg("Unable to unshare: %s\n" % (
errno.errorcode.get(errno_value, '?')),
noiselevel=-1)
else:
if unshare_pid:
main_child_pid = os.fork()
if main_child_pid == 0:
# pid namespace requires us to become init
binary, myargs = portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'),
_unicode_encode('' if uid is None else str(uid)),
_unicode_encode('' if gid is None else str(gid)),
_unicode_encode('' if groups is None else ','.join(str(group) for group in groups)),
_unicode_encode('' if umask is None else str(umask)),
_unicode_encode(','.join(str(fd) for fd in fd_pipes)),
binary] + myargs
uid = None
gid = None
groups = None
umask = None
else:
# Execute a supervisor process which will forward
# signals to init and forward exit status to the
# parent process. The supervisor process runs in
# the global pid namespace, so skip /proc remount
# and other setup that's intended only for the
# init process.
binary, myargs = portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'), str(main_child_pid)]
os.execve(binary, myargs, env)
if unshare_mount:
# mark the whole filesystem as slave to avoid
# mounts escaping the namespace
s = subprocess.Popen(['mount',
'--make-rslave', '/'])
mount_ret = s.wait()
if mount_ret != 0:
# TODO: should it be fatal maybe?
writemsg("Unable to mark mounts slave: %d\n" % (mount_ret,),
noiselevel=-1)
if unshare_pid:
# we need at least /proc being slave
s = subprocess.Popen(['mount',
'--make-slave', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
# can't proceed with shared /proc
writemsg("Unable to mark /proc slave: %d\n" % (mount_ret,),
noiselevel=-1)
os._exit(1)
# mount new /proc for our namespace
s = subprocess.Popen(['mount',
'-n', '-t', 'proc', 'proc', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
writemsg("Unable to mount new /proc: %d\n" % (mount_ret,),
noiselevel=-1)
os._exit(1)
if unshare_net:
# 'up' the loopback
IFF_UP = 0x1
ifreq = struct.pack('16sh', b'lo', IFF_UP)
SIOCSIFFLAGS = 0x8914
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
try:
fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq)
except IOError as e:
writemsg("Unable to enable loopback interface: %s\n" % (
errno.errorcode.get(e.errno, '?')),
noiselevel=-1)
sock.close()
except AttributeError:
# unshare() not supported by libc
pass
# Set requested process permissions.
if gid:
# Cast proxies to int, in case it matters.
os.setgid(int(gid))
if groups:
os.setgroups(groups)
if uid:
# Cast proxies to int, in case it matters.
os.setuid(int(uid))
if umask:
os.umask(umask)
if cwd is not None:
os.chdir(cwd)
if pre_exec:
pre_exec()
# And switch to the new process.
os.execve(binary, myargs, env)
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid,
cgroup):
"""
Execute a given binary with options
@param binary: Name of program to execute
@type binary: String
@param mycommand: Options for program
@type mycommand: String
@param opt_name: Name of process (defaults to binary)
@type opt_name: String
@param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 }
@type fd_pipes: Dictionary
@param env: Key,Value mapping for Environmental Variables
@type env: Dictionary
@param gid: Group ID to run the process under
@type gid: Integer
@param groups: Groups the Process should be in.
@type groups: Integer
@param uid: User ID to run the process under
@type uid: Integer
@param umask: an int representing a unix umask (see man chmod for umask details)
@type umask: Integer
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
@rtype: None
@return: Never returns (calls os.execve)
"""
# If the process we're creating hasn't been given a name
# assign it the name of the executable.
if not opt_name:
if binary is portage._python_interpreter:
# NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0]
# does not contain the full path of the binary.
opt_name = binary
else:
opt_name = os.path.basename(binary)
# Set up the command's argument list.
myargs = [opt_name]
myargs.extend(mycommand[1:])
# Avoid a potential UnicodeEncodeError from os.execve().
myargs = [_unicode_encode(x, encoding=_encodings['fs'],
errors='strict') for x in myargs]
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Quiet killing of subprocesses by SIGPIPE (see bug #309001).
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# Avoid issues triggered by inheritance of SIGQUIT handler from
# the parent process (see bug #289486).
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
_setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
# Add to cgroup
# it's better to do it from the child since we can guarantee
# it is done before we start forking children
if cgroup:
with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
f.write('%d\n' % os.getpid())
# Unshare (while still uid==0)
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
# from /usr/include/bits/sched.h
CLONE_NEWNS = 0x00020000
CLONE_NEWIPC = 0x08000000
CLONE_NEWPID = 0x20000000
CLONE_NEWNET = 0x40000000
flags = 0
if unshare_net:
flags |= CLONE_NEWNET
if unshare_ipc:
flags |= CLONE_NEWIPC
if unshare_mount:
# NEWNS = mount namespace
flags |= CLONE_NEWNS
if unshare_pid:
# we also need mount namespace for slave /proc
flags |= CLONE_NEWPID | CLONE_NEWNS
try:
if libc.unshare(flags) != 0:
writemsg("Unable to unshare: %s\n" % (
errno.errorcode.get(ctypes.get_errno(), '?')),
noiselevel=-1)
else:
if unshare_pid:
# pid namespace requires us to become init
fork_ret = os.fork()
if fork_ret != 0:
os.execv(portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'),
'%s' % fork_ret,
])
if unshare_mount:
# mark the whole filesystem as private to avoid
# mounts escaping the namespace
s = subprocess.Popen(['mount',
'--make-rprivate', '/'])
mount_ret = s.wait()
if mount_ret != 0:
# TODO: should it be fatal maybe?
writemsg("Unable to mark mounts private: %d\n" % (mount_ret,),
noiselevel=-1)
if unshare_pid:
# we need at least /proc being slave
s = subprocess.Popen(['mount',
'--make-slave', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
# can't proceed with shared /proc
writemsg("Unable to mark /proc slave: %d\n" % (mount_ret,),
noiselevel=-1)
os._exit(1)
# mount new /proc for our namespace
s = subprocess.Popen(['mount',
'-t', 'proc', 'proc', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
writemsg("Unable to mount new /proc: %d\n" % (mount_ret,),
noiselevel=-1)
os._exit(1)
if unshare_net:
# 'up' the loopback
IFF_UP = 0x1
ifreq = struct.pack('16sh', b'lo', IFF_UP)
SIOCSIFFLAGS = 0x8914
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
try:
fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq)
except IOError as e:
writemsg("Unable to enable loopback interface: %s\n" % (
errno.errorcode.get(e.errno, '?')),
noiselevel=-1)
sock.close()
except AttributeError:
# unshare() not supported by libc
pass
# Set requested process permissions.
if gid:
# Cast proxies to int, in case it matters.
os.setgid(int(gid))
if groups:
os.setgroups(groups)
if uid:
# Cast proxies to int, in case it matters.
os.setuid(int(uid))
if umask:
os.umask(umask)
if pre_exec:
pre_exec()
# And switch to the new process.
os.execve(binary, myargs, env)
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
dblink.merge().
"""
elog_reader_fd, elog_writer_fd = os.pipe()
fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
# of file descriptors in the subprocess can prevent
# access to open database connections such as that
# used by the sqlite metadata cache module.
blockers = self.blockers()
mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings,
treetype=self.treetype, vartree=self.vartree,
blockers=blockers, scheduler=self.scheduler,
pipe=elog_writer_fd)
fd_pipes[elog_writer_fd] = elog_writer_fd
self._elog_reg_id = self.scheduler.register(elog_reader_fd,
self._registered_events, self._elog_output_handler)
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
# extremely unlikely chance that the COUNTER values will not be
# ordered correctly unless we lock the vdb here.
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
counter = None
if not self.unmerge:
counter = self.vartree.dbapi.counter_tick()
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError(
"fork returned non-integer: %s" % (repr(pid),))
os.close(elog_writer_fd)
self._elog_reader_fd = elog_reader_fd
self._buf = ""
self._elog_keys = set()
# invalidate relevant vardbapi caches
if self.vartree.dbapi._categories is not None:
self.vartree.dbapi._categories = None
self.vartree.dbapi._pkgs_changed = True
self.vartree.dbapi._clear_pkg_cache(mylink)
portage.process.spawned_pids.append(pid)
return [pid]
os.close(elog_reader_fd)
# TODO: Find out why PyPy 1.8 with close_fds=True triggers
# "[Errno 9] Bad file descriptor" in subprocesses. It could
# be due to garbage collection of file objects that were not
# closed before going out of scope, since PyPy's garbage
# collector does not support the refcounting semantics that
# CPython does.
close_fds = platform.python_implementation() != 'PyPy'
portage.process._setup_pipes(fd_pipes, close_fds=close_fds)
# Use default signal handlers since the ones inherited
# from the parent process are irrelevant here.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
portage.output.havecolor = self.settings.get('NOCOLOR') \
not in ('yes', 'true')
# In this subprocess we want mylink._display_merge() to use
# stdout/stderr directly since they are pipes. This behavior
# is triggered when mylink._scheduler is None.
mylink._scheduler = None
# Avoid wastful updates of the vdb cache.
self.vartree.dbapi._flush_cache_enabled = False
# In this subprocess we don't want PORTAGE_BACKGROUND to
# suppress stdout/stderr output since they are pipes. We
# also don't want to open PORTAGE_LOG_FILE, since it will
# already be opened by the parent process, so we set the
# "subprocess" value for use in conditional logging code
# involving PORTAGE_LOG_FILE.
if not self.unmerge:
# unmerge phases have separate logs
if self.settings.get("PORTAGE_BACKGROUND") == "1":
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
else:
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
self.settings["PORTAGE_BACKGROUND"] = "subprocess"
self.settings.backup_changes("PORTAGE_BACKGROUND")
rval = 1
try:
if self.unmerge:
if not mylink.exists():
rval = os.EX_OK
elif mylink.unmerge(
ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
mylink.lockdb()
try:
mylink.delete()
finally:
mylink.unlockdb()
rval = os.EX_OK
else:
rval = mylink.merge(self.pkgloc, self.infloc,
myebuild=self.myebuild, mydbapi=self.mydbapi,
prev_mtimes=self.prev_mtimes, counter=counter)
except SystemExit:
raise
except:
traceback.print_exc()
finally:
# Call os._exit() from finally block, in order to suppress any
# finally blocks from earlier in the call stack. See bug #345289.
os._exit(rval)
def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
uid=None, gid=None, groups=None, umask=None, cwd=None, logfile=None,
path_lookup=True, pre_exec=None,
close_fds=(sys.version_info < (3, 4)), unshare_net=False,
unshare_ipc=False, unshare_mount=False, unshare_pid=False,
cgroup=None):
"""
Spawns a given command.
@param mycommand: the command to execute
@type mycommand: String or List (Popen style list)
@param env: If env is not None, it must be a mapping that defines the environment
variables for the new process; these are used instead of the default behavior
of inheriting the current process's environment.
@type env: None or Mapping
@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
@type opt_name: String
@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
(default is {0:stdin, 1:stdout, 2:stderr})
@type fd_pipes: Dictionary
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
@type returnpid: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@param gid: Group ID to spawn as; useful for dropping privilages
@type gid: Integer
@param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts.
@type groups: List
@param umask: An integer representing the umask for the process (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param logfile: name of a file to use for logging purposes
@type logfile: String
@param path_lookup: If the binary is not fully specified then look for it in PATH
@type path_lookup: Boolean
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param close_fds: If True, then close all file descriptors except those
referenced by fd_pipes (default is True for python3.3 and earlier, and False for
python3.4 and later due to non-inheritable file descriptor behavior from PEP 446).
@type close_fds: Boolean
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
logfile requires stdout and stderr to be assigned to this process (ie not pointed
somewhere else.)
"""
# mycommand is either a str or a list
if isinstance(mycommand, basestring):
mycommand = mycommand.split()
env = os.environ if env is None else env
if sys.hexversion < 0x3000000:
# Avoid a potential UnicodeEncodeError from os.execve().
env_bytes = {}
for k, v in env.items():
env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \
_unicode_encode(v, encoding=_encodings['content'])
env = env_bytes
del env_bytes
# If an absolute path to an executable file isn't given
# search for it unless we've been told not to.
binary = mycommand[0]
if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \
(not os.path.isabs(binary) or not os.path.isfile(binary)
or not os.access(binary, os.X_OK)):
binary = path_lookup and find_binary(binary) or None
if not binary:
raise CommandNotFound(mycommand[0])
# If we haven't been told what file descriptors to use
# default to propagating our stdin, stdout and stderr.
if fd_pipes is None:
fd_pipes = {
0:portage._get_stdin().fileno(),
1:sys.__stdout__.fileno(),
2:sys.__stderr__.fileno(),
}
# mypids will hold the pids of all processes created.
mypids = []
if logfile:
# Using a log file requires that stdout and stderr
# are assigned to the process we're running.
if 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
# Create a pipe
(pr, pw) = os.pipe()
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
mypids.extend(spawn(('tee', '-i', '-a', logfile),
returnpid=True, fd_pipes={0:pr,
1:fd_pipes[1], 2:fd_pipes[2]}))
# We don't need the read end of the pipe, so close it.
os.close(pr)
# Assign the write end of the pipe to our stdout and stderr.
fd_pipes[1] = pw
fd_pipes[2] = pw
# This caches the libc library lookup and _unshare_validator results
# in the current process, so that results are cached for use in
# child processes.
unshare_flags = 0
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
# from /usr/include/bits/sched.h
CLONE_NEWNS = 0x00020000
CLONE_NEWIPC = 0x08000000
CLONE_NEWPID = 0x20000000
CLONE_NEWNET = 0x40000000
if unshare_net:
unshare_flags |= CLONE_NEWNET
if unshare_ipc:
unshare_flags |= CLONE_NEWIPC
if unshare_mount:
# NEWNS = mount namespace
unshare_flags |= CLONE_NEWNS
if unshare_pid:
# we also need mount namespace for slave /proc
unshare_flags |= CLONE_NEWPID | CLONE_NEWNS
_unshare_validate(unshare_flags)
# Force instantiation of portage.data.userpriv_groups before the
# fork, so that the result is cached in the main process.
bool(groups)
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid == 0:
try:
_exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, cwd, pre_exec, close_fds,
unshare_net, unshare_ipc, unshare_mount, unshare_pid,
unshare_flags, cgroup)
except SystemExit:
raise
except Exception as e:
# We need to catch _any_ exception so that it doesn't
# propagate out of this function and cause exiting
# with anything other than os._exit()
writemsg("%s:\n %s\n" % (e, " ".join(mycommand)),
noiselevel=-1)
traceback.print_exc()
sys.stderr.flush()
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" % (repr(pid),))
# Add the pid to our local and the global pid lists.
mypids.append(pid)
# If we started a tee process the write side of the pipe is no
# longer needed, so close it.
if logfile:
os.close(pw)
# If the caller wants to handle cleaning up the processes, we tell
# it about all processes that were created.
if returnpid:
return mypids
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
# in the pipe are well behaved, it will die when the process
# it is reading from dies.
pid = mypids.pop(0)
# and wait for it.
retval = os.waitpid(pid, 0)[1]
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if os.waitpid(pid, os.WNOHANG)[0] == 0:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
# If it got a signal, return the signal that was sent.
if (retval & 0xff):
return ((retval & 0xff) << 8)
# Otherwise, return its exit code.
return (retval >> 8)
# Everything succeeded
return 0
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
dblink.merge().
"""
elog_reader_fd, elog_writer_fd = os.pipe()
fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
# of file descriptors in the subprocess can prevent
# access to open database connections such as that
# used by the sqlite metadata cache module.
blockers = self.blockers()
mylink = portage.dblink(self.mycat,
self.mypkg,
settings=self.settings,
treetype=self.treetype,
vartree=self.vartree,
blockers=blockers,
scheduler=self.scheduler,
pipe=elog_writer_fd)
fd_pipes[elog_writer_fd] = elog_writer_fd
self._elog_reg_id = self.scheduler.register(elog_reader_fd,
self._registered_events,
self._elog_output_handler)
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
# extremely unlikely chance that the COUNTER values will not be
# ordered correctly unless we lock the vdb here.
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
counter = None
if not self.unmerge:
counter = self.vartree.dbapi.counter_tick()
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" %
(repr(pid), ))
os.close(elog_writer_fd)
self._elog_reader_fd = elog_reader_fd
self._buf = ""
self._elog_keys = set()
# invalidate relevant vardbapi caches
if self.vartree.dbapi._categories is not None:
self.vartree.dbapi._categories = None
self.vartree.dbapi._pkgs_changed = True
self.vartree.dbapi._clear_pkg_cache(mylink)
portage.process.spawned_pids.append(pid)
return [pid]
os.close(elog_reader_fd)
# TODO: Find out why PyPy 1.8 with close_fds=True triggers
# "[Errno 9] Bad file descriptor" in subprocesses. It could
# be due to garbage collection of file objects that were not
# closed before going out of scope, since PyPy's garbage
# collector does not support the refcounting semantics that
# CPython does.
close_fds = platform.python_implementation() != 'PyPy'
portage.process._setup_pipes(fd_pipes, close_fds=close_fds)
# Use default signal handlers since the ones inherited
# from the parent process are irrelevant here.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
portage.output.havecolor = self.settings.get('NOCOLOR') \
not in ('yes', 'true')
# In this subprocess we want mylink._display_merge() to use
# stdout/stderr directly since they are pipes. This behavior
# is triggered when mylink._scheduler is None.
mylink._scheduler = None
# Avoid wastful updates of the vdb cache.
self.vartree.dbapi._flush_cache_enabled = False
# In this subprocess we don't want PORTAGE_BACKGROUND to
# suppress stdout/stderr output since they are pipes. We
# also don't want to open PORTAGE_LOG_FILE, since it will
# already be opened by the parent process, so we set the
# "subprocess" value for use in conditional logging code
# involving PORTAGE_LOG_FILE.
if not self.unmerge:
# unmerge phases have separate logs
if self.settings.get("PORTAGE_BACKGROUND") == "1":
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
else:
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
self.settings["PORTAGE_BACKGROUND"] = "subprocess"
self.settings.backup_changes("PORTAGE_BACKGROUND")
rval = 1
try:
if self.unmerge:
if not mylink.exists():
rval = os.EX_OK
elif mylink.unmerge(
ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
mylink.lockdb()
try:
mylink.delete()
finally:
mylink.unlockdb()
rval = os.EX_OK
else:
rval = mylink.merge(self.pkgloc,
self.infloc,
myebuild=self.myebuild,
mydbapi=self.mydbapi,
prev_mtimes=self.prev_mtimes,
counter=counter)
except SystemExit:
raise
except:
traceback.print_exc()
finally:
# Call os._exit() from finally block, in order to suppress any
# finally blocks from earlier in the call stack. See bug #345289.
os._exit(rval)
