def _spawn(self, args, fd_pipes=None, **kwargs):
"""
Override SpawnProcess._spawn to fork a subprocess that calls
self._run(). This uses multiprocessing.Process in order to leverage
any pre-fork and post-fork interpreter housekeeping that it provides,
promoting a healthy state for the forked interpreter.
"""
# Since multiprocessing.Process closes sys.__stdin__, create a
# temporary duplicate of fd_pipes[0] so that sys.__stdin__ can
# be restored in the subprocess, in case this is needed for
# things like PROPERTIES=interactive support.
stdin_dup = None
try:
stdin_fd = fd_pipes.get(0)
if stdin_fd is not None and stdin_fd == portage._get_stdin(
).fileno():
stdin_dup = os.dup(stdin_fd)
fcntl.fcntl(stdin_dup, fcntl.F_SETFD,
fcntl.fcntl(stdin_fd, fcntl.F_GETFD))
fd_pipes[0] = stdin_dup
self._proc = multiprocessing.Process(target=self._bootstrap,
args=(fd_pipes, ))
self._proc.start()
finally:
if stdin_dup is not None:
os.close(stdin_dup)
self._proc_join_task = asyncio.ensure_future(self._proc_join(
self._proc, loop=self.scheduler),
loop=self.scheduler)
self._proc_join_task.add_done_callback(
functools.partial(self._proc_join_done, self._proc))
return [self._proc.pid]
def xtermTitleReset():
global default_xterm_title
if default_xterm_title is None:
prompt_command = os.environ.get('PROMPT_COMMAND')
if prompt_command == "":
default_xterm_title = ""
elif prompt_command is not None:
if dotitles and \
'TERM' in os.environ and \
_legal_terms_re.match(os.environ['TERM']) is not None and \
sys.__stderr__.isatty():
from portage.process import find_binary, spawn
shell = os.environ.get("SHELL")
if not shell or not os.access(shell, os.EX_OK):
shell = find_binary("sh")
if shell:
spawn([shell, "-c", prompt_command], env=os.environ,
fd_pipes={
0: portage._get_stdin().fileno(),
1: sys.__stderr__.fileno(),
2: sys.__stderr__.fileno()
})
else:
os.system(prompt_command)
return
else:
pwd = os.environ.get('PWD','')
home = os.environ.get('HOME', '')
if home != '' and pwd.startswith(home):
pwd = '~' + pwd[len(home):]
default_xterm_title = '\x1b]0;%s@%s:%s\x07' % (
os.environ.get('LOGNAME', ''),
os.environ.get('HOSTNAME', '').split('.', 1)[0], pwd)
xtermTitle(default_xterm_title, raw=True)
def _start(self):
# Portage should always call setcpv prior to this
# point, but here we have a fallback as a convenience
# for external API consumers. It's important that
# this metadata access happens in the parent process,
# since closing of file descriptors in the subprocess
# can prevent access to open database connections such
# as that used by the sqlite metadata cache module.
cpv = "%s/%s" % (self.mycat, self.mypkg)
settings = self.settings
if cpv != settings.mycpv or "EAPI" not in settings.configdict["pkg"]:
settings.reload()
settings.reset()
settings.setcpv(cpv, mydb=self.mydbapi)
# This caches the libc library lookup in the current
# process, so that it's only done once rather than
# for each child process.
if platform.system() == "Linux" and "merge-sync" in settings.features:
find_library("c")
# Inherit stdin by default, so that the pdb SIGUSR1
# handler is usable for the subprocess.
if self.fd_pipes is None:
self.fd_pipes = {}
else:
self.fd_pipes = self.fd_pipes.copy()
self.fd_pipes.setdefault(0, portage._get_stdin().fileno())
self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD")
super(MergeProcess, self)._start()
def _start(self):
# Portage should always call setcpv prior to this
# point, but here we have a fallback as a convenience
# for external API consumers. It's important that
# this metadata access happens in the parent process,
# since closing of file descriptors in the subprocess
# can prevent access to open database connections such
# as that used by the sqlite metadata cache module.
cpv = "%s/%s" % (self.mycat, self.mypkg)
settings = self.settings
if cpv != settings.mycpv or \
"EAPI" not in settings.configdict["pkg"]:
settings.reload()
settings.reset()
settings.setcpv(cpv, mydb=self.mydbapi)
# This caches the libc library lookup in the current
# process, so that it's only done once rather than
# for each child process.
if platform.system() == "Linux" and \
"merge-sync" in settings.features:
find_library("c")
# Inherit stdin by default, so that the pdb SIGUSR1
# handler is usable for the subprocess.
if self.fd_pipes is None:
self.fd_pipes = {}
else:
self.fd_pipes = self.fd_pipes.copy()
self.fd_pipes.setdefault(0, portage._get_stdin().fileno())
super(MergeProcess, self)._start()
def file_get(baseurl=None,
dest=None,
conn=None,
fcmd=None,
filename=None,
fcmd_vars=None):
"""Takes a base url to connect to and read from.
URI should be in the form <proto>://[user[:pass]@]<site>[:port]<path>"""
if not fcmd:
warnings.warn(
"Use of portage.getbinpkg.file_get() without the fcmd "
"parameter is deprecated",
DeprecationWarning,
stacklevel=2,
)
return file_get_lib(baseurl, dest, conn)
variables = {}
if fcmd_vars is not None:
variables.update(fcmd_vars)
if "DISTDIR" not in variables:
if dest is None:
raise portage.exception.MissingParameter(
_("%s is missing required '%s' key") %
("fcmd_vars", "DISTDIR"))
variables["DISTDIR"] = dest
if "URI" not in variables:
if baseurl is None:
raise portage.exception.MissingParameter(
_("%s is missing required '%s' key") % ("fcmd_vars", "URI"))
variables["URI"] = baseurl
if "FILE" not in variables:
if filename is None:
filename = os.path.basename(variables["URI"])
variables["FILE"] = filename
from portage.util import varexpand
from portage.process import spawn
myfetch = portage.util.shlex_split(fcmd)
myfetch = [varexpand(x, mydict=variables) for x in myfetch]
fd_pipes = {
0: portage._get_stdin().fileno(),
1: sys.__stdout__.fileno(),
2: sys.__stdout__.fileno(),
}
sys.__stdout__.flush()
sys.__stderr__.flush()
retval = spawn(myfetch, env=os.environ.copy(), fd_pipes=fd_pipes)
if retval != os.EX_OK:
sys.stderr.write(_("Fetcher exited with a failure condition.\n"))
return 0
return 1
def _spawn_fetch(settings, args, **kwargs):
"""
Spawn a process with appropriate settings for fetching, including
userfetch and selinux support.
"""
global _userpriv_spawn_kwargs
# Redirect all output to stdout since some fetchers like
# wget pollute stderr (if portage detects a problem then it
# can send it's own message to stderr).
if "fd_pipes" not in kwargs:
kwargs["fd_pipes"] = {
0 : portage._get_stdin().fileno(),
1 : sys.__stdout__.fileno(),
2 : sys.__stdout__.fileno(),
}
logname = None
if "userfetch" in settings.features and \
os.getuid() == 0 and portage_gid and portage_uid and \
hasattr(os, "setgroups"):
kwargs.update(_userpriv_spawn_kwargs)
logname = portage.data._portage_username
spawn_func = spawn
if settings.selinux_enabled():
spawn_func = selinux.spawn_wrapper(spawn_func,
settings["PORTAGE_FETCH_T"])
# bash is an allowed entrypoint, while most binaries are not
if args[0] != BASH_BINARY:
args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
# Ensure that EBUILD_PHASE is set to fetch, so that config.environ()
# does not filter the calling environment (which may contain needed
# proxy variables, as in bug #315421).
phase_backup = settings.get('EBUILD_PHASE')
settings['EBUILD_PHASE'] = 'fetch'
env = settings.environ()
if logname is not None:
env["LOGNAME"] = logname
try:
rval = spawn_func(args, env=env, **kwargs)
finally:
if phase_backup is None:
settings.pop('EBUILD_PHASE', None)
else:
settings['EBUILD_PHASE'] = phase_backup
return rval
def _spawn_fetch(settings, args, **kwargs):
"""
Spawn a process with appropriate settings for fetching, including
userfetch and selinux support.
"""
global _userpriv_spawn_kwargs
# Redirect all output to stdout since some fetchers like
# wget pollute stderr (if portage detects a problem then it
# can send it's own message to stderr).
if "fd_pipes" not in kwargs:
kwargs["fd_pipes"] = {
0 : portage._get_stdin().fileno(),
1 : sys.__stdout__.fileno(),
2 : sys.__stdout__.fileno(),
}
logname = None
if "userfetch" in settings.features and \
os.getuid() == 0 and portage_gid and portage_uid and \
hasattr(os, "setgroups"):
kwargs.update(_userpriv_spawn_kwargs)
logname = portage.data._portage_username
spawn_func = spawn
if settings.selinux_enabled():
spawn_func = selinux.spawn_wrapper(spawn_func,
settings["PORTAGE_FETCH_T"])
# bash is an allowed entrypoint, while most binaries are not
if args[0] != BASH_BINARY:
args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
# Ensure that EBUILD_PHASE is set to fetch, so that config.environ()
# does not filter the calling environment (which may contain needed
# proxy variables, as in bug #315421).
phase_backup = settings.get('EBUILD_PHASE')
settings['EBUILD_PHASE'] = 'fetch'
env = settings.environ()
if logname is not None:
env["LOGNAME"] = logname
try:
rval = spawn_func(args, env=env, **kwargs)
finally:
if phase_backup is None:
settings.pop('EBUILD_PHASE', None)
else:
settings['EBUILD_PHASE'] = phase_backup
return rval
def file_get(baseurl=None, dest=None, conn=None, fcmd=None, filename=None,
fcmd_vars=None):
"""Takes a base url to connect to and read from.
URI should be in the form <proto>://[user[:pass]@]<site>[:port]<path>"""
if not fcmd:
warnings.warn("Use of portage.getbinpkg.file_get() without the fcmd "
"parameter is deprecated", DeprecationWarning, stacklevel=2)
return file_get_lib(baseurl, dest, conn)
variables = {}
if fcmd_vars is not None:
variables.update(fcmd_vars)
if "DISTDIR" not in variables:
if dest is None:
raise portage.exception.MissingParameter(
_("%s is missing required '%s' key") %
("fcmd_vars", "DISTDIR"))
variables["DISTDIR"] = dest
if "URI" not in variables:
if baseurl is None:
raise portage.exception.MissingParameter(
_("%s is missing required '%s' key") %
("fcmd_vars", "URI"))
variables["URI"] = baseurl
if "FILE" not in variables:
if filename is None:
filename = os.path.basename(variables["URI"])
variables["FILE"] = filename
from portage.util import varexpand
from portage.process import spawn
myfetch = portage.util.shlex_split(fcmd)
myfetch = [varexpand(x, mydict=variables) for x in myfetch]
fd_pipes = {
0: portage._get_stdin().fileno(),
1: sys.__stdout__.fileno(),
2: sys.__stdout__.fileno()
}
sys.__stdout__.flush()
sys.__stderr__.flush()
retval = spawn(myfetch, env=os.environ.copy(), fd_pipes=fd_pipes)
if retval != os.EX_OK:
sys.stderr.write(_("Fetcher exited with a failure condition.\n"))
return 0
return 1
def testLogfile(self):
logfile = None
try:
fd, logfile = tempfile.mkstemp()
os.close(fd)
null_fd = os.open('/dev/null', os.O_RDWR)
test_string = 2 * "blah blah blah\n"
proc = SpawnProcess(
args=[BASH_BINARY, "-c",
"echo -n '%s'" % test_string],
env={},
fd_pipes={
0: portage._get_stdin().fileno(),
1: null_fd,
2: null_fd
},
scheduler=global_event_loop(),
logfile=logfile)
global_event_loop().run_until_complete(proc.async_start())
os.close(null_fd)
self.assertEqual(proc.wait(), os.EX_OK)
f = io.open(_unicode_encode(logfile,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['content'],
errors='strict')
log_content = f.read()
f.close()
# When logging passes through a pty, this comparison will fail
# unless the oflag terminal attributes have the termios.OPOST
# bit disabled. Otherwise, tranformations such as \n -> \r\n
# may occur.
self.assertEqual(test_string, log_content)
finally:
if logfile:
try:
os.unlink(logfile)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
del e
def xtermTitleReset():
global default_xterm_title
if default_xterm_title is None:
prompt_command = os.environ.get("PROMPT_COMMAND")
if prompt_command == "":
default_xterm_title = ""
elif prompt_command is not None:
if (
dotitles
and "TERM" in os.environ
and _legal_terms_re.match(os.environ["TERM"]) is not None
and sys.__stderr__.isatty()
):
from portage.process import find_binary, spawn
shell = os.environ.get("SHELL")
if not shell or not os.access(shell, os.EX_OK):
shell = find_binary("sh")
if shell:
spawn(
[shell, "-c", prompt_command],
env=os.environ,
fd_pipes={
0: portage._get_stdin().fileno(),
1: sys.__stderr__.fileno(),
2: sys.__stderr__.fileno(),
},
)
else:
os.system(prompt_command)
return
else:
pwd = os.environ.get("PWD", "")
home = os.environ.get("HOME", "")
if home != "" and pwd.startswith(home):
pwd = "~" + pwd[len(home) :]
default_xterm_title = "\x1b]0;%s@%s:%s\x07" % (
os.environ.get("LOGNAME", ""),
os.environ.get("HOSTNAME", "").split(".", 1)[0],
pwd,
)
xtermTitle(default_xterm_title, raw=True)
def sanitize_fds():
"""
Set the inheritable flag to False for all open file descriptors,
except for those corresponding to stdin, stdout, and stderr. This
ensures that any unintentionally inherited file descriptors will
not be inherited by child processes.
"""
if _set_inheritable is not None:
whitelist = frozenset([
portage._get_stdin().fileno(),
sys.__stdout__.fileno(),
sys.__stderr__.fileno(),
])
for fd in get_open_fds():
if fd not in whitelist:
try:
_set_inheritable(fd, False)
except OSError:
pass
def testLogfile(self):
logfile = None
try:
fd, logfile = tempfile.mkstemp()
os.close(fd)
null_fd = os.open('/dev/null', os.O_RDWR)
test_string = 2 * "blah blah blah\n"
proc = SpawnProcess(
args=[BASH_BINARY, "-c",
"echo -n '%s'" % test_string],
env={},
fd_pipes={
0: portage._get_stdin().fileno(),
1: null_fd,
2: null_fd
},
scheduler=global_event_loop(),
logfile=logfile)
proc.start()
os.close(null_fd)
self.assertEqual(proc.wait(), os.EX_OK)
f = io.open(_unicode_encode(logfile,
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['content'], errors='strict')
log_content = f.read()
f.close()
# When logging passes through a pty, this comparison will fail
# unless the oflag terminal attributes have the termios.OPOST
# bit disabled. Otherwise, tranformations such as \n -> \r\n
# may occur.
self.assertEqual(test_string, log_content)
finally:
if logfile:
try:
os.unlink(logfile)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
del e
def _start(self):
pkg = self.pkg
pretend = self.pretend
bintree = pkg.root_config.trees["bintree"]
settings = bintree.settings
pkg_path = self.pkg_path
exists = os.path.exists(pkg_path)
resume = exists and os.path.basename(pkg_path) in bintree.invalids
if not (pretend or resume):
# Remove existing file or broken symlink.
try:
os.unlink(pkg_path)
except OSError:
pass
# urljoin doesn't work correctly with
# unrecognized protocols like sftp
fetchcommand = None
resumecommand = None
if bintree._remote_has_index:
remote_metadata = bintree._remotepkgs[bintree.dbapi._instance_key(
pkg.cpv)]
rel_uri = remote_metadata.get("PATH")
if not rel_uri:
rel_uri = pkg.cpv + ".tbz2"
remote_base_uri = remote_metadata["BASE_URI"]
uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
fetchcommand = remote_metadata.get('FETCHCOMMAND')
resumecommand = remote_metadata.get('RESUMECOMMAND')
else:
uri = settings["PORTAGE_BINHOST"].rstrip("/") + \
"/" + pkg.pf + ".tbz2"
if pretend:
portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1)
self.returncode = os.EX_OK
self._async_wait()
return
fcmd = None
if resume:
fcmd = resumecommand
else:
fcmd = fetchcommand
if fcmd is None:
protocol = urllib_parse_urlparse(uri)[0]
fcmd_prefix = "FETCHCOMMAND"
if resume:
fcmd_prefix = "RESUMECOMMAND"
fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
if not fcmd:
fcmd = settings.get(fcmd_prefix)
fcmd_vars = {
"DISTDIR": os.path.dirname(pkg_path),
"URI": uri,
"FILE": os.path.basename(pkg_path)
}
for k in ("PORTAGE_SSH_OPTS", ):
v = settings.get(k)
if v is not None:
fcmd_vars[k] = v
fetch_env = dict(settings.items())
fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \
for x in portage.util.shlex_split(fcmd)]
if self.fd_pipes is None:
self.fd_pipes = {}
fd_pipes = self.fd_pipes
# Redirect all output to stdout since some fetchers like
# wget pollute stderr (if portage detects a problem then it
# can send it's own message to stderr).
fd_pipes.setdefault(0, portage._get_stdin().fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stdout__.fileno())
self.args = fetch_args
self.env = fetch_env
if settings.selinux_enabled():
self._selinux_type = settings["PORTAGE_FETCH_T"]
self.log_filter_file = settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
SpawnProcess._start(self)
def spawn(
mycommand,
env=None,
opt_name=None,
fd_pipes=None,
returnpid=False,
uid=None,
gid=None,
groups=None,
umask=None,
cwd=None,
logfile=None,
path_lookup=True,
pre_exec=None,
close_fds=False,
unshare_net=False,
unshare_ipc=False,
unshare_mount=False,
unshare_pid=False,
cgroup=None,
):
"""
Spawns a given command.
@param mycommand: the command to execute
@type mycommand: String or List (Popen style list)
@param env: If env is not None, it must be a mapping that defines the environment
variables for the new process; these are used instead of the default behavior
of inheriting the current process's environment.
@type env: None or Mapping
@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
@type opt_name: String
@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
(default is {0:stdin, 1:stdout, 2:stderr})
@type fd_pipes: Dictionary
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
@type returnpid: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@param gid: Group ID to spawn as; useful for dropping privilages
@type gid: Integer
@param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts.
@type groups: List
@param umask: An integer representing the umask for the process (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param logfile: name of a file to use for logging purposes
@type logfile: String
@param path_lookup: If the binary is not fully specified then look for it in PATH
@type path_lookup: Boolean
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param close_fds: If True, then close all file descriptors except those
referenced by fd_pipes (default is True for python3.3 and earlier, and False for
python3.4 and later due to non-inheritable file descriptor behavior from PEP 446).
@type close_fds: Boolean
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
logfile requires stdout and stderr to be assigned to this process (ie not pointed
somewhere else.)
"""
# mycommand is either a str or a list
if isinstance(mycommand, str):
mycommand = mycommand.split()
env = os.environ if env is None else env
# If an absolute path to an executable file isn't given
# search for it unless we've been told not to.
binary = mycommand[0]
if binary not in (BASH_BINARY, SANDBOX_BINARY,
FAKEROOT_BINARY) and (not os.path.isabs(binary)
or not os.path.isfile(binary)
or not os.access(binary, os.X_OK)):
binary = path_lookup and find_binary(binary) or None
if not binary:
raise CommandNotFound(mycommand[0])
# If we haven't been told what file descriptors to use
# default to propagating our stdin, stdout and stderr.
if fd_pipes is None:
fd_pipes = {
0: portage._get_stdin().fileno(),
1: sys.__stdout__.fileno(),
2: sys.__stderr__.fileno(),
}
# mypids will hold the pids of all processes created.
mypids = []
if logfile:
# Using a log file requires that stdout and stderr
# are assigned to the process we're running.
if 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
# Create a pipe
(pr, pw) = os.pipe()
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
mypids.extend(
spawn(
("tee", "-i", "-a", logfile),
returnpid=True,
fd_pipes={
0: pr,
1: fd_pipes[1],
2: fd_pipes[2]
},
))
# We don't need the read end of the pipe, so close it.
os.close(pr)
# Assign the write end of the pipe to our stdout and stderr.
fd_pipes[1] = pw
fd_pipes[2] = pw
# Cache _has_ipv6() result for use in child processes.
_has_ipv6()
# This caches the libc library lookup and _unshare_validator results
# in the current process, so that results are cached for use in
# child processes.
unshare_flags = 0
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
# from /usr/include/bits/sched.h
CLONE_NEWNS = 0x00020000
CLONE_NEWUTS = 0x04000000
CLONE_NEWIPC = 0x08000000
CLONE_NEWPID = 0x20000000
CLONE_NEWNET = 0x40000000
if unshare_net:
# UTS namespace to override hostname
unshare_flags |= CLONE_NEWNET | CLONE_NEWUTS
if unshare_ipc:
unshare_flags |= CLONE_NEWIPC
if unshare_mount:
# NEWNS = mount namespace
unshare_flags |= CLONE_NEWNS
if unshare_pid:
# we also need mount namespace for slave /proc
unshare_flags |= CLONE_NEWPID | CLONE_NEWNS
_unshare_validate(unshare_flags)
# Force instantiation of portage.data.userpriv_groups before the
# fork, so that the result is cached in the main process.
bool(groups)
parent_pid = portage.getpid()
pid = None
try:
pid = os.fork()
if pid == 0:
portage._ForkWatcher.hook(portage._ForkWatcher)
try:
_exec(
binary,
mycommand,
opt_name,
fd_pipes,
env,
gid,
groups,
uid,
umask,
cwd,
pre_exec,
close_fds,
unshare_net,
unshare_ipc,
unshare_mount,
unshare_pid,
unshare_flags,
cgroup,
)
except SystemExit:
raise
except Exception as e:
# We need to catch _any_ exception so that it doesn't
# propagate out of this function and cause exiting
# with anything other than os._exit()
writemsg("%s:\n %s\n" % (e, " ".join(mycommand)),
noiselevel=-1)
traceback.print_exc()
sys.stderr.flush()
finally:
# Don't used portage.getpid() here, due to a race with the above
# portage._ForkWatcher cache update.
if pid == 0 or (pid is None and _os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" % (repr(pid), ))
# Add the pid to our local and the global pid lists.
mypids.append(pid)
# If we started a tee process the write side of the pipe is no
# longer needed, so close it.
if logfile:
os.close(pw)
# If the caller wants to handle cleaning up the processes, we tell
# it about all processes that were created.
if returnpid:
return mypids
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
# in the pipe are well behaved, it will die when the process
# it is reading from dies.
pid = mypids.pop(0)
# and wait for it.
retval = os.waitpid(pid, 0)[1]
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if os.waitpid(pid, os.WNOHANG)[0] == 0:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
# If it got a signal, return the signal that was sent.
if retval & 0xFF:
return (retval & 0xFF) << 8
# Otherwise, return its exit code.
return retval >> 8
# Everything succeeded
return 0
def spawn(mycommand,
env={},
opt_name=None,
fd_pipes=None,
returnpid=False,
uid=None,
gid=None,
groups=None,
umask=None,
logfile=None,
path_lookup=True,
pre_exec=None,
close_fds=(sys.version_info < (3, 4)),
unshare_net=False,
unshare_ipc=False,
cgroup=None):
"""
Spawns a given command.
@param mycommand: the command to execute
@type mycommand: String or List (Popen style list)
@param env: A dict of Key=Value pairs for env variables
@type env: Dictionary
@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
@type opt_name: String
@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
(default is {0:stdin, 1:stdout, 2:stderr})
@type fd_pipes: Dictionary
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
@type returnpid: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@param gid: Group ID to spawn as; useful for dropping privilages
@type gid: Integer
@param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts.
@type groups: List
@param umask: An integer representing the umask for the process (see man chmod for umask details)
@type umask: Integer
@param logfile: name of a file to use for logging purposes
@type logfile: String
@param path_lookup: If the binary is not fully specified then look for it in PATH
@type path_lookup: Boolean
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param close_fds: If True, then close all file descriptors except those
referenced by fd_pipes (default is True for python3.3 and earlier, and False for
python3.4 and later due to non-inheritable file descriptor behavior from PEP 446).
@type close_fds: Boolean
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
logfile requires stdout and stderr to be assigned to this process (ie not pointed
somewhere else.)
"""
# mycommand is either a str or a list
if isinstance(mycommand, basestring):
mycommand = mycommand.split()
if sys.hexversion < 0x3000000:
# Avoid a potential UnicodeEncodeError from os.execve().
env_bytes = {}
for k, v in env.items():
env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \
_unicode_encode(v, encoding=_encodings['content'])
env = env_bytes
del env_bytes
# If an absolute path to an executable file isn't given
# search for it unless we've been told not to.
binary = mycommand[0]
if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \
(not os.path.isabs(binary) or not os.path.isfile(binary)
or not os.access(binary, os.X_OK)):
binary = path_lookup and find_binary(binary) or None
if not binary:
raise CommandNotFound(mycommand[0])
# If we haven't been told what file descriptors to use
# default to propagating our stdin, stdout and stderr.
if fd_pipes is None:
fd_pipes = {
0: portage._get_stdin().fileno(),
1: sys.__stdout__.fileno(),
2: sys.__stderr__.fileno(),
}
# mypids will hold the pids of all processes created.
mypids = []
if logfile:
# Using a log file requires that stdout and stderr
# are assigned to the process we're running.
if 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
# Create a pipe
(pr, pw) = os.pipe()
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
mypids.extend(
spawn(('tee', '-i', '-a', logfile),
returnpid=True,
fd_pipes={
0: pr,
1: fd_pipes[1],
2: fd_pipes[2]
}))
# We don't need the read end of the pipe, so close it.
os.close(pr)
# Assign the write end of the pipe to our stdout and stderr.
fd_pipes[1] = pw
fd_pipes[2] = pw
# This caches the libc library lookup in the current
# process, so that it's only done once rather than
# for each child process.
if unshare_net or unshare_ipc:
find_library("c")
# Force instantiation of portage.data.userpriv_groups before the
# fork, so that the result is cached in the main process.
bool(groups)
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid == 0:
try:
_exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups,
uid, umask, pre_exec, close_fds, unshare_net,
unshare_ipc, cgroup)
except SystemExit:
raise
except Exception as e:
# We need to catch _any_ exception so that it doesn't
# propagate out of this function and cause exiting
# with anything other than os._exit()
writemsg("%s:\n %s\n" % (e, " ".join(mycommand)),
noiselevel=-1)
traceback.print_exc()
sys.stderr.flush()
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" % (repr(pid), ))
# Add the pid to our local and the global pid lists.
mypids.append(pid)
# If we started a tee process the write side of the pipe is no
# longer needed, so close it.
if logfile:
os.close(pw)
# If the caller wants to handle cleaning up the processes, we tell
# it about all processes that were created.
if returnpid:
return mypids
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
# in the pipe are well behaved, it will die when the process
# it is reading from dies.
pid = mypids.pop(0)
# and wait for it.
retval = os.waitpid(pid, 0)[1]
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if os.waitpid(pid, os.WNOHANG)[0] == 0:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
# If it got a signal, return the signal that was sent.
if (retval & 0xff):
return ((retval & 0xff) << 8)
# Otherwise, return its exit code.
return (retval >> 8)
# Everything succeeded
return 0
def _start(self):
if self.fd_pipes is None:
self.fd_pipes = {}
else:
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
master_fd, slave_fd = self._pipe(fd_pipes)
can_log = self._can_log(slave_fd)
if can_log:
log_file_path = self.logfile
else:
log_file_path = None
null_input = None
if not self.background or 0 in fd_pipes:
# Subclasses such as AbstractEbuildProcess may have already passed
# in a null file descriptor in fd_pipes, so use that when given.
pass
else:
# TODO: Use job control functions like tcsetpgrp() to control
# access to stdin. Until then, use /dev/null so that any
# attempts to read from stdin will immediately return EOF
# instead of blocking indefinitely.
null_input = os.open('/dev/null', os.O_RDWR)
fd_pipes[0] = null_input
fd_pipes.setdefault(0, portage._get_stdin().fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stderr__.fileno())
# flush any pending output
stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
for fd in fd_pipes.values():
if fd in stdout_filenos:
sys.__stdout__.flush()
sys.__stderr__.flush()
break
fd_pipes_orig = fd_pipes.copy()
if log_file_path is not None or self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
else:
# Create a dummy pipe that PipeLogger uses to efficiently
# monitor for process exit by listening for the EOF event.
# Re-use of the allocated fd number for the key in fd_pipes
# guarantees that the keys will not collide for similarly
# allocated pipes which are used by callers such as
# FileDigester and MergeProcess. See the _setup_pipes
# docstring for more benefits of this allocation approach.
self._dummy_pipe_fd = slave_fd
fd_pipes[slave_fd] = slave_fd
kwargs = {}
for k in self._spawn_kwarg_names:
v = getattr(self, k)
if v is not None:
kwargs[k] = v
kwargs["fd_pipes"] = fd_pipes
kwargs["returnpid"] = True
kwargs.pop("logfile", None)
retval = self._spawn(self.args, **kwargs)
os.close(slave_fd)
if null_input is not None:
os.close(null_input)
if isinstance(retval, int):
# spawn failed
self._unregister()
self._set_returncode((self.pid, retval))
self._async_wait()
return
self.pid = retval[0]
stdout_fd = None
if can_log and not self.background:
stdout_fd = os.dup(fd_pipes_orig[1])
# FD_CLOEXEC is enabled by default in Python >=3.4.
if sys.hexversion < 0x3040000 and fcntl is not None:
try:
fcntl.FD_CLOEXEC
except AttributeError:
pass
else:
fcntl.fcntl(stdout_fd, fcntl.F_SETFD,
fcntl.fcntl(stdout_fd,
fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
self._pipe_logger = PipeLogger(background=self.background,
scheduler=self.scheduler, input_fd=master_fd,
log_file_path=log_file_path,
stdout_fd=stdout_fd)
self._pipe_logger.addExitListener(self._pipe_logger_exit)
self._pipe_logger.start()
self._registered = True
def _start(self):
if self.fd_pipes is None:
self.fd_pipes = {}
else:
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
master_fd, slave_fd = self._pipe(fd_pipes)
can_log = self._can_log(slave_fd)
if can_log:
log_file_path = self.logfile
else:
log_file_path = None
null_input = None
if not self.background or 0 in fd_pipes:
# Subclasses such as AbstractEbuildProcess may have already passed
# in a null file descriptor in fd_pipes, so use that when given.
pass
else:
# TODO: Use job control functions like tcsetpgrp() to control
# access to stdin. Until then, use /dev/null so that any
# attempts to read from stdin will immediately return EOF
# instead of blocking indefinitely.
null_input = os.open('/dev/null', os.O_RDWR)
fd_pipes[0] = null_input
fd_pipes.setdefault(0, portage._get_stdin().fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stderr__.fileno())
# flush any pending output
stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
for fd in fd_pipes.values():
if fd in stdout_filenos:
sys.__stdout__.flush()
sys.__stderr__.flush()
break
fd_pipes_orig = fd_pipes.copy()
if log_file_path is not None or self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
else:
# Create a dummy pipe that PipeLogger uses to efficiently
# monitor for process exit by listening for the EOF event.
# Re-use of the allocated fd number for the key in fd_pipes
# guarantees that the keys will not collide for similarly
# allocated pipes which are used by callers such as
# FileDigester and MergeProcess. See the _setup_pipes
# docstring for more benefits of this allocation approach.
self._dummy_pipe_fd = slave_fd
fd_pipes[slave_fd] = slave_fd
kwargs = {}
for k in self._spawn_kwarg_names:
v = getattr(self, k)
if v is not None:
kwargs[k] = v
kwargs["fd_pipes"] = fd_pipes
kwargs["returnpid"] = True
kwargs.pop("logfile", None)
retval = self._spawn(self.args, **kwargs)
os.close(slave_fd)
if null_input is not None:
os.close(null_input)
if isinstance(retval, int):
# spawn failed
self.returncode = retval
self._async_wait()
return
self.pid = retval[0]
stdout_fd = None
if can_log and not self.background:
stdout_fd = os.dup(fd_pipes_orig[1])
build_logger = BuildLogger(env=self.env,
log_path=log_file_path,
log_filter_file=self.log_filter_file,
scheduler=self.scheduler)
build_logger.start()
pipe_logger = PipeLogger(background=self.background,
scheduler=self.scheduler,
input_fd=master_fd,
log_file_path=build_logger.stdin,
stdout_fd=stdout_fd)
pipe_logger.start()
self._registered = True
self._main_task = asyncio.ensure_future(self._main(
build_logger, pipe_logger),
loop=self.scheduler)
self._main_task.add_done_callback(self._main_exit)
def _start(self):
pkg = self.pkg
pretend = self.pretend
bintree = pkg.root_config.trees["bintree"]
settings = bintree.settings
use_locks = "distlocks" in settings.features
pkg_path = self.pkg_path
if not pretend:
portage.util.ensure_dirs(os.path.dirname(pkg_path))
if use_locks:
self.lock()
exists = os.path.exists(pkg_path)
resume = exists and os.path.basename(pkg_path) in bintree.invalids
if not (pretend or resume):
# Remove existing file or broken symlink.
try:
os.unlink(pkg_path)
except OSError:
pass
# urljoin doesn't work correctly with
# unrecognized protocols like sftp
if bintree._remote_has_index:
instance_key = bintree.dbapi._instance_key(pkg.cpv)
rel_uri = bintree._remotepkgs[instance_key].get("PATH")
if not rel_uri:
rel_uri = pkg.cpv + ".tbz2"
remote_base_uri = bintree._remotepkgs[
instance_key]["BASE_URI"]
uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
else:
uri = settings["PORTAGE_BINHOST"].rstrip("/") + \
"/" + pkg.pf + ".tbz2"
if pretend:
portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1)
self._set_returncode((self.pid, os.EX_OK << 8))
self._async_wait()
return
protocol = urllib_parse_urlparse(uri)[0]
fcmd_prefix = "FETCHCOMMAND"
if resume:
fcmd_prefix = "RESUMECOMMAND"
fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
if not fcmd:
fcmd = settings.get(fcmd_prefix)
fcmd_vars = {
"DISTDIR" : os.path.dirname(pkg_path),
"URI" : uri,
"FILE" : os.path.basename(pkg_path)
}
for k in ("PORTAGE_SSH_OPTS",):
try:
fcmd_vars[k] = settings[k]
except KeyError:
pass
fetch_env = dict(settings.items())
fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \
for x in portage.util.shlex_split(fcmd)]
if self.fd_pipes is None:
self.fd_pipes = {}
fd_pipes = self.fd_pipes
# Redirect all output to stdout since some fetchers like
# wget pollute stderr (if portage detects a problem then it
# can send it's own message to stderr).
fd_pipes.setdefault(0, portage._get_stdin().fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stdout__.fileno())
self.args = fetch_args
self.env = fetch_env
if settings.selinux_enabled():
self._selinux_type = settings["PORTAGE_FETCH_T"]
SpawnProcess._start(self)
def pre_sync(self, repo):
msg = ">>> Syncing repository '%s' into '%s'..." \
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
try:
st = os.stat(repo.location)
except OSError:
st = None
self.usersync_uid = None
spawn_kwargs = {}
# Redirect command stderr to stdout, in order to prevent
# spurious cron job emails (bug 566132).
spawn_kwargs["fd_pipes"] = {
0: portage._get_stdin().fileno(),
1: sys.__stdout__.fileno(),
2: sys.__stdout__.fileno()
}
spawn_kwargs["env"] = self.settings.environ()
if repo.sync_user is not None:
def get_sync_user_data(sync_user):
user = None
group = None
home = None
logname = None
spl = sync_user.split(':', 1)
if spl[0]:
username = spl[0]
try:
try:
pw = pwd.getpwnam(username)
except KeyError:
pw = pwd.getpwuid(int(username))
except (ValueError, KeyError):
writemsg("!!! User '%s' invalid or does not exist\n"
% username, noiselevel=-1)
return (logname, user, group, home)
user = pw.pw_uid
group = pw.pw_gid
home = pw.pw_dir
logname = pw.pw_name
if len(spl) > 1:
groupname = spl[1]
try:
try:
gp = grp.getgrnam(groupname)
except KeyError:
pw = grp.getgrgid(int(groupname))
except (ValueError, KeyError):
writemsg("!!! Group '%s' invalid or does not exist\n"
% groupname, noiselevel=-1)
return (logname, user, group, home)
group = gp.gr_gid
return (logname, user, group, home)
# user or user:group
(logname, uid, gid, home) = get_sync_user_data(
repo.sync_user)
if uid is not None:
spawn_kwargs["uid"] = uid
self.usersync_uid = uid
if gid is not None:
spawn_kwargs["gid"] = gid
spawn_kwargs["groups"] = [gid]
if home is not None:
spawn_kwargs["env"]["HOME"] = home
if logname is not None:
spawn_kwargs["env"]["LOGNAME"] = logname
if st is None:
perms = {'mode': 0o755}
# respect sync-user if set
if 'umask' in spawn_kwargs:
perms['mode'] &= ~spawn_kwargs['umask']
if 'uid' in spawn_kwargs:
perms['uid'] = spawn_kwargs['uid']
if 'gid' in spawn_kwargs:
perms['gid'] = spawn_kwargs['gid']
portage.util.ensure_dirs(repo.location, **perms)
st = os.stat(repo.location)
if (repo.sync_user is None and
'usersync' in self.settings.features and
portage.data.secpass >= 2 and
(st.st_uid != os.getuid() and st.st_mode & 0o700 or
st.st_gid != os.getgid() and st.st_mode & 0o070)):
try:
pw = pwd.getpwuid(st.st_uid)
except KeyError:
pass
else:
# Drop privileges when syncing, in order to match
# existing uid/gid settings.
self.usersync_uid = st.st_uid
spawn_kwargs["uid"] = st.st_uid
spawn_kwargs["gid"] = st.st_gid
spawn_kwargs["groups"] = [st.st_gid]
spawn_kwargs["env"]["HOME"] = pw.pw_dir
spawn_kwargs["env"]["LOGNAME"] = pw.pw_name
umask = 0o002
if not st.st_mode & 0o020:
umask = umask | 0o020
spawn_kwargs["umask"] = umask
# override the defaults when sync_umask is set
if repo.sync_umask is not None:
spawn_kwargs["umask"] = int(repo.sync_umask, 8)
spawn_kwargs.setdefault("umask", 0o022)
self.spawn_kwargs = spawn_kwargs
if self.usersync_uid is not None:
# PORTAGE_TMPDIR is used below, so validate it and
# bail out if necessary.
rval = _check_temp_dir(self.settings)
if rval != os.EX_OK:
return rval
os.umask(0o022)
return os.EX_OK
def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
uid=None, gid=None, groups=None, umask=None, logfile=None,
path_lookup=True, pre_exec=None, close_fds=True, unshare_net=False,
unshare_ipc=False, cgroup=None):
"""
Spawns a given command.
@param mycommand: the command to execute
@type mycommand: String or List (Popen style list)
@param env: A dict of Key=Value pairs for env variables
@type env: Dictionary
@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
@type opt_name: String
@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
(default is {0:stdin, 1:stdout, 2:stderr})
@type fd_pipes: Dictionary
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
@type returnpid: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@param gid: Group ID to spawn as; useful for dropping privilages
@type gid: Integer
@param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts.
@type groups: List
@param umask: An integer representing the umask for the process (see man chmod for umask details)
@type umask: Integer
@param logfile: name of a file to use for logging purposes
@type logfile: String
@param path_lookup: If the binary is not fully specified then look for it in PATH
@type path_lookup: Boolean
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param close_fds: If True, then close all file descriptors except those
referenced by fd_pipes (default is True).
@type close_fds: Boolean
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
logfile requires stdout and stderr to be assigned to this process (ie not pointed
somewhere else.)
"""
# mycommand is either a str or a list
if isinstance(mycommand, basestring):
mycommand = mycommand.split()
if sys.hexversion < 0x3000000:
# Avoid a potential UnicodeEncodeError from os.execve().
env_bytes = {}
for k, v in env.items():
env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \
_unicode_encode(v, encoding=_encodings['content'])
env = env_bytes
del env_bytes
# If an absolute path to an executable file isn't given
# search for it unless we've been told not to.
binary = mycommand[0]
if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \
(not os.path.isabs(binary) or not os.path.isfile(binary)
or not os.access(binary, os.X_OK)):
binary = path_lookup and find_binary(binary) or None
if not binary:
raise CommandNotFound(mycommand[0])
# If we haven't been told what file descriptors to use
# default to propagating our stdin, stdout and stderr.
if fd_pipes is None:
fd_pipes = {
0:portage._get_stdin().fileno(),
1:sys.__stdout__.fileno(),
2:sys.__stderr__.fileno(),
}
# mypids will hold the pids of all processes created.
mypids = []
if logfile:
# Using a log file requires that stdout and stderr
# are assigned to the process we're running.
if 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
# Create a pipe
(pr, pw) = os.pipe()
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
mypids.extend(spawn(('tee', '-i', '-a', logfile),
returnpid=True, fd_pipes={0:pr,
1:fd_pipes[1], 2:fd_pipes[2]}))
# We don't need the read end of the pipe, so close it.
os.close(pr)
# Assign the write end of the pipe to our stdout and stderr.
fd_pipes[1] = pw
fd_pipes[2] = pw
# This caches the libc library lookup in the current
# process, so that it's only done once rather than
# for each child process.
if unshare_net or unshare_ipc:
find_library("c")
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid == 0:
try:
_exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, pre_exec, close_fds,
unshare_net, unshare_ipc, cgroup)
except SystemExit:
raise
except Exception as e:
# We need to catch _any_ exception so that it doesn't
# propagate out of this function and cause exiting
# with anything other than os._exit()
writemsg("%s:\n %s\n" % (e, " ".join(mycommand)),
noiselevel=-1)
traceback.print_exc()
sys.stderr.flush()
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" % (repr(pid),))
# Add the pid to our local and the global pid lists.
mypids.append(pid)
# If we started a tee process the write side of the pipe is no
# longer needed, so close it.
if logfile:
os.close(pw)
# If the caller wants to handle cleaning up the processes, we tell
# it about all processes that were created.
if returnpid:
return mypids
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
# in the pipe are well behaved, it will die when the process
# it is reading from dies.
pid = mypids.pop(0)
# and wait for it.
retval = os.waitpid(pid, 0)[1]
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if os.waitpid(pid, os.WNOHANG)[0] == 0:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
# If it got a signal, return the signal that was sent.
if (retval & 0xff):
return ((retval & 0xff) << 8)
# Otherwise, return its exit code.
return (retval >> 8)
# Everything succeeded
return 0
