def _setup_pipes(fd_pipes, close_fds=True):
"""Setup pipes for a forked process.
WARNING: When not followed by exec, the close_fds behavior
can trigger interference from destructors that close file
descriptors. This interference happens when the garbage
collector intermittently executes such destructors after their
corresponding file descriptors have been re-used, leading
to intermittent "[Errno 9] Bad file descriptor" exceptions in
forked processes. This problem has been observed with PyPy 1.8,
and also with CPython under some circumstances (as triggered
by xmpppy in bug #374335). In order to close a safe subset of
file descriptors, see portage.locks._close_fds().
"""
my_fds = {}
# To protect from cases where direct assignment could
# clobber needed fds ({1:2, 2:1}) we first dupe the fds
# into unused fds.
for fd in fd_pipes:
my_fds[fd] = os.dup(fd_pipes[fd])
# Then assign them to what they should be.
for fd in my_fds:
os.dup2(my_fds[fd], fd)
if close_fds:
# Then close _all_ fds that haven't been explicitly
# requested to be kept open.
for fd in get_open_fds():
if fd not in my_fds:
try:
os.close(fd)
except OSError:
pass
def _setup_pipes(fd_pipes):
"""Setup pipes for a forked process."""
my_fds = {}
# To protect from cases where direct assignment could
# clobber needed fds ({1:2, 2:1}) we first dupe the fds
# into unused fds.
for fd in fd_pipes:
my_fds[fd] = os.dup(fd_pipes[fd])
# Then assign them to what they should be.
for fd in my_fds:
os.dup2(my_fds[fd], fd)
# Then close _all_ fds that haven't been explicitly
# requested to be kept open.
for fd in get_open_fds():
if fd not in my_fds:
try:
os.close(fd)
except OSError:
pass
def __init__(self, environment):
self.conf = environment
pidfile = open(self.conf.get('pid_file'), 'w')
if (self.conf.get('no_daemon') == False):
print "forking to background"
if (os.fork() == 0):
os.setpgid(0,0);
pidfile.write(str(os.getpid()));
pidfile.close();
fd = os.open("/dev/null", os.O_WRONLY);
os.dup2(fd,1);
os.close(fd);
#self.main_loop()
else:
sys.exit()
else:
print "Keeping in foreground"
pidfile.write(str(os.getpid()));
pidfile.close();
def _setup_pipes(fd_pipes, close_fds=True):
"""Setup pipes for a forked process."""
my_fds = {}
# To protect from cases where direct assignment could
# clobber needed fds ({1:2, 2:1}) we first dupe the fds
# into unused fds.
for fd in fd_pipes:
my_fds[fd] = os.dup(fd_pipes[fd])
# Then assign them to what they should be.
for fd in my_fds:
os.dup2(my_fds[fd], fd)
if close_fds:
# Then close _all_ fds that haven't been explicitly
# requested to be kept open.
for fd in get_open_fds():
if fd not in my_fds:
try:
os.close(fd)
except OSError:
pass
def _bootstrap(self, fd_pipes):
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Unregister SIGCHLD handler and wakeup_fd for the parent
# process's event loop (bug 655656).
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
try:
wakeup_fd = signal.set_wakeup_fd(-1)
if wakeup_fd > 0:
os.close(wakeup_fd)
except (ValueError, OSError):
pass
portage.locks._close_fds()
# We don't exec, so use close_fds=False
# (see _setup_pipes docstring).
portage.process._setup_pipes(fd_pipes, close_fds=False)
# Since multiprocessing.Process closes sys.__stdin__ and
# makes sys.stdin refer to os.devnull, restore it when
# appropriate.
if 0 in fd_pipes:
# It's possible that sys.stdin.fileno() is already 0,
# and in that case the above _setup_pipes call will
# have already updated its identity via dup2. Otherwise,
# perform the dup2 call now, and also copy the file
# descriptor flags.
if sys.stdin.fileno() != 0:
os.dup2(0, sys.stdin.fileno())
fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFD,
fcntl.fcntl(0, fcntl.F_GETFD))
sys.__stdin__ = sys.stdin
sys.exit(self._run())
def _setup_pipes(fd_pipes, close_fds=True, inheritable=None):
"""Setup pipes for a forked process.
Even when close_fds is False, file descriptors referenced as
values in fd_pipes are automatically closed if they do not also
occur as keys in fd_pipes. It is assumed that the caller will
explicitly add them to the fd_pipes keys if they are intended
to remain open. This allows for convenient elimination of
unnecessary duplicate file descriptors.
WARNING: When not followed by exec, the close_fds behavior
can trigger interference from destructors that close file
descriptors. This interference happens when the garbage
collector intermittently executes such destructors after their
corresponding file descriptors have been re-used, leading
to intermittent "[Errno 9] Bad file descriptor" exceptions in
forked processes. This problem has been observed with PyPy 1.8,
and also with CPython under some circumstances (as triggered
by xmpppy in bug #374335). In order to close a safe subset of
file descriptors, see portage.locks._close_fds().
NOTE: When not followed by exec, even when close_fds is False,
it's still possible for dup2() calls to cause interference in a
way that's similar to the way that close_fds interferes (since
dup2() has to close the target fd if it happens to be open).
It's possible to avoid such interference by using allocated
file descriptors as the keys in fd_pipes. For example:
pr, pw = os.pipe()
fd_pipes[pw] = pw
By using the allocated pw file descriptor as the key in fd_pipes,
it's not necessary for dup2() to close a file descriptor (it
actually does nothing in this case), which avoids possible
interference.
"""
reverse_map = {}
# To protect from cases where direct assignment could
# clobber needed fds ({1:2, 2:1}) we create a reverse map
# in order to know when it's necessary to create temporary
# backup copies with os.dup().
for newfd, oldfd in fd_pipes.items():
newfds = reverse_map.get(oldfd)
if newfds is None:
newfds = []
reverse_map[oldfd] = newfds
newfds.append(newfd)
# Assign newfds via dup2(), making temporary backups when
# necessary, and closing oldfd if the caller has not
# explicitly requested for it to remain open by adding
# it to the keys of fd_pipes.
while reverse_map:
oldfd, newfds = reverse_map.popitem()
old_fdflags = None
for newfd in newfds:
if newfd in reverse_map:
# Make a temporary backup before re-assignment, assuming
# that backup_fd won't collide with a key in reverse_map
# (since all of the keys correspond to open file
# descriptors, and os.dup() only allocates a previously
# unused file discriptors).
backup_fd = os.dup(newfd)
reverse_map[backup_fd] = reverse_map.pop(newfd)
if oldfd != newfd:
os.dup2(oldfd, newfd)
if _set_inheritable is not None:
# Don't do this unless _set_inheritable is available,
# since it's used below to ensure correct state, and
# otherwise /dev/null stdin fails to inherit (at least
# with Python versions from 3.1 to 3.3).
if old_fdflags is None:
old_fdflags = fcntl.fcntl(oldfd, fcntl.F_GETFD)
fcntl.fcntl(newfd, fcntl.F_SETFD, old_fdflags)
if _set_inheritable is not None:
inheritable_state = None
if not (old_fdflags is None or _FD_CLOEXEC is None):
inheritable_state = not bool(old_fdflags & _FD_CLOEXEC)
if inheritable is not None:
if inheritable_state is not inheritable:
_set_inheritable(newfd, inheritable)
elif newfd in (0, 1, 2):
if inheritable_state is not True:
_set_inheritable(newfd, True)
if oldfd not in fd_pipes:
# If oldfd is not a key in fd_pipes, then it's safe
# to close now, since we've already made all of the
# requested duplicates. This also closes every
# backup_fd that may have been created on previous
# iterations of this loop.
os.close(oldfd)
if close_fds:
# Then close _all_ fds that haven't been explicitly
# requested to be kept open.
for fd in get_open_fds():
if fd not in fd_pipes:
try:
os.close(fd)
except OSError:
pass
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
pre_exec):
"""
Execute a given binary with options
@param binary: Name of program to execute
@type binary: String
@param mycommand: Options for program
@type mycommand: String
@param opt_name: Name of process (defaults to binary)
@type opt_name: String
@param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 }
@type fd_pipes: Dictionary
@param env: Key,Value mapping for Environmental Variables
@type env: Dictionary
@param gid: Group ID to run the process under
@type gid: Integer
@param groups: Groups the Process should be in.
@type groups: Integer
@param uid: User ID to run the process under
@type uid: Integer
@param umask: an int representing a unix umask (see man chmod for umask details)
@type umask: Integer
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@rtype: None
@returns: Never returns (calls os.execve)
"""
# If the process we're creating hasn't been given a name
# assign it the name of the executable.
if not opt_name:
opt_name = os.path.basename(binary)
# Set up the command's argument list.
myargs = [opt_name]
myargs.extend(mycommand[1:])
# Set up the command's pipes.
my_fds = {}
# To protect from cases where direct assignment could
# clobber needed fds ({1:2, 2:1}) we first dupe the fds
# into unused fds.
for fd in fd_pipes:
my_fds[fd] = os.dup(fd_pipes[fd])
# Then assign them to what they should be.
for fd in my_fds:
os.dup2(my_fds[fd], fd)
# Then close _all_ fds that haven't been explictly
# requested to be kept open.
for fd in get_open_fds():
if fd not in my_fds:
try:
os.close(fd)
except OSError:
pass
# Set requested process permissions.
if gid:
os.setgid(gid)
if groups:
os.setgroups(groups)
if uid:
os.setuid(uid)
if umask:
os.umask(umask)
if pre_exec:
pre_exec()
# And switch to the new process.
os.execve(binary, myargs, env)
def _setup_pipes(fd_pipes, close_fds=True, inheritable=None):
"""Setup pipes for a forked process.
Even when close_fds is False, file descriptors referenced as
values in fd_pipes are automatically closed if they do not also
occur as keys in fd_pipes. It is assumed that the caller will
explicitly add them to the fd_pipes keys if they are intended
to remain open. This allows for convenient elimination of
unnecessary duplicate file descriptors.
WARNING: When not followed by exec, the close_fds behavior
can trigger interference from destructors that close file
descriptors. This interference happens when the garbage
collector intermittently executes such destructors after their
corresponding file descriptors have been re-used, leading
to intermittent "[Errno 9] Bad file descriptor" exceptions in
forked processes. This problem has been observed with PyPy 1.8,
and also with CPython under some circumstances (as triggered
by xmpppy in bug #374335). In order to close a safe subset of
file descriptors, see portage.locks._close_fds().
NOTE: When not followed by exec, even when close_fds is False,
it's still possible for dup2() calls to cause interference in a
way that's similar to the way that close_fds interferes (since
dup2() has to close the target fd if it happens to be open).
It's possible to avoid such interference by using allocated
file descriptors as the keys in fd_pipes. For example:
pr, pw = os.pipe()
fd_pipes[pw] = pw
By using the allocated pw file descriptor as the key in fd_pipes,
it's not necessary for dup2() to close a file descriptor (it
actually does nothing in this case), which avoids possible
interference.
"""
reverse_map = {}
# To protect from cases where direct assignment could
# clobber needed fds ({1:2, 2:1}) we create a reverse map
# in order to know when it's necessary to create temporary
# backup copies with os.dup().
for newfd, oldfd in fd_pipes.items():
newfds = reverse_map.get(oldfd)
if newfds is None:
newfds = []
reverse_map[oldfd] = newfds
newfds.append(newfd)
# Assign newfds via dup2(), making temporary backups when
# necessary, and closing oldfd if the caller has not
# explicitly requested for it to remain open by adding
# it to the keys of fd_pipes.
while reverse_map:
oldfd, newfds = reverse_map.popitem()
old_fdflags = None
for newfd in newfds:
if newfd in reverse_map:
# Make a temporary backup before re-assignment, assuming
# that backup_fd won't collide with a key in reverse_map
# (since all of the keys correspond to open file
# descriptors, and os.dup() only allocates a previously
# unused file discriptors).
backup_fd = os.dup(newfd)
reverse_map[backup_fd] = reverse_map.pop(newfd)
if oldfd != newfd:
os.dup2(oldfd, newfd)
if _set_inheritable is not None:
# Don't do this unless _set_inheritable is available,
# since it's used below to ensure correct state, and
# otherwise /dev/null stdin fails to inherit (at least
# with Python versions from 3.1 to 3.3).
if old_fdflags is None:
old_fdflags = fcntl.fcntl(oldfd, fcntl.F_GETFD)
fcntl.fcntl(newfd, fcntl.F_SETFD, old_fdflags)
if _set_inheritable is not None:
inheritable_state = None
if not (old_fdflags is None or _FD_CLOEXEC is None):
inheritable_state = not bool(old_fdflags & _FD_CLOEXEC)
if inheritable is not None:
if inheritable_state is not inheritable:
_set_inheritable(newfd, inheritable)
elif newfd in (0, 1, 2):
if inheritable_state is not True:
_set_inheritable(newfd, True)
if oldfd not in fd_pipes:
# If oldfd is not a key in fd_pipes, then it's safe
# to close now, since we've already made all of the
# requested duplicates. This also closes every
# backup_fd that may have been created on previous
# iterations of this loop.
os.close(oldfd)
if close_fds:
# Then close _all_ fds that haven't been explicitly
# requested to be kept open.
for fd in get_open_fds():
if fd not in fd_pipes:
try:
os.close(fd)
except OSError:
pass
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
pre_exec):
"""
Execute a given binary with options
@param binary: Name of program to execute
@type binary: String
@param mycommand: Options for program
@type mycommand: String
@param opt_name: Name of process (defaults to binary)
@type opt_name: String
@param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 }
@type fd_pipes: Dictionary
@param env: Key,Value mapping for Environmental Variables
@type env: Dictionary
@param gid: Group ID to run the process under
@type gid: Integer
@param groups: Groups the Process should be in.
@type groups: Integer
@param uid: User ID to run the process under
@type uid: Integer
@param umask: an int representing a unix umask (see man chmod for umask details)
@type umask: Integer
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@rtype: None
@returns: Never returns (calls os.execve)
"""
# If the process we're creating hasn't been given a name
# assign it the name of the executable.
if not opt_name:
opt_name = os.path.basename(binary)
# Set up the command's argument list.
myargs = [opt_name]
myargs.extend(mycommand[1:])
# Set up the command's pipes.
my_fds = {}
# To protect from cases where direct assignment could
# clobber needed fds ({1:2, 2:1}) we first dupe the fds
# into unused fds.
for fd in fd_pipes:
my_fds[fd] = os.dup(fd_pipes[fd])
# Then assign them to what they should be.
for fd in my_fds:
os.dup2(my_fds[fd], fd)
# Then close _all_ fds that haven't been explictly
# requested to be kept open.
for fd in get_open_fds():
if fd not in my_fds:
try:
os.close(fd)
except OSError:
pass
# Set requested process permissions.
if gid:
os.setgid(gid)
if groups:
os.setgroups(groups)
if uid:
os.setuid(uid)
if umask:
os.umask(umask)
if pre_exec:
pre_exec()
# And switch to the new process.
os.execve(binary, myargs, env)
