def _testLockNonblock(self): tempdir = tempfile.mkdtemp() try: path = os.path.join(tempdir, 'lock_me') lock1 = portage.locks.lockfile(path) pid = os.fork() if pid == 0: portage.locks._close_fds() # Disable close_fds since we don't exec # (see _setup_pipes docstring). portage.process._setup_pipes({0:0, 1:1, 2:2}, close_fds=False) rval = 2 try: try: lock2 = portage.locks.lockfile(path, flags=os.O_NONBLOCK) except portage.exception.TryAgain: rval = os.EX_OK else: rval = 1 portage.locks.unlockfile(lock2) except SystemExit: raise except: traceback.print_exc() finally: os._exit(rval) self.assertEqual(pid > 0, True) pid, status = os.waitpid(pid, 0) self.assertEqual(os.WIFEXITED(status), True) self.assertEqual(os.WEXITSTATUS(status), os.EX_OK) portage.locks.unlockfile(lock1) finally: shutil.rmtree(tempdir)
def _create_pty_or_pipe(copy_term_size=None): """ Try to create a pty and if then fails then create a normal pipe instead. @param copy_term_size: If a tty file descriptor is given then the term size will be copied to the pty. @type copy_term_size: int @rtype: tuple @return: A tuple of (is_pty, master_fd, slave_fd) where is_pty is True if a pty was successfully allocated, and False if a normal pipe was allocated. """ got_pty = False global _disable_openpty, _fbsd_test_pty if _fbsd_test_pty and not _disable_openpty: # Test for python openpty breakage after freebsd7 to freebsd8 # upgrade, which results in a 'Function not implemented' error # and the process being killed. pid = os.fork() if pid == 0: pty.openpty() os._exit(os.EX_OK) pid, status = os.waitpid(pid, 0) if (status & 0xff) == 140: _disable_openpty = True _fbsd_test_pty = False if _disable_openpty: master_fd, slave_fd = os.pipe() else: try: master_fd, slave_fd = pty.openpty() got_pty = True except EnvironmentError as e: _disable_openpty = True writemsg("openpty failed: '%s'\n" % str(e), noiselevel=-1) del e master_fd, slave_fd = os.pipe() if got_pty: # Disable post-processing of output since otherwise weird # things like \n -> \r\n transformations may occur. mode = termios.tcgetattr(slave_fd) mode[1] &= ~termios.OPOST termios.tcsetattr(slave_fd, termios.TCSANOW, mode) if got_pty and \ copy_term_size is not None and \ os.isatty(copy_term_size): rows, columns = get_term_size() set_term_size(rows, columns, slave_fd) return (got_pty, master_fd, slave_fd)
def _create_pty_or_pipe(copy_term_size=None): """ Try to create a pty and if then fails then create a normal pipe instead. @param copy_term_size: If a tty file descriptor is given then the term size will be copied to the pty. @type copy_term_size: int @rtype: tuple @returns: A tuple of (is_pty, master_fd, slave_fd) where is_pty is True if a pty was successfully allocated, and False if a normal pipe was allocated. """ got_pty = False global _disable_openpty, _fbsd_test_pty if _fbsd_test_pty and not _disable_openpty: # Test for python openpty breakage after freebsd7 to freebsd8 # upgrade, which results in a 'Function not implemented' error # and the process being killed. pid = os.fork() if pid == 0: pty.openpty() os._exit(os.EX_OK) pid, status = os.waitpid(pid, 0) if (status & 0xff) == 140: _disable_openpty = True _fbsd_test_pty = False if _disable_openpty: master_fd, slave_fd = os.pipe() else: try: master_fd, slave_fd = pty.openpty() got_pty = True except EnvironmentError as e: _disable_openpty = True writemsg("openpty failed: '%s'\n" % str(e), noiselevel=-1) del e master_fd, slave_fd = os.pipe() if got_pty: # Disable post-processing of output since otherwise weird # things like \n -> \r\n transformations may occur. mode = termios.tcgetattr(slave_fd) mode[1] &= ~termios.OPOST termios.tcsetattr(slave_fd, termios.TCSANOW, mode) if got_pty and \ copy_term_size is not None and \ os.isatty(copy_term_size): rows, columns = get_term_size() set_term_size(rows, columns, slave_fd) return (got_pty, master_fd, slave_fd)
def _spawn(self, args, fd_pipes=None, **kwargs): """ Fork a subprocess, apply local settings, and call fetch(). """ parent_pid = os.getpid() pid = None try: pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError( "fork returned non-integer: %s" % (repr(pid),)) return [pid] rval = 1 try: # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Unregister SIGCHLD handler and wakeup_fd for the parent # process's event loop (bug 655656). signal.signal(signal.SIGCHLD, signal.SIG_DFL) try: wakeup_fd = signal.set_wakeup_fd(-1) if wakeup_fd > 0: os.close(wakeup_fd) except (ValueError, OSError): pass portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) rval = self._run() except SystemExit: raise except: traceback.print_exc() # os._exit() skips stderr flush! sys.stderr.flush() finally: os._exit(rval) finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1)
def _spawn(self, args, fd_pipes=None, **kwargs): """ Fork a subprocess, apply local settings, and call fetch(). """ pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError( "fork returned non-integer: %s" % (repr(pid), )) portage.process.spawned_pids.append(pid) return [pid] # TODO: Find out why PyPy 1.8 with close_fds=True triggers # "[Errno 9] Bad file descriptor" in subprocesses. It could # be due to garbage collection of file objects that were not # closed before going out of scope, since PyPy's garbage # collector does not support the refcounting semantics that # CPython does. close_fds = platform.python_implementation() != 'PyPy' portage.process._setup_pipes(fd_pipes, close_fds=close_fds) # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Force consistent color output, in case we are capturing fetch # output through a normal pipe due to unavailability of ptys. portage.output.havecolor = self._settings.get('NOCOLOR') \ not in ('yes', 'true') rval = 1 allow_missing = self._get_manifest().allow_missing try: if fetch( self._uri_map, self._settings, fetchonly=self.fetchonly, digests=copy.deepcopy(self._get_digests()), allow_missing_digests=allow_missing): rval = os.EX_OK except SystemExit: raise except: traceback.print_exc() finally: # Call os._exit() from finally block, in order to suppress any # finally blocks from earlier in the call stack. See bug #345289. os._exit(rval)
def _spawn(self, args, fd_pipes=None, **kwargs): """ Fork a subprocess, apply local settings, and call fetch(). """ pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid), )) portage.process.spawned_pids.append(pid) return [pid] # TODO: Find out why PyPy 1.8 with close_fds=True triggers # "[Errno 9] Bad file descriptor" in subprocesses. It could # be due to garbage collection of file objects that were not # closed before going out of scope, since PyPy's garbage # collector does not support the refcounting semantics that # CPython does. close_fds = platform.python_implementation() != 'PyPy' portage.process._setup_pipes(fd_pipes, close_fds=close_fds) # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Force consistent color output, in case we are capturing fetch # output through a normal pipe due to unavailability of ptys. portage.output.havecolor = self._settings.get('NOCOLOR') \ not in ('yes', 'true') rval = 1 allow_missing = self._get_manifest().allow_missing try: if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly, digests=copy.deepcopy(self._get_digests()), allow_missing_digests=allow_missing): rval = os.EX_OK except SystemExit: raise except: traceback.print_exc() finally: # Call os._exit() from finally block, in order to suppress any # finally blocks from earlier in the call stack. See bug #345289. os._exit(rval)
def _spawn(self, args, fd_pipes=None, **kwargs): """ Fork a subprocess, apply local settings, and call fetch(). """ parent_pid = os.getpid() pid = None try: pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError( "fork returned non-integer: %s" % (repr(pid),)) portage.process.spawned_pids.append(pid) return [pid] rval = 1 try: # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) rval = self._run() except SystemExit: raise except: traceback.print_exc() finally: os._exit(rval) finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1)
def _spawn(self, args, fd_pipes=None, **kwargs): """ Fork a subprocess, apply local settings, and call fetch(). """ pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid), )) portage.process.spawned_pids.append(pid) return [pid] portage.locks._close_fds() # Disable close_fds since we don't exec (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Force consistent color output, in case we are capturing fetch # output through a normal pipe due to unavailability of ptys. portage.output.havecolor = self._settings.get('NOCOLOR') \ not in ('yes', 'true') rval = 1 allow_missing = self._get_manifest().allow_missing try: if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly, digests=copy.deepcopy(self._get_digests()), allow_missing_digests=allow_missing): rval = os.EX_OK except SystemExit: raise except: traceback.print_exc() finally: # Call os._exit() from finally block, in order to suppress any # finally blocks from earlier in the call stack. See bug #345289. os._exit(rval)
def _spawn(self, args, fd_pipes=None, **kwargs): """ Fork a subprocess, apply local settings, and call fetch(). """ pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError( "fork returned non-integer: %s" % (repr(pid),)) portage.process.spawned_pids.append(pid) return [pid] portage.locks._close_fds() # Disable close_fds since we don't exec (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Force consistent color output, in case we are capturing fetch # output through a normal pipe due to unavailability of ptys. portage.output.havecolor = self._settings.get('NOCOLOR') \ not in ('yes', 'true') rval = 1 allow_missing = self._get_manifest().allow_missing try: if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly, digests=copy.deepcopy(self._get_digests()), allow_missing_digests=allow_missing): rval = os.EX_OK except SystemExit: raise except: traceback.print_exc() finally: # Call os._exit() from finally block, in order to suppress any # finally blocks from earlier in the call stack. See bug #345289. os._exit(rval)
def __init__(self, environment): self.conf = environment pidfile = open(self.conf.get('pid_file'), 'w') if (self.conf.get('no_daemon') == False): print "forking to background" if (os.fork() == 0): os.setpgid(0,0); pidfile.write(str(os.getpid())); pidfile.close(); fd = os.open("/dev/null", os.O_WRONLY); os.dup2(fd,1); os.close(fd); #self.main_loop() else: sys.exit() else: print "Keeping in foreground" pidfile.write(str(os.getpid())); pidfile.close();
def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, logfile=None, path_lookup=True, pre_exec=None): """ Spawns a given command. @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: A dict of Key=Value pairs for env variables @type env: Dictionary @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @param gid: Group ID to spawn as; useful for dropping privilages @type gid: Integer @param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts. @type groups: List @param umask: An integer representing the umask for the process (see man chmod for umask details) @type umask: Integer @param logfile: name of a file to use for logging purposes @type logfile: String @param path_lookup: If the binary is not fully specified then look for it in PATH @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) """ # mycommand is either a str or a list if isinstance(mycommand, basestring): mycommand = mycommand.split() if sys.hexversion < 0x3000000: # Avoid a potential UnicodeEncodeError from os.execve(). env_bytes = {} for k, v in env.items(): env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \ _unicode_encode(v, encoding=_encodings['content']) env = env_bytes del env_bytes # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \ (not os.path.isabs(binary) or not os.path.isfile(binary) or not os.access(binary, os.X_OK)): binary = path_lookup and find_binary(binary) or None if not binary: raise CommandNotFound(mycommand[0]) # If we haven't been told what file descriptors to use # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { 0:sys.__stdin__.fileno(), 1:sys.__stdout__.fileno(), 2:sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. mypids = [] if logfile: # Using a log file requires that stdout and stderr # are assigned to the process we're running. if 1 not in fd_pipes or 2 not in fd_pipes: raise ValueError(fd_pipes) # Create a pipe (pr, pw) = os.pipe() # Create a tee process, giving it our stdout and stderr # as well as the read end of the pipe. mypids.extend(spawn(('tee', '-i', '-a', logfile), returnpid=True, fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]})) # We don't need the read end of the pipe, so close it. os.close(pr) # Assign the write end of the pipe to our stdout and stderr. fd_pipes[1] = pw fd_pipes[2] = pw pid = os.fork() if pid == 0: try: _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, pre_exec) except SystemExit: raise except Exception as e: # We need to catch _any_ exception so that it doesn't # propagate out of this function and cause exiting # with anything other than os._exit() sys.stderr.write("%s:\n %s\n" % (e, " ".join(mycommand))) traceback.print_exc() sys.stderr.flush() os._exit(1) if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid),)) # Add the pid to our local and the global pid lists. mypids.append(pid) spawned_pids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. if logfile: os.close(pw) # If the caller wants to handle cleaning up the processes, we tell # it about all processes that were created. if returnpid: return mypids # Otherwise we clean them up. while mypids: # Pull the last reader in the pipe chain. If all processes # in the pipe are well behaved, it will die when the process # it is reading from dies. pid = mypids.pop(0) # and wait for it. retval = os.waitpid(pid, 0)[1] # When it's done, we can remove it from the # global pid list as well. spawned_pids.remove(pid) if retval: # If it failed, kill off anything else that # isn't dead yet. for pid in mypids: # With waitpid and WNOHANG, only check the # first element of the tuple since the second # element may vary (bug #337465). if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) spawned_pids.remove(pid) # If it got a signal, return the signal that was sent. if (retval & 0xff): return ((retval & 0xff) << 8) # Otherwise, return its exit code. return (retval >> 8) # Everything succeeded return 0
def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call dblink.merge(). TODO: Share code with ForkProcess. """ elog_reader_fd, elog_writer_fd = os.pipe() fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) blockers = None if self.blockers is not None: # Query blockers in the main process, since closing # of file descriptors in the subprocess can prevent # access to open database connections such as that # used by the sqlite metadata cache module. blockers = self.blockers() mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, blockers=blockers, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd self._elog_reg_id = self.scheduler.io_add_watch( elog_reader_fd, self._registered_events, self._elog_output_handler) # If a concurrent emerge process tries to install a package # in the same SLOT as this one at the same time, there is an # extremely unlikely chance that the COUNTER values will not be # ordered correctly unless we lock the vdb here. # FEATURES=parallel-install skips this lock in order to # improve performance, and the risk is practically negligible. self._lock_vdb() counter = None if not self.unmerge: counter = self.vartree.dbapi.counter_tick() parent_pid = os.getpid() pid = None try: pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid), )) os.close(elog_writer_fd) self._elog_reader_fd = elog_reader_fd self._buf = "" self._elog_keys = set() # invalidate relevant vardbapi caches if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None self.vartree.dbapi._pkgs_changed = True self.vartree.dbapi._clear_pkg_cache(mylink) portage.process.spawned_pids.append(pid) return [pid] os.close(elog_reader_fd) # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) portage.output.havecolor = self.settings.get('NOCOLOR') \ not in ('yes', 'true') # Avoid wastful updates of the vdb cache. self.vartree.dbapi._flush_cache_enabled = False # In this subprocess we don't want PORTAGE_BACKGROUND to # suppress stdout/stderr output since they are pipes. We # also don't want to open PORTAGE_LOG_FILE, since it will # already be opened by the parent process, so we set the # "subprocess" value for use in conditional logging code # involving PORTAGE_LOG_FILE. if not self.unmerge: # unmerge phases have separate logs if self.settings.get("PORTAGE_BACKGROUND") == "1": self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" else: self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") self.settings["PORTAGE_BACKGROUND"] = "subprocess" self.settings.backup_changes("PORTAGE_BACKGROUND") rval = 1 try: if self.unmerge: if not mylink.exists(): rval = os.EX_OK elif mylink.unmerge( ldpath_mtimes=self.prev_mtimes) == os.EX_OK: mylink.lockdb() try: mylink.delete() finally: mylink.unlockdb() rval = os.EX_OK else: rval = mylink.merge(self.pkgloc, self.infloc, myebuild=self.myebuild, mydbapi=self.mydbapi, prev_mtimes=self.prev_mtimes, counter=counter) except SystemExit: raise except: traceback.print_exc() finally: os._exit(rval) finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1)
def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call dblink.merge(). TODO: Share code with ForkProcess. """ elog_reader_fd, elog_writer_fd = os.pipe() fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) blockers = None if self.blockers is not None: # Query blockers in the main process, since closing # of file descriptors in the subprocess can prevent # access to open database connections such as that # used by the sqlite metadata cache module. blockers = self.blockers() mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, blockers=blockers, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd self._elog_reg_id = self.scheduler.io_add_watch(elog_reader_fd, self._registered_events, self._elog_output_handler) # If a concurrent emerge process tries to install a package # in the same SLOT as this one at the same time, there is an # extremely unlikely chance that the COUNTER values will not be # ordered correctly unless we lock the vdb here. # FEATURES=parallel-install skips this lock in order to # improve performance, and the risk is practically negligible. self._lock_vdb() counter = None if not self.unmerge: counter = self.vartree.dbapi.counter_tick() parent_pid = os.getpid() pid = None try: pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError( "fork returned non-integer: %s" % (repr(pid),)) os.close(elog_writer_fd) self._elog_reader_fd = elog_reader_fd self._buf = "" self._elog_keys = set() # invalidate relevant vardbapi caches if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None self.vartree.dbapi._pkgs_changed = True self.vartree.dbapi._clear_pkg_cache(mylink) portage.process.spawned_pids.append(pid) return [pid] os.close(elog_reader_fd) # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) portage.output.havecolor = self.settings.get('NOCOLOR') \ not in ('yes', 'true') # Avoid wastful updates of the vdb cache. self.vartree.dbapi._flush_cache_enabled = False # In this subprocess we don't want PORTAGE_BACKGROUND to # suppress stdout/stderr output since they are pipes. We # also don't want to open PORTAGE_LOG_FILE, since it will # already be opened by the parent process, so we set the # "subprocess" value for use in conditional logging code # involving PORTAGE_LOG_FILE. if not self.unmerge: # unmerge phases have separate logs if self.settings.get("PORTAGE_BACKGROUND") == "1": self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" else: self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") self.settings["PORTAGE_BACKGROUND"] = "subprocess" self.settings.backup_changes("PORTAGE_BACKGROUND") rval = 1 try: if self.unmerge: if not mylink.exists(): rval = os.EX_OK elif mylink.unmerge( ldpath_mtimes=self.prev_mtimes) == os.EX_OK: mylink.lockdb() try: mylink.delete() finally: mylink.unlockdb() rval = os.EX_OK else: rval = mylink.merge(self.pkgloc, self.infloc, myebuild=self.myebuild, mydbapi=self.mydbapi, prev_mtimes=self.prev_mtimes, counter=counter) except SystemExit: raise except: traceback.print_exc() finally: os._exit(rval) finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1)
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid, unshare_flags, cgroup): """ Execute a given binary with options @param binary: Name of program to execute @type binary: String @param mycommand: Options for program @type mycommand: String @param opt_name: Name of process (defaults to binary) @type opt_name: String @param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 } @type fd_pipes: Dictionary @param env: Key,Value mapping for Environmental Variables @type env: Dictionary @param gid: Group ID to run the process under @type gid: Integer @param groups: Groups the Process should be in. @type groups: List @param uid: User ID to run the process under @type uid: Integer @param umask: an int representing a unix umask (see man chmod for umask details) @type umask: Integer @param cwd: Current working directory @type cwd: String @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param unshare_mount: If True, mount namespace will be unshared and mounts will be private to the namespace @type unshare_mount: Boolean @param unshare_pid: If True, PID ns will be unshared from the spawned process @type unshare_pid: Boolean @param unshare_flags: Flags for the unshare(2) function @type unshare_flags: Integer @param cgroup: CGroup path to bind the process to @type cgroup: String @rtype: None @return: Never returns (calls os.execve) """ # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: if binary is portage._python_interpreter: # NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0] # does not contain the full path of the binary. opt_name = binary else: opt_name = os.path.basename(binary) # Set up the command's argument list. myargs = [opt_name] myargs.extend(mycommand[1:]) # Avoid a potential UnicodeEncodeError from os.execve(). myargs = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict') for x in myargs] # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Unregister SIGCHLD handler and wakeup_fd for the parent # process's event loop (bug 655656). signal.signal(signal.SIGCHLD, signal.SIG_DFL) try: wakeup_fd = signal.set_wakeup_fd(-1) if wakeup_fd > 0: os.close(wakeup_fd) except (ValueError, OSError): pass # Quiet killing of subprocesses by SIGPIPE (see bug #309001). signal.signal(signal.SIGPIPE, signal.SIG_DFL) # Avoid issues triggered by inheritance of SIGQUIT handler from # the parent process (see bug #289486). signal.signal(signal.SIGQUIT, signal.SIG_DFL) _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True) # Add to cgroup # it's better to do it from the child since we can guarantee # it is done before we start forking children if cgroup: with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: f.write('%d\n' % os.getpid()) # Unshare (while still uid==0) if unshare_net or unshare_ipc or unshare_mount or unshare_pid: filename = find_library("c") if filename is not None: libc = LoadLibrary(filename) if libc is not None: try: # Since a failed unshare call could corrupt process # state, first validate that the call can succeed. # The parent process should call _unshare_validate # before it forks, so that all child processes can # reuse _unshare_validate results that have been # cached by the parent process. errno_value = _unshare_validate(unshare_flags) if errno_value == 0 and libc.unshare(unshare_flags) != 0: errno_value = ctypes.get_errno() if errno_value != 0: involved_features = [] if unshare_ipc: involved_features.append('ipc-sandbox') if unshare_mount: involved_features.append('mount-sandbox') if unshare_net: involved_features.append('network-sandbox') if unshare_pid: involved_features.append('pid-sandbox') writemsg("Unable to unshare: %s (for FEATURES=\"%s\")\n" % ( errno.errorcode.get(errno_value, '?'), ' '.join(involved_features)), noiselevel=-1) else: if unshare_pid: main_child_pid = os.fork() if main_child_pid == 0: # pid namespace requires us to become init binary, myargs = portage._python_interpreter, [ portage._python_interpreter, os.path.join(portage._bin_path, 'pid-ns-init'), _unicode_encode('' if uid is None else str(uid)), _unicode_encode('' if gid is None else str(gid)), _unicode_encode('' if groups is None else ','.join(str(group) for group in groups)), _unicode_encode('' if umask is None else str(umask)), _unicode_encode(','.join(str(fd) for fd in fd_pipes)), binary] + myargs uid = None gid = None groups = None umask = None else: # Execute a supervisor process which will forward # signals to init and forward exit status to the # parent process. The supervisor process runs in # the global pid namespace, so skip /proc remount # and other setup that's intended only for the # init process. binary, myargs = portage._python_interpreter, [ portage._python_interpreter, os.path.join(portage._bin_path, 'pid-ns-init'), str(main_child_pid)] os.execve(binary, myargs, env) if unshare_mount: # mark the whole filesystem as slave to avoid # mounts escaping the namespace s = subprocess.Popen(['mount', '--make-rslave', '/']) mount_ret = s.wait() if mount_ret != 0: # TODO: should it be fatal maybe? writemsg("Unable to mark mounts slave: %d\n" % (mount_ret,), noiselevel=-1) if unshare_pid: # we need at least /proc being slave s = subprocess.Popen(['mount', '--make-slave', '/proc']) mount_ret = s.wait() if mount_ret != 0: # can't proceed with shared /proc writemsg("Unable to mark /proc slave: %d\n" % (mount_ret,), noiselevel=-1) os._exit(1) # mount new /proc for our namespace s = subprocess.Popen(['mount', '-n', '-t', 'proc', 'proc', '/proc']) mount_ret = s.wait() if mount_ret != 0: writemsg("Unable to mount new /proc: %d\n" % (mount_ret,), noiselevel=-1) os._exit(1) if unshare_net: # use 'localhost' to avoid hostname resolution problems try: socket.sethostname('localhost') except Exception as e: writemsg("Unable to set hostname: %s (for FEATURES=\"network-sandbox\")\n" % ( e,), noiselevel=-1) _configure_loopback_interface() except AttributeError: # unshare() not supported by libc pass # Set requested process permissions. if gid: # Cast proxies to int, in case it matters. os.setgid(int(gid)) if groups: os.setgroups(groups) if uid: # Cast proxies to int, in case it matters. os.setuid(int(uid)) if umask: os.umask(umask) if cwd is not None: os.chdir(cwd) if pre_exec: pre_exec() # And switch to the new process. os.execve(binary, myargs, env)
def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, cwd=None, logfile=None, path_lookup=True, pre_exec=None, close_fds=(sys.version_info < (3, 4)), unshare_net=False, unshare_ipc=False, unshare_mount=False, unshare_pid=False, cgroup=None): """ Spawns a given command. @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: If env is not None, it must be a mapping that defines the environment variables for the new process; these are used instead of the default behavior of inheriting the current process's environment. @type env: None or Mapping @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example (default is {0:stdin, 1:stdout, 2:stderr}) @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @param gid: Group ID to spawn as; useful for dropping privilages @type gid: Integer @param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts. @type groups: List @param umask: An integer representing the umask for the process (see man chmod for umask details) @type umask: Integer @param cwd: Current working directory @type cwd: String @param logfile: name of a file to use for logging purposes @type logfile: String @param path_lookup: If the binary is not fully specified then look for it in PATH @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param close_fds: If True, then close all file descriptors except those referenced by fd_pipes (default is True for python3.3 and earlier, and False for python3.4 and later due to non-inheritable file descriptor behavior from PEP 446). @type close_fds: Boolean @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param unshare_mount: If True, mount namespace will be unshared and mounts will be private to the namespace @type unshare_mount: Boolean @param unshare_pid: If True, PID ns will be unshared from the spawned process @type unshare_pid: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) """ # mycommand is either a str or a list if isinstance(mycommand, basestring): mycommand = mycommand.split() env = os.environ if env is None else env if sys.hexversion < 0x3000000: # Avoid a potential UnicodeEncodeError from os.execve(). env_bytes = {} for k, v in env.items(): env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \ _unicode_encode(v, encoding=_encodings['content']) env = env_bytes del env_bytes # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \ (not os.path.isabs(binary) or not os.path.isfile(binary) or not os.access(binary, os.X_OK)): binary = path_lookup and find_binary(binary) or None if not binary: raise CommandNotFound(mycommand[0]) # If we haven't been told what file descriptors to use # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { 0:portage._get_stdin().fileno(), 1:sys.__stdout__.fileno(), 2:sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. mypids = [] if logfile: # Using a log file requires that stdout and stderr # are assigned to the process we're running. if 1 not in fd_pipes or 2 not in fd_pipes: raise ValueError(fd_pipes) # Create a pipe (pr, pw) = os.pipe() # Create a tee process, giving it our stdout and stderr # as well as the read end of the pipe. mypids.extend(spawn(('tee', '-i', '-a', logfile), returnpid=True, fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]})) # We don't need the read end of the pipe, so close it. os.close(pr) # Assign the write end of the pipe to our stdout and stderr. fd_pipes[1] = pw fd_pipes[2] = pw # Cache _has_ipv6() result for use in child processes. _has_ipv6() # This caches the libc library lookup and _unshare_validator results # in the current process, so that results are cached for use in # child processes. unshare_flags = 0 if unshare_net or unshare_ipc or unshare_mount or unshare_pid: # from /usr/include/bits/sched.h CLONE_NEWNS = 0x00020000 CLONE_NEWUTS = 0x04000000 CLONE_NEWIPC = 0x08000000 CLONE_NEWPID = 0x20000000 CLONE_NEWNET = 0x40000000 if unshare_net: # UTS namespace to override hostname unshare_flags |= CLONE_NEWNET | CLONE_NEWUTS if unshare_ipc: unshare_flags |= CLONE_NEWIPC if unshare_mount: # NEWNS = mount namespace unshare_flags |= CLONE_NEWNS if unshare_pid: # we also need mount namespace for slave /proc unshare_flags |= CLONE_NEWPID | CLONE_NEWNS _unshare_validate(unshare_flags) # Force instantiation of portage.data.userpriv_groups before the # fork, so that the result is cached in the main process. bool(groups) parent_pid = os.getpid() pid = None try: pid = os.fork() if pid == 0: try: _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid, unshare_flags, cgroup) except SystemExit: raise except Exception as e: # We need to catch _any_ exception so that it doesn't # propagate out of this function and cause exiting # with anything other than os._exit() writemsg("%s:\n %s\n" % (e, " ".join(mycommand)), noiselevel=-1) traceback.print_exc() sys.stderr.flush() finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1) if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid),)) # Add the pid to our local and the global pid lists. mypids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. if logfile: os.close(pw) # If the caller wants to handle cleaning up the processes, we tell # it about all processes that were created. if returnpid: return mypids # Otherwise we clean them up. while mypids: # Pull the last reader in the pipe chain. If all processes # in the pipe are well behaved, it will die when the process # it is reading from dies. pid = mypids.pop(0) # and wait for it. retval = os.waitpid(pid, 0)[1] if retval: # If it failed, kill off anything else that # isn't dead yet. for pid in mypids: # With waitpid and WNOHANG, only check the # first element of the tuple since the second # element may vary (bug #337465). if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) # If it got a signal, return the signal that was sent. if (retval & 0xff): return ((retval & 0xff) << 8) # Otherwise, return its exit code. return (retval >> 8) # Everything succeeded return 0
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid, cgroup): """ Execute a given binary with options @param binary: Name of program to execute @type binary: String @param mycommand: Options for program @type mycommand: String @param opt_name: Name of process (defaults to binary) @type opt_name: String @param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 } @type fd_pipes: Dictionary @param env: Key,Value mapping for Environmental Variables @type env: Dictionary @param gid: Group ID to run the process under @type gid: Integer @param groups: Groups the Process should be in. @type groups: Integer @param uid: User ID to run the process under @type uid: Integer @param umask: an int representing a unix umask (see man chmod for umask details) @type umask: Integer @param cwd: Current working directory @type cwd: String @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param unshare_mount: If True, mount namespace will be unshared and mounts will be private to the namespace @type unshare_mount: Boolean @param unshare_pid: If True, PID ns will be unshared from the spawned process @type unshare_pid: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String @rtype: None @return: Never returns (calls os.execve) """ # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: if binary is portage._python_interpreter: # NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0] # does not contain the full path of the binary. opt_name = binary else: opt_name = os.path.basename(binary) # Set up the command's argument list. myargs = [opt_name] myargs.extend(mycommand[1:]) # Avoid a potential UnicodeEncodeError from os.execve(). myargs = [ _unicode_encode(x, encoding=_encodings['fs'], errors='strict') for x in myargs ] # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Unregister SIGCHLD handler and wakeup_fd for the parent # process's event loop (bug 655656). signal.signal(signal.SIGCHLD, signal.SIG_DFL) try: wakeup_fd = signal.set_wakeup_fd(-1) if wakeup_fd > 0: os.close(wakeup_fd) except (ValueError, OSError): pass # Quiet killing of subprocesses by SIGPIPE (see bug #309001). signal.signal(signal.SIGPIPE, signal.SIG_DFL) # Avoid issues triggered by inheritance of SIGQUIT handler from # the parent process (see bug #289486). signal.signal(signal.SIGQUIT, signal.SIG_DFL) _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True) # Add to cgroup # it's better to do it from the child since we can guarantee # it is done before we start forking children if cgroup: with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: f.write('%d\n' % os.getpid()) # Unshare (while still uid==0) if unshare_net or unshare_ipc or unshare_mount or unshare_pid: filename = find_library("c") if filename is not None: libc = LoadLibrary(filename) if libc is not None: # from /usr/include/bits/sched.h CLONE_NEWNS = 0x00020000 CLONE_NEWIPC = 0x08000000 CLONE_NEWPID = 0x20000000 CLONE_NEWNET = 0x40000000 flags = 0 if unshare_net: flags |= CLONE_NEWNET if unshare_ipc: flags |= CLONE_NEWIPC if unshare_mount: # NEWNS = mount namespace flags |= CLONE_NEWNS if unshare_pid: # we also need mount namespace for slave /proc flags |= CLONE_NEWPID | CLONE_NEWNS try: if libc.unshare(flags) != 0: writemsg( "Unable to unshare: %s\n" % (errno.errorcode.get(ctypes.get_errno(), '?')), noiselevel=-1) else: if unshare_pid: # pid namespace requires us to become init fork_ret = os.fork() if fork_ret != 0: os.execv(portage._python_interpreter, [ portage._python_interpreter, os.path.join(portage._bin_path, 'pid-ns-init'), '%s' % fork_ret, ]) if unshare_mount: # mark the whole filesystem as slave to avoid # mounts escaping the namespace s = subprocess.Popen( ['mount', '--make-rslave', '/']) mount_ret = s.wait() if mount_ret != 0: # TODO: should it be fatal maybe? writemsg("Unable to mark mounts slave: %d\n" % (mount_ret, ), noiselevel=-1) if unshare_pid: # we need at least /proc being slave s = subprocess.Popen( ['mount', '--make-slave', '/proc']) mount_ret = s.wait() if mount_ret != 0: # can't proceed with shared /proc writemsg("Unable to mark /proc slave: %d\n" % (mount_ret, ), noiselevel=-1) os._exit(1) # mount new /proc for our namespace s = subprocess.Popen( ['mount', '-t', 'proc', 'proc', '/proc']) mount_ret = s.wait() if mount_ret != 0: writemsg("Unable to mount new /proc: %d\n" % (mount_ret, ), noiselevel=-1) os._exit(1) if unshare_net: # 'up' the loopback IFF_UP = 0x1 ifreq = struct.pack('16sh', b'lo', IFF_UP) SIOCSIFFLAGS = 0x8914 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) try: fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq) except IOError as e: writemsg( "Unable to enable loopback interface: %s\n" % (errno.errorcode.get(e.errno, '?')), noiselevel=-1) sock.close() except AttributeError: # unshare() not supported by libc pass # Set requested process permissions. if gid: # Cast proxies to int, in case it matters. os.setgid(int(gid)) if groups: os.setgroups(groups) if uid: # Cast proxies to int, in case it matters. os.setuid(int(uid)) if umask: os.umask(umask) if cwd is not None: os.chdir(cwd) if pre_exec: pre_exec() # And switch to the new process. os.execve(binary, myargs, env)
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid, unshare_flags, cgroup): """ Execute a given binary with options @param binary: Name of program to execute @type binary: String @param mycommand: Options for program @type mycommand: String @param opt_name: Name of process (defaults to binary) @type opt_name: String @param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 } @type fd_pipes: Dictionary @param env: Key,Value mapping for Environmental Variables @type env: Dictionary @param gid: Group ID to run the process under @type gid: Integer @param groups: Groups the Process should be in. @type groups: List @param uid: User ID to run the process under @type uid: Integer @param umask: an int representing a unix umask (see man chmod for umask details) @type umask: Integer @param cwd: Current working directory @type cwd: String @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param unshare_mount: If True, mount namespace will be unshared and mounts will be private to the namespace @type unshare_mount: Boolean @param unshare_pid: If True, PID ns will be unshared from the spawned process @type unshare_pid: Boolean @param unshare_flags: Flags for the unshare(2) function @type unshare_flags: Integer @param cgroup: CGroup path to bind the process to @type cgroup: String @rtype: None @return: Never returns (calls os.execve) """ # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: if binary is portage._python_interpreter: # NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0] # does not contain the full path of the binary. opt_name = binary else: opt_name = os.path.basename(binary) # Set up the command's argument list. myargs = [opt_name] myargs.extend(mycommand[1:]) # Avoid a potential UnicodeEncodeError from os.execve(). myargs = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict') for x in myargs] # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Unregister SIGCHLD handler and wakeup_fd for the parent # process's event loop (bug 655656). signal.signal(signal.SIGCHLD, signal.SIG_DFL) try: wakeup_fd = signal.set_wakeup_fd(-1) if wakeup_fd > 0: os.close(wakeup_fd) except (ValueError, OSError): pass # Quiet killing of subprocesses by SIGPIPE (see bug #309001). signal.signal(signal.SIGPIPE, signal.SIG_DFL) # Avoid issues triggered by inheritance of SIGQUIT handler from # the parent process (see bug #289486). signal.signal(signal.SIGQUIT, signal.SIG_DFL) _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True) # Add to cgroup # it's better to do it from the child since we can guarantee # it is done before we start forking children if cgroup: with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: f.write('%d\n' % os.getpid()) # Unshare (while still uid==0) if unshare_net or unshare_ipc or unshare_mount or unshare_pid: filename = find_library("c") if filename is not None: libc = LoadLibrary(filename) if libc is not None: try: # Since a failed unshare call could corrupt process # state, first validate that the call can succeed. # The parent process should call _unshare_validate # before it forks, so that all child processes can # reuse _unshare_validate results that have been # cached by the parent process. errno_value = _unshare_validate(unshare_flags) if errno_value == 0 and libc.unshare(unshare_flags) != 0: errno_value = ctypes.get_errno() if errno_value != 0: writemsg("Unable to unshare: %s\n" % ( errno.errorcode.get(errno_value, '?')), noiselevel=-1) else: if unshare_pid: main_child_pid = os.fork() if main_child_pid == 0: # pid namespace requires us to become init binary, myargs = portage._python_interpreter, [ portage._python_interpreter, os.path.join(portage._bin_path, 'pid-ns-init'), _unicode_encode('' if uid is None else str(uid)), _unicode_encode('' if gid is None else str(gid)), _unicode_encode('' if groups is None else ','.join(str(group) for group in groups)), _unicode_encode('' if umask is None else str(umask)), _unicode_encode(','.join(str(fd) for fd in fd_pipes)), binary] + myargs uid = None gid = None groups = None umask = None else: # Execute a supervisor process which will forward # signals to init and forward exit status to the # parent process. The supervisor process runs in # the global pid namespace, so skip /proc remount # and other setup that's intended only for the # init process. binary, myargs = portage._python_interpreter, [ portage._python_interpreter, os.path.join(portage._bin_path, 'pid-ns-init'), str(main_child_pid)] os.execve(binary, myargs, env) if unshare_mount: # mark the whole filesystem as slave to avoid # mounts escaping the namespace s = subprocess.Popen(['mount', '--make-rslave', '/']) mount_ret = s.wait() if mount_ret != 0: # TODO: should it be fatal maybe? writemsg("Unable to mark mounts slave: %d\n" % (mount_ret,), noiselevel=-1) if unshare_pid: # we need at least /proc being slave s = subprocess.Popen(['mount', '--make-slave', '/proc']) mount_ret = s.wait() if mount_ret != 0: # can't proceed with shared /proc writemsg("Unable to mark /proc slave: %d\n" % (mount_ret,), noiselevel=-1) os._exit(1) # mount new /proc for our namespace s = subprocess.Popen(['mount', '-n', '-t', 'proc', 'proc', '/proc']) mount_ret = s.wait() if mount_ret != 0: writemsg("Unable to mount new /proc: %d\n" % (mount_ret,), noiselevel=-1) os._exit(1) if unshare_net: # 'up' the loopback IFF_UP = 0x1 ifreq = struct.pack('16sh', b'lo', IFF_UP) SIOCSIFFLAGS = 0x8914 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) try: fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq) except IOError as e: writemsg("Unable to enable loopback interface: %s\n" % ( errno.errorcode.get(e.errno, '?')), noiselevel=-1) sock.close() except AttributeError: # unshare() not supported by libc pass # Set requested process permissions. if gid: # Cast proxies to int, in case it matters. os.setgid(int(gid)) if groups: os.setgroups(groups) if uid: # Cast proxies to int, in case it matters. os.setuid(int(uid)) if umask: os.umask(umask) if cwd is not None: os.chdir(cwd) if pre_exec: pre_exec() # And switch to the new process. os.execve(binary, myargs, env)
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid, cgroup): """ Execute a given binary with options @param binary: Name of program to execute @type binary: String @param mycommand: Options for program @type mycommand: String @param opt_name: Name of process (defaults to binary) @type opt_name: String @param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 } @type fd_pipes: Dictionary @param env: Key,Value mapping for Environmental Variables @type env: Dictionary @param gid: Group ID to run the process under @type gid: Integer @param groups: Groups the Process should be in. @type groups: Integer @param uid: User ID to run the process under @type uid: Integer @param umask: an int representing a unix umask (see man chmod for umask details) @type umask: Integer @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param unshare_mount: If True, mount namespace will be unshared and mounts will be private to the namespace @type unshare_mount: Boolean @param unshare_pid: If True, PID ns will be unshared from the spawned process @type unshare_pid: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String @rtype: None @return: Never returns (calls os.execve) """ # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: if binary is portage._python_interpreter: # NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0] # does not contain the full path of the binary. opt_name = binary else: opt_name = os.path.basename(binary) # Set up the command's argument list. myargs = [opt_name] myargs.extend(mycommand[1:]) # Avoid a potential UnicodeEncodeError from os.execve(). myargs = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict') for x in myargs] # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Quiet killing of subprocesses by SIGPIPE (see bug #309001). signal.signal(signal.SIGPIPE, signal.SIG_DFL) # Avoid issues triggered by inheritance of SIGQUIT handler from # the parent process (see bug #289486). signal.signal(signal.SIGQUIT, signal.SIG_DFL) _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True) # Add to cgroup # it's better to do it from the child since we can guarantee # it is done before we start forking children if cgroup: with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: f.write('%d\n' % os.getpid()) # Unshare (while still uid==0) if unshare_net or unshare_ipc or unshare_mount or unshare_pid: filename = find_library("c") if filename is not None: libc = LoadLibrary(filename) if libc is not None: # from /usr/include/bits/sched.h CLONE_NEWNS = 0x00020000 CLONE_NEWIPC = 0x08000000 CLONE_NEWPID = 0x20000000 CLONE_NEWNET = 0x40000000 flags = 0 if unshare_net: flags |= CLONE_NEWNET if unshare_ipc: flags |= CLONE_NEWIPC if unshare_mount: # NEWNS = mount namespace flags |= CLONE_NEWNS if unshare_pid: # we also need mount namespace for slave /proc flags |= CLONE_NEWPID | CLONE_NEWNS try: if libc.unshare(flags) != 0: writemsg("Unable to unshare: %s\n" % ( errno.errorcode.get(ctypes.get_errno(), '?')), noiselevel=-1) else: if unshare_pid: # pid namespace requires us to become init fork_ret = os.fork() if fork_ret != 0: os.execv(portage._python_interpreter, [ portage._python_interpreter, os.path.join(portage._bin_path, 'pid-ns-init'), '%s' % fork_ret, ]) if unshare_mount: # mark the whole filesystem as private to avoid # mounts escaping the namespace s = subprocess.Popen(['mount', '--make-rprivate', '/']) mount_ret = s.wait() if mount_ret != 0: # TODO: should it be fatal maybe? writemsg("Unable to mark mounts private: %d\n" % (mount_ret,), noiselevel=-1) if unshare_pid: # we need at least /proc being slave s = subprocess.Popen(['mount', '--make-slave', '/proc']) mount_ret = s.wait() if mount_ret != 0: # can't proceed with shared /proc writemsg("Unable to mark /proc slave: %d\n" % (mount_ret,), noiselevel=-1) os._exit(1) # mount new /proc for our namespace s = subprocess.Popen(['mount', '-t', 'proc', 'proc', '/proc']) mount_ret = s.wait() if mount_ret != 0: writemsg("Unable to mount new /proc: %d\n" % (mount_ret,), noiselevel=-1) os._exit(1) if unshare_net: # 'up' the loopback IFF_UP = 0x1 ifreq = struct.pack('16sh', b'lo', IFF_UP) SIOCSIFFLAGS = 0x8914 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) try: fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq) except IOError as e: writemsg("Unable to enable loopback interface: %s\n" % ( errno.errorcode.get(e.errno, '?')), noiselevel=-1) sock.close() except AttributeError: # unshare() not supported by libc pass # Set requested process permissions. if gid: # Cast proxies to int, in case it matters. os.setgid(int(gid)) if groups: os.setgroups(groups) if uid: # Cast proxies to int, in case it matters. os.setuid(int(uid)) if umask: os.umask(umask) if pre_exec: pre_exec() # And switch to the new process. os.execve(binary, myargs, env)
def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call dblink.merge(). """ elog_reader_fd, elog_writer_fd = os.pipe() fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) blockers = None if self.blockers is not None: # Query blockers in the main process, since closing # of file descriptors in the subprocess can prevent # access to open database connections such as that # used by the sqlite metadata cache module. blockers = self.blockers() mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, blockers=blockers, scheduler=self.scheduler, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd self._elog_reg_id = self.scheduler.register(elog_reader_fd, self._registered_events, self._elog_output_handler) # If a concurrent emerge process tries to install a package # in the same SLOT as this one at the same time, there is an # extremely unlikely chance that the COUNTER values will not be # ordered correctly unless we lock the vdb here. # FEATURES=parallel-install skips this lock in order to # improve performance, and the risk is practically negligible. self._lock_vdb() counter = None if not self.unmerge: counter = self.vartree.dbapi.counter_tick() pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError( "fork returned non-integer: %s" % (repr(pid),)) os.close(elog_writer_fd) self._elog_reader_fd = elog_reader_fd self._buf = "" self._elog_keys = set() # invalidate relevant vardbapi caches if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None self.vartree.dbapi._pkgs_changed = True self.vartree.dbapi._clear_pkg_cache(mylink) portage.process.spawned_pids.append(pid) return [pid] os.close(elog_reader_fd) # TODO: Find out why PyPy 1.8 with close_fds=True triggers # "[Errno 9] Bad file descriptor" in subprocesses. It could # be due to garbage collection of file objects that were not # closed before going out of scope, since PyPy's garbage # collector does not support the refcounting semantics that # CPython does. close_fds = platform.python_implementation() != 'PyPy' portage.process._setup_pipes(fd_pipes, close_fds=close_fds) # Use default signal handlers since the ones inherited # from the parent process are irrelevant here. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) portage.output.havecolor = self.settings.get('NOCOLOR') \ not in ('yes', 'true') # In this subprocess we want mylink._display_merge() to use # stdout/stderr directly since they are pipes. This behavior # is triggered when mylink._scheduler is None. mylink._scheduler = None # Avoid wastful updates of the vdb cache. self.vartree.dbapi._flush_cache_enabled = False # In this subprocess we don't want PORTAGE_BACKGROUND to # suppress stdout/stderr output since they are pipes. We # also don't want to open PORTAGE_LOG_FILE, since it will # already be opened by the parent process, so we set the # "subprocess" value for use in conditional logging code # involving PORTAGE_LOG_FILE. if not self.unmerge: # unmerge phases have separate logs if self.settings.get("PORTAGE_BACKGROUND") == "1": self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" else: self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") self.settings["PORTAGE_BACKGROUND"] = "subprocess" self.settings.backup_changes("PORTAGE_BACKGROUND") rval = 1 try: if self.unmerge: if not mylink.exists(): rval = os.EX_OK elif mylink.unmerge( ldpath_mtimes=self.prev_mtimes) == os.EX_OK: mylink.lockdb() try: mylink.delete() finally: mylink.unlockdb() rval = os.EX_OK else: rval = mylink.merge(self.pkgloc, self.infloc, myebuild=self.myebuild, mydbapi=self.mydbapi, prev_mtimes=self.prev_mtimes, counter=counter) except SystemExit: raise except: traceback.print_exc() finally: # Call os._exit() from finally block, in order to suppress any # finally blocks from earlier in the call stack. See bug #345289. os._exit(rval)
def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, cwd=None, logfile=None, path_lookup=True, pre_exec=None, close_fds=(sys.version_info < (3, 4)), unshare_net=False, unshare_ipc=False, unshare_mount=False, unshare_pid=False, cgroup=None): """ Spawns a given command. @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: If env is not None, it must be a mapping that defines the environment variables for the new process; these are used instead of the default behavior of inheriting the current process's environment. @type env: None or Mapping @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example (default is {0:stdin, 1:stdout, 2:stderr}) @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @param gid: Group ID to spawn as; useful for dropping privilages @type gid: Integer @param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts. @type groups: List @param umask: An integer representing the umask for the process (see man chmod for umask details) @type umask: Integer @param cwd: Current working directory @type cwd: String @param logfile: name of a file to use for logging purposes @type logfile: String @param path_lookup: If the binary is not fully specified then look for it in PATH @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param close_fds: If True, then close all file descriptors except those referenced by fd_pipes (default is True for python3.3 and earlier, and False for python3.4 and later due to non-inheritable file descriptor behavior from PEP 446). @type close_fds: Boolean @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param unshare_mount: If True, mount namespace will be unshared and mounts will be private to the namespace @type unshare_mount: Boolean @param unshare_pid: If True, PID ns will be unshared from the spawned process @type unshare_pid: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) """ # mycommand is either a str or a list if isinstance(mycommand, basestring): mycommand = mycommand.split() env = os.environ if env is None else env if sys.hexversion < 0x3000000: # Avoid a potential UnicodeEncodeError from os.execve(). env_bytes = {} for k, v in env.items(): env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \ _unicode_encode(v, encoding=_encodings['content']) env = env_bytes del env_bytes # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \ (not os.path.isabs(binary) or not os.path.isfile(binary) or not os.access(binary, os.X_OK)): binary = path_lookup and find_binary(binary) or None if not binary: raise CommandNotFound(mycommand[0]) # If we haven't been told what file descriptors to use # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { 0:portage._get_stdin().fileno(), 1:sys.__stdout__.fileno(), 2:sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. mypids = [] if logfile: # Using a log file requires that stdout and stderr # are assigned to the process we're running. if 1 not in fd_pipes or 2 not in fd_pipes: raise ValueError(fd_pipes) # Create a pipe (pr, pw) = os.pipe() # Create a tee process, giving it our stdout and stderr # as well as the read end of the pipe. mypids.extend(spawn(('tee', '-i', '-a', logfile), returnpid=True, fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]})) # We don't need the read end of the pipe, so close it. os.close(pr) # Assign the write end of the pipe to our stdout and stderr. fd_pipes[1] = pw fd_pipes[2] = pw # This caches the libc library lookup and _unshare_validator results # in the current process, so that results are cached for use in # child processes. unshare_flags = 0 if unshare_net or unshare_ipc or unshare_mount or unshare_pid: # from /usr/include/bits/sched.h CLONE_NEWNS = 0x00020000 CLONE_NEWIPC = 0x08000000 CLONE_NEWPID = 0x20000000 CLONE_NEWNET = 0x40000000 if unshare_net: unshare_flags |= CLONE_NEWNET if unshare_ipc: unshare_flags |= CLONE_NEWIPC if unshare_mount: # NEWNS = mount namespace unshare_flags |= CLONE_NEWNS if unshare_pid: # we also need mount namespace for slave /proc unshare_flags |= CLONE_NEWPID | CLONE_NEWNS _unshare_validate(unshare_flags) # Force instantiation of portage.data.userpriv_groups before the # fork, so that the result is cached in the main process. bool(groups) parent_pid = os.getpid() pid = None try: pid = os.fork() if pid == 0: try: _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid, unshare_flags, cgroup) except SystemExit: raise except Exception as e: # We need to catch _any_ exception so that it doesn't # propagate out of this function and cause exiting # with anything other than os._exit() writemsg("%s:\n %s\n" % (e, " ".join(mycommand)), noiselevel=-1) traceback.print_exc() sys.stderr.flush() finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1) if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid),)) # Add the pid to our local and the global pid lists. mypids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. if logfile: os.close(pw) # If the caller wants to handle cleaning up the processes, we tell # it about all processes that were created. if returnpid: return mypids # Otherwise we clean them up. while mypids: # Pull the last reader in the pipe chain. If all processes # in the pipe are well behaved, it will die when the process # it is reading from dies. pid = mypids.pop(0) # and wait for it. retval = os.waitpid(pid, 0)[1] if retval: # If it failed, kill off anything else that # isn't dead yet. for pid in mypids: # With waitpid and WNOHANG, only check the # first element of the tuple since the second # element may vary (bug #337465). if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) # If it got a signal, return the signal that was sent. if (retval & 0xff): return ((retval & 0xff) << 8) # Otherwise, return its exit code. return (retval >> 8) # Everything succeeded return 0
def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call dblink.merge(). """ elog_reader_fd, elog_writer_fd = os.pipe() fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) blockers = None if self.blockers is not None: # Query blockers in the main process, since closing # of file descriptors in the subprocess can prevent # access to open database connections such as that # used by the sqlite metadata cache module. blockers = self.blockers() mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, blockers=blockers, scheduler=self.scheduler, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd self._elog_reg_id = self.scheduler.register(elog_reader_fd, self._registered_events, self._elog_output_handler) # If a concurrent emerge process tries to install a package # in the same SLOT as this one at the same time, there is an # extremely unlikely chance that the COUNTER values will not be # ordered correctly unless we lock the vdb here. # FEATURES=parallel-install skips this lock in order to # improve performance, and the risk is practically negligible. self._lock_vdb() counter = None if not self.unmerge: counter = self.vartree.dbapi.counter_tick() pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid), )) os.close(elog_writer_fd) self._elog_reader_fd = elog_reader_fd self._buf = "" self._elog_keys = set() # invalidate relevant vardbapi caches if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None self.vartree.dbapi._pkgs_changed = True self.vartree.dbapi._clear_pkg_cache(mylink) portage.process.spawned_pids.append(pid) return [pid] os.close(elog_reader_fd) # TODO: Find out why PyPy 1.8 with close_fds=True triggers # "[Errno 9] Bad file descriptor" in subprocesses. It could # be due to garbage collection of file objects that were not # closed before going out of scope, since PyPy's garbage # collector does not support the refcounting semantics that # CPython does. close_fds = platform.python_implementation() != 'PyPy' portage.process._setup_pipes(fd_pipes, close_fds=close_fds) # Use default signal handlers since the ones inherited # from the parent process are irrelevant here. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) portage.output.havecolor = self.settings.get('NOCOLOR') \ not in ('yes', 'true') # In this subprocess we want mylink._display_merge() to use # stdout/stderr directly since they are pipes. This behavior # is triggered when mylink._scheduler is None. mylink._scheduler = None # Avoid wastful updates of the vdb cache. self.vartree.dbapi._flush_cache_enabled = False # In this subprocess we don't want PORTAGE_BACKGROUND to # suppress stdout/stderr output since they are pipes. We # also don't want to open PORTAGE_LOG_FILE, since it will # already be opened by the parent process, so we set the # "subprocess" value for use in conditional logging code # involving PORTAGE_LOG_FILE. if not self.unmerge: # unmerge phases have separate logs if self.settings.get("PORTAGE_BACKGROUND") == "1": self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" else: self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") self.settings["PORTAGE_BACKGROUND"] = "subprocess" self.settings.backup_changes("PORTAGE_BACKGROUND") rval = 1 try: if self.unmerge: if not mylink.exists(): rval = os.EX_OK elif mylink.unmerge( ldpath_mtimes=self.prev_mtimes) == os.EX_OK: mylink.lockdb() try: mylink.delete() finally: mylink.unlockdb() rval = os.EX_OK else: rval = mylink.merge(self.pkgloc, self.infloc, myebuild=self.myebuild, mydbapi=self.mydbapi, prev_mtimes=self.prev_mtimes, counter=counter) except SystemExit: raise except: traceback.print_exc() finally: # Call os._exit() from finally block, in order to suppress any # finally blocks from earlier in the call stack. See bug #345289. os._exit(rval)