def _start(self): # Portage should always call setcpv prior to this # point, but here we have a fallback as a convenience # for external API consumers. It's important that # this metadata access happens in the parent process, # since closing of file descriptors in the subprocess # can prevent access to open database connections such # as that used by the sqlite metadata cache module. cpv = "%s/%s" % (self.mycat, self.mypkg) settings = self.settings if cpv != settings.mycpv or "EAPI" not in settings.configdict["pkg"]: settings.reload() settings.reset() settings.setcpv(cpv, mydb=self.mydbapi) # This caches the libc library lookup in the current # process, so that it's only done once rather than # for each child process. if platform.system() == "Linux" and "merge-sync" in settings.features: find_library("c") # Inherit stdin by default, so that the pdb SIGUSR1 # handler is usable for the subprocess. if self.fd_pipes is None: self.fd_pipes = {} else: self.fd_pipes = self.fd_pipes.copy() self.fd_pipes.setdefault(0, portage._get_stdin().fileno()) self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD") super(MergeProcess, self)._start()
def _start(self): # Portage should always call setcpv prior to this # point, but here we have a fallback as a convenience # for external API consumers. It's important that # this metadata access happens in the parent process, # since closing of file descriptors in the subprocess # can prevent access to open database connections such # as that used by the sqlite metadata cache module. cpv = "%s/%s" % (self.mycat, self.mypkg) settings = self.settings if cpv != settings.mycpv or \ "EAPI" not in settings.configdict["pkg"]: settings.reload() settings.reset() settings.setcpv(cpv, mydb=self.mydbapi) # This caches the libc library lookup in the current # process, so that it's only done once rather than # for each child process. if platform.system() == "Linux" and \ "merge-sync" in settings.features: find_library("c") # Inherit stdin by default, so that the pdb SIGUSR1 # handler is usable for the subprocess. if self.fd_pipes is None: self.fd_pipes = {} else: self.fd_pipes = self.fd_pipes.copy() self.fd_pipes.setdefault(0, portage._get_stdin().fileno()) super(MergeProcess, self)._start()
def _validate(cls, flags): """ Perform validation. @param flags: unshare flags @type flags: int @rtype: int @returns: errno value, or 0 if no error occurred. """ filename = find_library("c") if filename is None: return errno.ENOTSUP libc = LoadLibrary(filename) if libc is None: return errno.ENOTSUP parent_pipe, subproc_pipe = multiprocessing.Pipe(duplex=False) proc = multiprocessing.Process( target=cls._run_subproc, args=(subproc_pipe, cls._validate_subproc, (libc.unshare, flags))) proc.start() subproc_pipe.close() result = parent_pipe.recv() parent_pipe.close() proc.join() return result
def _validate(cls, flags): """ Perform validation. @param flags: unshare flags @type flags: int @rtype: int @returns: errno value, or 0 if no error occurred. """ filename = find_library("c") if filename is None: return errno.ENOTSUP libc = LoadLibrary(filename) if libc is None: return errno.ENOTSUP parent_pipe, subproc_pipe = multiprocessing.Pipe(duplex=False) proc = multiprocessing.Process( target=cls._run_subproc, args=(subproc_pipe, cls._validate_subproc, (libc.unshare, flags))) proc.start() subproc_pipe.close() result = parent_pipe.recv() parent_pipe.close() proc.join() return result
def _get_syncfs(): filename = find_library("c") if filename is not None: library = LoadLibrary(filename) if library is not None: try: return library.syncfs except AttributeError: pass return None
def _get_syncfs(): filename = find_library("c") if filename is not None: library = LoadLibrary(filename) if library is not None: try: return library.syncfs except AttributeError: pass return None
def _check_locale(silent): """ The inner locale check function. """ try: from portage.util import libc except ImportError: libc_fn = find_library("c") if libc_fn is None: return None libc = LoadLibrary(libc_fn) if libc is None: return None lc = list(range(ord('a'), ord('z') + 1)) uc = list(range(ord('A'), ord('Z') + 1)) rlc = [libc.tolower(c) for c in uc] ruc = [libc.toupper(c) for c in lc] if lc != rlc or uc != ruc: if silent: return False msg = ("WARNING: The LC_CTYPE variable is set to a locale " + "that specifies transformation between lowercase " + "and uppercase ASCII characters that is different than " + "the one specified by POSIX locale. This can break " + "ebuilds and cause issues in programs that rely on " + "the common character conversion scheme. " + "Please consider enabling another locale (such as " + "en_US.UTF-8) in /etc/locale.gen and setting it " + "as LC_CTYPE in make.conf.") msg = [l for l in textwrap.wrap(msg, 70)] msg.append("") chars = lambda l: ''.join(_unicode_decode(chr(x)) for x in l) if uc != ruc: msg.extend([ " %s -> %s" % (chars(lc), chars(ruc)), " %28s: %s" % ('expected', chars(uc)) ]) if lc != rlc: msg.extend([ " %s -> %s" % (chars(uc), chars(rlc)), " %28s: %s" % ('expected', chars(lc)) ]) writemsg_level("".join(["!!! %s\n" % l for l in msg]), level=logging.ERROR, noiselevel=-1) return False return True
def _check_locale(silent): """ The inner locale check function. """ try: from portage.util import libc except ImportError: libc_fn = find_library("c") if libc_fn is None: return None libc = LoadLibrary(libc_fn) if libc is None: return None lc = list(range(ord('a'), ord('z')+1)) uc = list(range(ord('A'), ord('Z')+1)) rlc = [libc.tolower(c) for c in uc] ruc = [libc.toupper(c) for c in lc] if lc != rlc or uc != ruc: if silent: return False msg = ("WARNING: The LC_CTYPE variable is set to a locale " + "that specifies transformation between lowercase " + "and uppercase ASCII characters that is different than " + "the one specified by POSIX locale. This can break " + "ebuilds and cause issues in programs that rely on " + "the common character conversion scheme. " + "Please consider enabling another locale (such as " + "en_US.UTF-8) in /etc/locale.gen and setting it " + "as LC_CTYPE in make.conf.") msg = [l for l in textwrap.wrap(msg, 70)] msg.append("") chars = lambda l: ''.join(_unicode_decode(chr(x)) for x in l) if uc != ruc: msg.extend([ " %s -> %s" % (chars(lc), chars(ruc)), " %28s: %s" % ('expected', chars(uc))]) if lc != rlc: msg.extend([ " %s -> %s" % (chars(uc), chars(rlc)), " %28s: %s" % ('expected', chars(lc))]) writemsg_level("".join(["!!! %s\n" % l for l in msg]), level=logging.ERROR, noiselevel=-1) return False return True
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, pre_exec, close_fds, unshare_net, unshare_ipc, cgroup): """ Execute a given binary with options @param binary: Name of program to execute @type binary: String @param mycommand: Options for program @type mycommand: String @param opt_name: Name of process (defaults to binary) @type opt_name: String @param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 } @type fd_pipes: Dictionary @param env: Key,Value mapping for Environmental Variables @type env: Dictionary @param gid: Group ID to run the process under @type gid: Integer @param groups: Groups the Process should be in. @type groups: Integer @param uid: User ID to run the process under @type uid: Integer @param umask: an int representing a unix umask (see man chmod for umask details) @type umask: Integer @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String @rtype: None @return: Never returns (calls os.execve) """ # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: if binary is portage._python_interpreter: # NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0] # does not contain the full path of the binary. opt_name = binary else: opt_name = os.path.basename(binary) # Set up the command's argument list. myargs = [opt_name] myargs.extend(mycommand[1:]) # Avoid a potential UnicodeEncodeError from os.execve(). myargs = [ _unicode_encode(x, encoding=_encodings['fs'], errors='strict') for x in myargs ] # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Quiet killing of subprocesses by SIGPIPE (see bug #309001). signal.signal(signal.SIGPIPE, signal.SIG_DFL) # Avoid issues triggered by inheritance of SIGQUIT handler from # the parent process (see bug #289486). signal.signal(signal.SIGQUIT, signal.SIG_DFL) _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True) # Add to cgroup # it's better to do it from the child since we can guarantee # it is done before we start forking children if cgroup: with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: f.write('%d\n' % os.getpid()) # Unshare (while still uid==0) if unshare_net or unshare_ipc: filename = find_library("c") if filename is not None: libc = LoadLibrary(filename) if libc is not None: CLONE_NEWIPC = 0x08000000 CLONE_NEWNET = 0x40000000 flags = 0 if unshare_net: flags |= CLONE_NEWNET if unshare_ipc: flags |= CLONE_NEWIPC try: if libc.unshare(flags) != 0: writemsg( "Unable to unshare: %s\n" % (errno.errorcode.get(ctypes.get_errno(), '?')), noiselevel=-1) else: if unshare_net: # 'up' the loopback IFF_UP = 0x1 ifreq = struct.pack('16sh', b'lo', IFF_UP) SIOCSIFFLAGS = 0x8914 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) try: fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq) except IOError as e: writemsg( "Unable to enable loopback interface: %s\n" % (errno.errorcode.get(e.errno, '?')), noiselevel=-1) sock.close() except AttributeError: # unshare() not supported by libc pass # Set requested process permissions. if gid: # Cast proxies to int, in case it matters. os.setgid(int(gid)) if groups: os.setgroups(groups) if uid: # Cast proxies to int, in case it matters. os.setuid(int(uid)) if umask: os.umask(umask) if pre_exec: pre_exec() # And switch to the new process. os.execve(binary, myargs, env)
def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, logfile=None, path_lookup=True, pre_exec=None, close_fds=(sys.version_info < (3, 4)), unshare_net=False, unshare_ipc=False, cgroup=None): """ Spawns a given command. @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: A dict of Key=Value pairs for env variables @type env: Dictionary @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example (default is {0:stdin, 1:stdout, 2:stderr}) @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @param gid: Group ID to spawn as; useful for dropping privilages @type gid: Integer @param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts. @type groups: List @param umask: An integer representing the umask for the process (see man chmod for umask details) @type umask: Integer @param logfile: name of a file to use for logging purposes @type logfile: String @param path_lookup: If the binary is not fully specified then look for it in PATH @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param close_fds: If True, then close all file descriptors except those referenced by fd_pipes (default is True for python3.3 and earlier, and False for python3.4 and later due to non-inheritable file descriptor behavior from PEP 446). @type close_fds: Boolean @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) """ # mycommand is either a str or a list if isinstance(mycommand, basestring): mycommand = mycommand.split() if sys.hexversion < 0x3000000: # Avoid a potential UnicodeEncodeError from os.execve(). env_bytes = {} for k, v in env.items(): env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \ _unicode_encode(v, encoding=_encodings['content']) env = env_bytes del env_bytes # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \ (not os.path.isabs(binary) or not os.path.isfile(binary) or not os.access(binary, os.X_OK)): binary = path_lookup and find_binary(binary) or None if not binary: raise CommandNotFound(mycommand[0]) # If we haven't been told what file descriptors to use # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { 0: portage._get_stdin().fileno(), 1: sys.__stdout__.fileno(), 2: sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. mypids = [] if logfile: # Using a log file requires that stdout and stderr # are assigned to the process we're running. if 1 not in fd_pipes or 2 not in fd_pipes: raise ValueError(fd_pipes) # Create a pipe (pr, pw) = os.pipe() # Create a tee process, giving it our stdout and stderr # as well as the read end of the pipe. mypids.extend( spawn(('tee', '-i', '-a', logfile), returnpid=True, fd_pipes={ 0: pr, 1: fd_pipes[1], 2: fd_pipes[2] })) # We don't need the read end of the pipe, so close it. os.close(pr) # Assign the write end of the pipe to our stdout and stderr. fd_pipes[1] = pw fd_pipes[2] = pw # This caches the libc library lookup in the current # process, so that it's only done once rather than # for each child process. if unshare_net or unshare_ipc: find_library("c") # Force instantiation of portage.data.userpriv_groups before the # fork, so that the result is cached in the main process. bool(groups) parent_pid = os.getpid() pid = None try: pid = os.fork() if pid == 0: try: _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, pre_exec, close_fds, unshare_net, unshare_ipc, cgroup) except SystemExit: raise except Exception as e: # We need to catch _any_ exception so that it doesn't # propagate out of this function and cause exiting # with anything other than os._exit() writemsg("%s:\n %s\n" % (e, " ".join(mycommand)), noiselevel=-1) traceback.print_exc() sys.stderr.flush() finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1) if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid), )) # Add the pid to our local and the global pid lists. mypids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. if logfile: os.close(pw) # If the caller wants to handle cleaning up the processes, we tell # it about all processes that were created. if returnpid: return mypids # Otherwise we clean them up. while mypids: # Pull the last reader in the pipe chain. If all processes # in the pipe are well behaved, it will die when the process # it is reading from dies. pid = mypids.pop(0) # and wait for it. retval = os.waitpid(pid, 0)[1] if retval: # If it failed, kill off anything else that # isn't dead yet. for pid in mypids: # With waitpid and WNOHANG, only check the # first element of the tuple since the second # element may vary (bug #337465). if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) # If it got a signal, return the signal that was sent. if (retval & 0xff): return ((retval & 0xff) << 8) # Otherwise, return its exit code. return (retval >> 8) # Everything succeeded return 0
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid, unshare_flags, cgroup): """ Execute a given binary with options @param binary: Name of program to execute @type binary: String @param mycommand: Options for program @type mycommand: String @param opt_name: Name of process (defaults to binary) @type opt_name: String @param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 } @type fd_pipes: Dictionary @param env: Key,Value mapping for Environmental Variables @type env: Dictionary @param gid: Group ID to run the process under @type gid: Integer @param groups: Groups the Process should be in. @type groups: List @param uid: User ID to run the process under @type uid: Integer @param umask: an int representing a unix umask (see man chmod for umask details) @type umask: Integer @param cwd: Current working directory @type cwd: String @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param unshare_mount: If True, mount namespace will be unshared and mounts will be private to the namespace @type unshare_mount: Boolean @param unshare_pid: If True, PID ns will be unshared from the spawned process @type unshare_pid: Boolean @param unshare_flags: Flags for the unshare(2) function @type unshare_flags: Integer @param cgroup: CGroup path to bind the process to @type cgroup: String @rtype: None @return: Never returns (calls os.execve) """ # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: if binary is portage._python_interpreter: # NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0] # does not contain the full path of the binary. opt_name = binary else: opt_name = os.path.basename(binary) # Set up the command's argument list. myargs = [opt_name] myargs.extend(mycommand[1:]) # Avoid a potential UnicodeEncodeError from os.execve(). myargs = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict') for x in myargs] # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Unregister SIGCHLD handler and wakeup_fd for the parent # process's event loop (bug 655656). signal.signal(signal.SIGCHLD, signal.SIG_DFL) try: wakeup_fd = signal.set_wakeup_fd(-1) if wakeup_fd > 0: os.close(wakeup_fd) except (ValueError, OSError): pass # Quiet killing of subprocesses by SIGPIPE (see bug #309001). signal.signal(signal.SIGPIPE, signal.SIG_DFL) # Avoid issues triggered by inheritance of SIGQUIT handler from # the parent process (see bug #289486). signal.signal(signal.SIGQUIT, signal.SIG_DFL) _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True) # Add to cgroup # it's better to do it from the child since we can guarantee # it is done before we start forking children if cgroup: with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: f.write('%d\n' % os.getpid()) # Unshare (while still uid==0) if unshare_net or unshare_ipc or unshare_mount or unshare_pid: filename = find_library("c") if filename is not None: libc = LoadLibrary(filename) if libc is not None: try: # Since a failed unshare call could corrupt process # state, first validate that the call can succeed. # The parent process should call _unshare_validate # before it forks, so that all child processes can # reuse _unshare_validate results that have been # cached by the parent process. errno_value = _unshare_validate(unshare_flags) if errno_value == 0 and libc.unshare(unshare_flags) != 0: errno_value = ctypes.get_errno() if errno_value != 0: involved_features = [] if unshare_ipc: involved_features.append('ipc-sandbox') if unshare_mount: involved_features.append('mount-sandbox') if unshare_net: involved_features.append('network-sandbox') if unshare_pid: involved_features.append('pid-sandbox') writemsg("Unable to unshare: %s (for FEATURES=\"%s\")\n" % ( errno.errorcode.get(errno_value, '?'), ' '.join(involved_features)), noiselevel=-1) else: if unshare_pid: main_child_pid = os.fork() if main_child_pid == 0: # pid namespace requires us to become init binary, myargs = portage._python_interpreter, [ portage._python_interpreter, os.path.join(portage._bin_path, 'pid-ns-init'), _unicode_encode('' if uid is None else str(uid)), _unicode_encode('' if gid is None else str(gid)), _unicode_encode('' if groups is None else ','.join(str(group) for group in groups)), _unicode_encode('' if umask is None else str(umask)), _unicode_encode(','.join(str(fd) for fd in fd_pipes)), binary] + myargs uid = None gid = None groups = None umask = None else: # Execute a supervisor process which will forward # signals to init and forward exit status to the # parent process. The supervisor process runs in # the global pid namespace, so skip /proc remount # and other setup that's intended only for the # init process. binary, myargs = portage._python_interpreter, [ portage._python_interpreter, os.path.join(portage._bin_path, 'pid-ns-init'), str(main_child_pid)] os.execve(binary, myargs, env) if unshare_mount: # mark the whole filesystem as slave to avoid # mounts escaping the namespace s = subprocess.Popen(['mount', '--make-rslave', '/']) mount_ret = s.wait() if mount_ret != 0: # TODO: should it be fatal maybe? writemsg("Unable to mark mounts slave: %d\n" % (mount_ret,), noiselevel=-1) if unshare_pid: # we need at least /proc being slave s = subprocess.Popen(['mount', '--make-slave', '/proc']) mount_ret = s.wait() if mount_ret != 0: # can't proceed with shared /proc writemsg("Unable to mark /proc slave: %d\n" % (mount_ret,), noiselevel=-1) os._exit(1) # mount new /proc for our namespace s = subprocess.Popen(['mount', '-n', '-t', 'proc', 'proc', '/proc']) mount_ret = s.wait() if mount_ret != 0: writemsg("Unable to mount new /proc: %d\n" % (mount_ret,), noiselevel=-1) os._exit(1) if unshare_net: # use 'localhost' to avoid hostname resolution problems try: socket.sethostname('localhost') except Exception as e: writemsg("Unable to set hostname: %s (for FEATURES=\"network-sandbox\")\n" % ( e,), noiselevel=-1) _configure_loopback_interface() except AttributeError: # unshare() not supported by libc pass # Set requested process permissions. if gid: # Cast proxies to int, in case it matters. os.setgid(int(gid)) if groups: os.setgroups(groups) if uid: # Cast proxies to int, in case it matters. os.setuid(int(uid)) if umask: os.umask(umask) if cwd is not None: os.chdir(cwd) if pre_exec: pre_exec() # And switch to the new process. os.execve(binary, myargs, env)
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, pre_exec, close_fds, unshare_net, unshare_ipc, cgroup): """ Execute a given binary with options @param binary: Name of program to execute @type binary: String @param mycommand: Options for program @type mycommand: String @param opt_name: Name of process (defaults to binary) @type opt_name: String @param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 } @type fd_pipes: Dictionary @param env: Key,Value mapping for Environmental Variables @type env: Dictionary @param gid: Group ID to run the process under @type gid: Integer @param groups: Groups the Process should be in. @type groups: Integer @param uid: User ID to run the process under @type uid: Integer @param umask: an int representing a unix umask (see man chmod for umask details) @type umask: Integer @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String @rtype: None @return: Never returns (calls os.execve) """ # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: if binary is portage._python_interpreter: # NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0] # does not contain the full path of the binary. opt_name = binary else: opt_name = os.path.basename(binary) # Set up the command's argument list. myargs = [opt_name] myargs.extend(mycommand[1:]) # Avoid a potential UnicodeEncodeError from os.execve(). myargs = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict') for x in myargs] # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Quiet killing of subprocesses by SIGPIPE (see bug #309001). signal.signal(signal.SIGPIPE, signal.SIG_DFL) # Avoid issues triggered by inheritance of SIGQUIT handler from # the parent process (see bug #289486). signal.signal(signal.SIGQUIT, signal.SIG_DFL) _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True) # Add to cgroup # it's better to do it from the child since we can guarantee # it is done before we start forking children if cgroup: with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: f.write('%d\n' % os.getpid()) # Unshare (while still uid==0) if unshare_net or unshare_ipc: filename = find_library("c") if filename is not None: libc = LoadLibrary(filename) if libc is not None: CLONE_NEWIPC = 0x08000000 CLONE_NEWNET = 0x40000000 flags = 0 if unshare_net: flags |= CLONE_NEWNET if unshare_ipc: flags |= CLONE_NEWIPC try: if libc.unshare(flags) != 0: writemsg("Unable to unshare: %s\n" % ( errno.errorcode.get(ctypes.get_errno(), '?')), noiselevel=-1) else: if unshare_net: # 'up' the loopback IFF_UP = 0x1 ifreq = struct.pack('16sh', b'lo', IFF_UP) SIOCSIFFLAGS = 0x8914 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) try: fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq) except IOError as e: writemsg("Unable to enable loopback interface: %s\n" % ( errno.errorcode.get(e.errno, '?')), noiselevel=-1) sock.close() except AttributeError: # unshare() not supported by libc pass # Set requested process permissions. if gid: # Cast proxies to int, in case it matters. os.setgid(int(gid)) if groups: os.setgroups(groups) if uid: # Cast proxies to int, in case it matters. os.setuid(int(uid)) if umask: os.umask(umask) if pre_exec: pre_exec() # And switch to the new process. os.execve(binary, myargs, env)
def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, logfile=None, path_lookup=True, pre_exec=None, close_fds=True, unshare_net=False, unshare_ipc=False, cgroup=None): """ Spawns a given command. @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: A dict of Key=Value pairs for env variables @type env: Dictionary @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example (default is {0:stdin, 1:stdout, 2:stderr}) @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @param gid: Group ID to spawn as; useful for dropping privilages @type gid: Integer @param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts. @type groups: List @param umask: An integer representing the umask for the process (see man chmod for umask details) @type umask: Integer @param logfile: name of a file to use for logging purposes @type logfile: String @param path_lookup: If the binary is not fully specified then look for it in PATH @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param close_fds: If True, then close all file descriptors except those referenced by fd_pipes (default is True). @type close_fds: Boolean @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) """ # mycommand is either a str or a list if isinstance(mycommand, basestring): mycommand = mycommand.split() if sys.hexversion < 0x3000000: # Avoid a potential UnicodeEncodeError from os.execve(). env_bytes = {} for k, v in env.items(): env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \ _unicode_encode(v, encoding=_encodings['content']) env = env_bytes del env_bytes # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \ (not os.path.isabs(binary) or not os.path.isfile(binary) or not os.access(binary, os.X_OK)): binary = path_lookup and find_binary(binary) or None if not binary: raise CommandNotFound(mycommand[0]) # If we haven't been told what file descriptors to use # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { 0:portage._get_stdin().fileno(), 1:sys.__stdout__.fileno(), 2:sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. mypids = [] if logfile: # Using a log file requires that stdout and stderr # are assigned to the process we're running. if 1 not in fd_pipes or 2 not in fd_pipes: raise ValueError(fd_pipes) # Create a pipe (pr, pw) = os.pipe() # Create a tee process, giving it our stdout and stderr # as well as the read end of the pipe. mypids.extend(spawn(('tee', '-i', '-a', logfile), returnpid=True, fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]})) # We don't need the read end of the pipe, so close it. os.close(pr) # Assign the write end of the pipe to our stdout and stderr. fd_pipes[1] = pw fd_pipes[2] = pw # This caches the libc library lookup in the current # process, so that it's only done once rather than # for each child process. if unshare_net or unshare_ipc: find_library("c") parent_pid = os.getpid() pid = None try: pid = os.fork() if pid == 0: try: _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, pre_exec, close_fds, unshare_net, unshare_ipc, cgroup) except SystemExit: raise except Exception as e: # We need to catch _any_ exception so that it doesn't # propagate out of this function and cause exiting # with anything other than os._exit() writemsg("%s:\n %s\n" % (e, " ".join(mycommand)), noiselevel=-1) traceback.print_exc() sys.stderr.flush() finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1) if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid),)) # Add the pid to our local and the global pid lists. mypids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. if logfile: os.close(pw) # If the caller wants to handle cleaning up the processes, we tell # it about all processes that were created. if returnpid: return mypids # Otherwise we clean them up. while mypids: # Pull the last reader in the pipe chain. If all processes # in the pipe are well behaved, it will die when the process # it is reading from dies. pid = mypids.pop(0) # and wait for it. retval = os.waitpid(pid, 0)[1] if retval: # If it failed, kill off anything else that # isn't dead yet. for pid in mypids: # With waitpid and WNOHANG, only check the # first element of the tuple since the second # element may vary (bug #337465). if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) # If it got a signal, return the signal that was sent. if (retval & 0xff): return ((retval & 0xff) << 8) # Otherwise, return its exit code. return (retval >> 8) # Everything succeeded return 0
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid, unshare_flags, cgroup): """ Execute a given binary with options @param binary: Name of program to execute @type binary: String @param mycommand: Options for program @type mycommand: String @param opt_name: Name of process (defaults to binary) @type opt_name: String @param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 } @type fd_pipes: Dictionary @param env: Key,Value mapping for Environmental Variables @type env: Dictionary @param gid: Group ID to run the process under @type gid: Integer @param groups: Groups the Process should be in. @type groups: List @param uid: User ID to run the process under @type uid: Integer @param umask: an int representing a unix umask (see man chmod for umask details) @type umask: Integer @param cwd: Current working directory @type cwd: String @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param unshare_mount: If True, mount namespace will be unshared and mounts will be private to the namespace @type unshare_mount: Boolean @param unshare_pid: If True, PID ns will be unshared from the spawned process @type unshare_pid: Boolean @param unshare_flags: Flags for the unshare(2) function @type unshare_flags: Integer @param cgroup: CGroup path to bind the process to @type cgroup: String @rtype: None @return: Never returns (calls os.execve) """ # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: if binary is portage._python_interpreter: # NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0] # does not contain the full path of the binary. opt_name = binary else: opt_name = os.path.basename(binary) # Set up the command's argument list. myargs = [opt_name] myargs.extend(mycommand[1:]) # Avoid a potential UnicodeEncodeError from os.execve(). myargs = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict') for x in myargs] # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Unregister SIGCHLD handler and wakeup_fd for the parent # process's event loop (bug 655656). signal.signal(signal.SIGCHLD, signal.SIG_DFL) try: wakeup_fd = signal.set_wakeup_fd(-1) if wakeup_fd > 0: os.close(wakeup_fd) except (ValueError, OSError): pass # Quiet killing of subprocesses by SIGPIPE (see bug #309001). signal.signal(signal.SIGPIPE, signal.SIG_DFL) # Avoid issues triggered by inheritance of SIGQUIT handler from # the parent process (see bug #289486). signal.signal(signal.SIGQUIT, signal.SIG_DFL) _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True) # Add to cgroup # it's better to do it from the child since we can guarantee # it is done before we start forking children if cgroup: with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: f.write('%d\n' % os.getpid()) # Unshare (while still uid==0) if unshare_net or unshare_ipc or unshare_mount or unshare_pid: filename = find_library("c") if filename is not None: libc = LoadLibrary(filename) if libc is not None: try: # Since a failed unshare call could corrupt process # state, first validate that the call can succeed. # The parent process should call _unshare_validate # before it forks, so that all child processes can # reuse _unshare_validate results that have been # cached by the parent process. errno_value = _unshare_validate(unshare_flags) if errno_value == 0 and libc.unshare(unshare_flags) != 0: errno_value = ctypes.get_errno() if errno_value != 0: writemsg("Unable to unshare: %s\n" % ( errno.errorcode.get(errno_value, '?')), noiselevel=-1) else: if unshare_pid: main_child_pid = os.fork() if main_child_pid == 0: # pid namespace requires us to become init binary, myargs = portage._python_interpreter, [ portage._python_interpreter, os.path.join(portage._bin_path, 'pid-ns-init'), _unicode_encode('' if uid is None else str(uid)), _unicode_encode('' if gid is None else str(gid)), _unicode_encode('' if groups is None else ','.join(str(group) for group in groups)), _unicode_encode('' if umask is None else str(umask)), _unicode_encode(','.join(str(fd) for fd in fd_pipes)), binary] + myargs uid = None gid = None groups = None umask = None else: # Execute a supervisor process which will forward # signals to init and forward exit status to the # parent process. The supervisor process runs in # the global pid namespace, so skip /proc remount # and other setup that's intended only for the # init process. binary, myargs = portage._python_interpreter, [ portage._python_interpreter, os.path.join(portage._bin_path, 'pid-ns-init'), str(main_child_pid)] os.execve(binary, myargs, env) if unshare_mount: # mark the whole filesystem as slave to avoid # mounts escaping the namespace s = subprocess.Popen(['mount', '--make-rslave', '/']) mount_ret = s.wait() if mount_ret != 0: # TODO: should it be fatal maybe? writemsg("Unable to mark mounts slave: %d\n" % (mount_ret,), noiselevel=-1) if unshare_pid: # we need at least /proc being slave s = subprocess.Popen(['mount', '--make-slave', '/proc']) mount_ret = s.wait() if mount_ret != 0: # can't proceed with shared /proc writemsg("Unable to mark /proc slave: %d\n" % (mount_ret,), noiselevel=-1) os._exit(1) # mount new /proc for our namespace s = subprocess.Popen(['mount', '-n', '-t', 'proc', 'proc', '/proc']) mount_ret = s.wait() if mount_ret != 0: writemsg("Unable to mount new /proc: %d\n" % (mount_ret,), noiselevel=-1) os._exit(1) if unshare_net: # 'up' the loopback IFF_UP = 0x1 ifreq = struct.pack('16sh', b'lo', IFF_UP) SIOCSIFFLAGS = 0x8914 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) try: fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq) except IOError as e: writemsg("Unable to enable loopback interface: %s\n" % ( errno.errorcode.get(e.errno, '?')), noiselevel=-1) sock.close() except AttributeError: # unshare() not supported by libc pass # Set requested process permissions. if gid: # Cast proxies to int, in case it matters. os.setgid(int(gid)) if groups: os.setgroups(groups) if uid: # Cast proxies to int, in case it matters. os.setuid(int(uid)) if umask: os.umask(umask) if cwd is not None: os.chdir(cwd) if pre_exec: pre_exec() # And switch to the new process. os.execve(binary, myargs, env)
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid, cgroup): """ Execute a given binary with options @param binary: Name of program to execute @type binary: String @param mycommand: Options for program @type mycommand: String @param opt_name: Name of process (defaults to binary) @type opt_name: String @param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 } @type fd_pipes: Dictionary @param env: Key,Value mapping for Environmental Variables @type env: Dictionary @param gid: Group ID to run the process under @type gid: Integer @param groups: Groups the Process should be in. @type groups: Integer @param uid: User ID to run the process under @type uid: Integer @param umask: an int representing a unix umask (see man chmod for umask details) @type umask: Integer @param cwd: Current working directory @type cwd: String @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param unshare_mount: If True, mount namespace will be unshared and mounts will be private to the namespace @type unshare_mount: Boolean @param unshare_pid: If True, PID ns will be unshared from the spawned process @type unshare_pid: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String @rtype: None @return: Never returns (calls os.execve) """ # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: if binary is portage._python_interpreter: # NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0] # does not contain the full path of the binary. opt_name = binary else: opt_name = os.path.basename(binary) # Set up the command's argument list. myargs = [opt_name] myargs.extend(mycommand[1:]) # Avoid a potential UnicodeEncodeError from os.execve(). myargs = [ _unicode_encode(x, encoding=_encodings['fs'], errors='strict') for x in myargs ] # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Unregister SIGCHLD handler and wakeup_fd for the parent # process's event loop (bug 655656). signal.signal(signal.SIGCHLD, signal.SIG_DFL) try: wakeup_fd = signal.set_wakeup_fd(-1) if wakeup_fd > 0: os.close(wakeup_fd) except (ValueError, OSError): pass # Quiet killing of subprocesses by SIGPIPE (see bug #309001). signal.signal(signal.SIGPIPE, signal.SIG_DFL) # Avoid issues triggered by inheritance of SIGQUIT handler from # the parent process (see bug #289486). signal.signal(signal.SIGQUIT, signal.SIG_DFL) _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True) # Add to cgroup # it's better to do it from the child since we can guarantee # it is done before we start forking children if cgroup: with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: f.write('%d\n' % os.getpid()) # Unshare (while still uid==0) if unshare_net or unshare_ipc or unshare_mount or unshare_pid: filename = find_library("c") if filename is not None: libc = LoadLibrary(filename) if libc is not None: # from /usr/include/bits/sched.h CLONE_NEWNS = 0x00020000 CLONE_NEWIPC = 0x08000000 CLONE_NEWPID = 0x20000000 CLONE_NEWNET = 0x40000000 flags = 0 if unshare_net: flags |= CLONE_NEWNET if unshare_ipc: flags |= CLONE_NEWIPC if unshare_mount: # NEWNS = mount namespace flags |= CLONE_NEWNS if unshare_pid: # we also need mount namespace for slave /proc flags |= CLONE_NEWPID | CLONE_NEWNS try: if libc.unshare(flags) != 0: writemsg( "Unable to unshare: %s\n" % (errno.errorcode.get(ctypes.get_errno(), '?')), noiselevel=-1) else: if unshare_pid: # pid namespace requires us to become init fork_ret = os.fork() if fork_ret != 0: os.execv(portage._python_interpreter, [ portage._python_interpreter, os.path.join(portage._bin_path, 'pid-ns-init'), '%s' % fork_ret, ]) if unshare_mount: # mark the whole filesystem as slave to avoid # mounts escaping the namespace s = subprocess.Popen( ['mount', '--make-rslave', '/']) mount_ret = s.wait() if mount_ret != 0: # TODO: should it be fatal maybe? writemsg("Unable to mark mounts slave: %d\n" % (mount_ret, ), noiselevel=-1) if unshare_pid: # we need at least /proc being slave s = subprocess.Popen( ['mount', '--make-slave', '/proc']) mount_ret = s.wait() if mount_ret != 0: # can't proceed with shared /proc writemsg("Unable to mark /proc slave: %d\n" % (mount_ret, ), noiselevel=-1) os._exit(1) # mount new /proc for our namespace s = subprocess.Popen( ['mount', '-t', 'proc', 'proc', '/proc']) mount_ret = s.wait() if mount_ret != 0: writemsg("Unable to mount new /proc: %d\n" % (mount_ret, ), noiselevel=-1) os._exit(1) if unshare_net: # 'up' the loopback IFF_UP = 0x1 ifreq = struct.pack('16sh', b'lo', IFF_UP) SIOCSIFFLAGS = 0x8914 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) try: fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq) except IOError as e: writemsg( "Unable to enable loopback interface: %s\n" % (errno.errorcode.get(e.errno, '?')), noiselevel=-1) sock.close() except AttributeError: # unshare() not supported by libc pass # Set requested process permissions. if gid: # Cast proxies to int, in case it matters. os.setgid(int(gid)) if groups: os.setgroups(groups) if uid: # Cast proxies to int, in case it matters. os.setuid(int(uid)) if umask: os.umask(umask) if cwd is not None: os.chdir(cwd) if pre_exec: pre_exec() # And switch to the new process. os.execve(binary, myargs, env)