def _spawn(self, args, fd_pipes=None, **kwargs): """ Override SpawnProcess._spawn to fork a subprocess that calls self._run(). This uses multiprocessing.Process in order to leverage any pre-fork and post-fork interpreter housekeeping that it provides, promoting a healthy state for the forked interpreter. """ # Since multiprocessing.Process closes sys.__stdin__, create a # temporary duplicate of fd_pipes[0] so that sys.__stdin__ can # be restored in the subprocess, in case this is needed for # things like PROPERTIES=interactive support. stdin_dup = None try: stdin_fd = fd_pipes.get(0) if stdin_fd is not None and stdin_fd == portage._get_stdin( ).fileno(): stdin_dup = os.dup(stdin_fd) fcntl.fcntl(stdin_dup, fcntl.F_SETFD, fcntl.fcntl(stdin_fd, fcntl.F_GETFD)) fd_pipes[0] = stdin_dup self._proc = multiprocessing.Process(target=self._bootstrap, args=(fd_pipes, )) self._proc.start() finally: if stdin_dup is not None: os.close(stdin_dup) self._proc_join_task = asyncio.ensure_future(self._proc_join( self._proc, loop=self.scheduler), loop=self.scheduler) self._proc_join_task.add_done_callback( functools.partial(self._proc_join_done, self._proc)) return [self._proc.pid]
def xtermTitleReset(): global default_xterm_title if default_xterm_title is None: prompt_command = os.environ.get('PROMPT_COMMAND') if prompt_command == "": default_xterm_title = "" elif prompt_command is not None: if dotitles and \ 'TERM' in os.environ and \ _legal_terms_re.match(os.environ['TERM']) is not None and \ sys.__stderr__.isatty(): from portage.process import find_binary, spawn shell = os.environ.get("SHELL") if not shell or not os.access(shell, os.EX_OK): shell = find_binary("sh") if shell: spawn([shell, "-c", prompt_command], env=os.environ, fd_pipes={ 0: portage._get_stdin().fileno(), 1: sys.__stderr__.fileno(), 2: sys.__stderr__.fileno() }) else: os.system(prompt_command) return else: pwd = os.environ.get('PWD','') home = os.environ.get('HOME', '') if home != '' and pwd.startswith(home): pwd = '~' + pwd[len(home):] default_xterm_title = '\x1b]0;%s@%s:%s\x07' % ( os.environ.get('LOGNAME', ''), os.environ.get('HOSTNAME', '').split('.', 1)[0], pwd) xtermTitle(default_xterm_title, raw=True)
def _start(self): # Portage should always call setcpv prior to this # point, but here we have a fallback as a convenience # for external API consumers. It's important that # this metadata access happens in the parent process, # since closing of file descriptors in the subprocess # can prevent access to open database connections such # as that used by the sqlite metadata cache module. cpv = "%s/%s" % (self.mycat, self.mypkg) settings = self.settings if cpv != settings.mycpv or "EAPI" not in settings.configdict["pkg"]: settings.reload() settings.reset() settings.setcpv(cpv, mydb=self.mydbapi) # This caches the libc library lookup in the current # process, so that it's only done once rather than # for each child process. if platform.system() == "Linux" and "merge-sync" in settings.features: find_library("c") # Inherit stdin by default, so that the pdb SIGUSR1 # handler is usable for the subprocess. if self.fd_pipes is None: self.fd_pipes = {} else: self.fd_pipes = self.fd_pipes.copy() self.fd_pipes.setdefault(0, portage._get_stdin().fileno()) self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD") super(MergeProcess, self)._start()
def _start(self): # Portage should always call setcpv prior to this # point, but here we have a fallback as a convenience # for external API consumers. It's important that # this metadata access happens in the parent process, # since closing of file descriptors in the subprocess # can prevent access to open database connections such # as that used by the sqlite metadata cache module. cpv = "%s/%s" % (self.mycat, self.mypkg) settings = self.settings if cpv != settings.mycpv or \ "EAPI" not in settings.configdict["pkg"]: settings.reload() settings.reset() settings.setcpv(cpv, mydb=self.mydbapi) # This caches the libc library lookup in the current # process, so that it's only done once rather than # for each child process. if platform.system() == "Linux" and \ "merge-sync" in settings.features: find_library("c") # Inherit stdin by default, so that the pdb SIGUSR1 # handler is usable for the subprocess. if self.fd_pipes is None: self.fd_pipes = {} else: self.fd_pipes = self.fd_pipes.copy() self.fd_pipes.setdefault(0, portage._get_stdin().fileno()) super(MergeProcess, self)._start()
def file_get(baseurl=None, dest=None, conn=None, fcmd=None, filename=None, fcmd_vars=None): """Takes a base url to connect to and read from. URI should be in the form <proto>://[user[:pass]@]<site>[:port]<path>""" if not fcmd: warnings.warn( "Use of portage.getbinpkg.file_get() without the fcmd " "parameter is deprecated", DeprecationWarning, stacklevel=2, ) return file_get_lib(baseurl, dest, conn) variables = {} if fcmd_vars is not None: variables.update(fcmd_vars) if "DISTDIR" not in variables: if dest is None: raise portage.exception.MissingParameter( _("%s is missing required '%s' key") % ("fcmd_vars", "DISTDIR")) variables["DISTDIR"] = dest if "URI" not in variables: if baseurl is None: raise portage.exception.MissingParameter( _("%s is missing required '%s' key") % ("fcmd_vars", "URI")) variables["URI"] = baseurl if "FILE" not in variables: if filename is None: filename = os.path.basename(variables["URI"]) variables["FILE"] = filename from portage.util import varexpand from portage.process import spawn myfetch = portage.util.shlex_split(fcmd) myfetch = [varexpand(x, mydict=variables) for x in myfetch] fd_pipes = { 0: portage._get_stdin().fileno(), 1: sys.__stdout__.fileno(), 2: sys.__stdout__.fileno(), } sys.__stdout__.flush() sys.__stderr__.flush() retval = spawn(myfetch, env=os.environ.copy(), fd_pipes=fd_pipes) if retval != os.EX_OK: sys.stderr.write(_("Fetcher exited with a failure condition.\n")) return 0 return 1
def _spawn_fetch(settings, args, **kwargs): """ Spawn a process with appropriate settings for fetching, including userfetch and selinux support. """ global _userpriv_spawn_kwargs # Redirect all output to stdout since some fetchers like # wget pollute stderr (if portage detects a problem then it # can send it's own message to stderr). if "fd_pipes" not in kwargs: kwargs["fd_pipes"] = { 0 : portage._get_stdin().fileno(), 1 : sys.__stdout__.fileno(), 2 : sys.__stdout__.fileno(), } logname = None if "userfetch" in settings.features and \ os.getuid() == 0 and portage_gid and portage_uid and \ hasattr(os, "setgroups"): kwargs.update(_userpriv_spawn_kwargs) logname = portage.data._portage_username spawn_func = spawn if settings.selinux_enabled(): spawn_func = selinux.spawn_wrapper(spawn_func, settings["PORTAGE_FETCH_T"]) # bash is an allowed entrypoint, while most binaries are not if args[0] != BASH_BINARY: args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args # Ensure that EBUILD_PHASE is set to fetch, so that config.environ() # does not filter the calling environment (which may contain needed # proxy variables, as in bug #315421). phase_backup = settings.get('EBUILD_PHASE') settings['EBUILD_PHASE'] = 'fetch' env = settings.environ() if logname is not None: env["LOGNAME"] = logname try: rval = spawn_func(args, env=env, **kwargs) finally: if phase_backup is None: settings.pop('EBUILD_PHASE', None) else: settings['EBUILD_PHASE'] = phase_backup return rval
def file_get(baseurl=None, dest=None, conn=None, fcmd=None, filename=None, fcmd_vars=None): """Takes a base url to connect to and read from. URI should be in the form <proto>://[user[:pass]@]<site>[:port]<path>""" if not fcmd: warnings.warn("Use of portage.getbinpkg.file_get() without the fcmd " "parameter is deprecated", DeprecationWarning, stacklevel=2) return file_get_lib(baseurl, dest, conn) variables = {} if fcmd_vars is not None: variables.update(fcmd_vars) if "DISTDIR" not in variables: if dest is None: raise portage.exception.MissingParameter( _("%s is missing required '%s' key") % ("fcmd_vars", "DISTDIR")) variables["DISTDIR"] = dest if "URI" not in variables: if baseurl is None: raise portage.exception.MissingParameter( _("%s is missing required '%s' key") % ("fcmd_vars", "URI")) variables["URI"] = baseurl if "FILE" not in variables: if filename is None: filename = os.path.basename(variables["URI"]) variables["FILE"] = filename from portage.util import varexpand from portage.process import spawn myfetch = portage.util.shlex_split(fcmd) myfetch = [varexpand(x, mydict=variables) for x in myfetch] fd_pipes = { 0: portage._get_stdin().fileno(), 1: sys.__stdout__.fileno(), 2: sys.__stdout__.fileno() } sys.__stdout__.flush() sys.__stderr__.flush() retval = spawn(myfetch, env=os.environ.copy(), fd_pipes=fd_pipes) if retval != os.EX_OK: sys.stderr.write(_("Fetcher exited with a failure condition.\n")) return 0 return 1
def testLogfile(self): logfile = None try: fd, logfile = tempfile.mkstemp() os.close(fd) null_fd = os.open('/dev/null', os.O_RDWR) test_string = 2 * "blah blah blah\n" proc = SpawnProcess( args=[BASH_BINARY, "-c", "echo -n '%s'" % test_string], env={}, fd_pipes={ 0: portage._get_stdin().fileno(), 1: null_fd, 2: null_fd }, scheduler=global_event_loop(), logfile=logfile) global_event_loop().run_until_complete(proc.async_start()) os.close(null_fd) self.assertEqual(proc.wait(), os.EX_OK) f = io.open(_unicode_encode(logfile, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='strict') log_content = f.read() f.close() # When logging passes through a pty, this comparison will fail # unless the oflag terminal attributes have the termios.OPOST # bit disabled. Otherwise, tranformations such as \n -> \r\n # may occur. self.assertEqual(test_string, log_content) finally: if logfile: try: os.unlink(logfile) except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e
def xtermTitleReset(): global default_xterm_title if default_xterm_title is None: prompt_command = os.environ.get("PROMPT_COMMAND") if prompt_command == "": default_xterm_title = "" elif prompt_command is not None: if ( dotitles and "TERM" in os.environ and _legal_terms_re.match(os.environ["TERM"]) is not None and sys.__stderr__.isatty() ): from portage.process import find_binary, spawn shell = os.environ.get("SHELL") if not shell or not os.access(shell, os.EX_OK): shell = find_binary("sh") if shell: spawn( [shell, "-c", prompt_command], env=os.environ, fd_pipes={ 0: portage._get_stdin().fileno(), 1: sys.__stderr__.fileno(), 2: sys.__stderr__.fileno(), }, ) else: os.system(prompt_command) return else: pwd = os.environ.get("PWD", "") home = os.environ.get("HOME", "") if home != "" and pwd.startswith(home): pwd = "~" + pwd[len(home) :] default_xterm_title = "\x1b]0;%s@%s:%s\x07" % ( os.environ.get("LOGNAME", ""), os.environ.get("HOSTNAME", "").split(".", 1)[0], pwd, ) xtermTitle(default_xterm_title, raw=True)
def sanitize_fds(): """ Set the inheritable flag to False for all open file descriptors, except for those corresponding to stdin, stdout, and stderr. This ensures that any unintentionally inherited file descriptors will not be inherited by child processes. """ if _set_inheritable is not None: whitelist = frozenset([ portage._get_stdin().fileno(), sys.__stdout__.fileno(), sys.__stderr__.fileno(), ]) for fd in get_open_fds(): if fd not in whitelist: try: _set_inheritable(fd, False) except OSError: pass
def testLogfile(self): logfile = None try: fd, logfile = tempfile.mkstemp() os.close(fd) null_fd = os.open('/dev/null', os.O_RDWR) test_string = 2 * "blah blah blah\n" proc = SpawnProcess( args=[BASH_BINARY, "-c", "echo -n '%s'" % test_string], env={}, fd_pipes={ 0: portage._get_stdin().fileno(), 1: null_fd, 2: null_fd }, scheduler=global_event_loop(), logfile=logfile) proc.start() os.close(null_fd) self.assertEqual(proc.wait(), os.EX_OK) f = io.open(_unicode_encode(logfile, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='strict') log_content = f.read() f.close() # When logging passes through a pty, this comparison will fail # unless the oflag terminal attributes have the termios.OPOST # bit disabled. Otherwise, tranformations such as \n -> \r\n # may occur. self.assertEqual(test_string, log_content) finally: if logfile: try: os.unlink(logfile) except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e
def _start(self): pkg = self.pkg pretend = self.pretend bintree = pkg.root_config.trees["bintree"] settings = bintree.settings pkg_path = self.pkg_path exists = os.path.exists(pkg_path) resume = exists and os.path.basename(pkg_path) in bintree.invalids if not (pretend or resume): # Remove existing file or broken symlink. try: os.unlink(pkg_path) except OSError: pass # urljoin doesn't work correctly with # unrecognized protocols like sftp fetchcommand = None resumecommand = None if bintree._remote_has_index: remote_metadata = bintree._remotepkgs[bintree.dbapi._instance_key( pkg.cpv)] rel_uri = remote_metadata.get("PATH") if not rel_uri: rel_uri = pkg.cpv + ".tbz2" remote_base_uri = remote_metadata["BASE_URI"] uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/") fetchcommand = remote_metadata.get('FETCHCOMMAND') resumecommand = remote_metadata.get('RESUMECOMMAND') else: uri = settings["PORTAGE_BINHOST"].rstrip("/") + \ "/" + pkg.pf + ".tbz2" if pretend: portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1) self.returncode = os.EX_OK self._async_wait() return fcmd = None if resume: fcmd = resumecommand else: fcmd = fetchcommand if fcmd is None: protocol = urllib_parse_urlparse(uri)[0] fcmd_prefix = "FETCHCOMMAND" if resume: fcmd_prefix = "RESUMECOMMAND" fcmd = settings.get(fcmd_prefix + "_" + protocol.upper()) if not fcmd: fcmd = settings.get(fcmd_prefix) fcmd_vars = { "DISTDIR": os.path.dirname(pkg_path), "URI": uri, "FILE": os.path.basename(pkg_path) } for k in ("PORTAGE_SSH_OPTS", ): v = settings.get(k) if v is not None: fcmd_vars[k] = v fetch_env = dict(settings.items()) fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \ for x in portage.util.shlex_split(fcmd)] if self.fd_pipes is None: self.fd_pipes = {} fd_pipes = self.fd_pipes # Redirect all output to stdout since some fetchers like # wget pollute stderr (if portage detects a problem then it # can send it's own message to stderr). fd_pipes.setdefault(0, portage._get_stdin().fileno()) fd_pipes.setdefault(1, sys.__stdout__.fileno()) fd_pipes.setdefault(2, sys.__stdout__.fileno()) self.args = fetch_args self.env = fetch_env if settings.selinux_enabled(): self._selinux_type = settings["PORTAGE_FETCH_T"] self.log_filter_file = settings.get('PORTAGE_LOG_FILTER_FILE_CMD') SpawnProcess._start(self)
def spawn( mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, cwd=None, logfile=None, path_lookup=True, pre_exec=None, close_fds=False, unshare_net=False, unshare_ipc=False, unshare_mount=False, unshare_pid=False, cgroup=None, ): """ Spawns a given command. @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: If env is not None, it must be a mapping that defines the environment variables for the new process; these are used instead of the default behavior of inheriting the current process's environment. @type env: None or Mapping @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example (default is {0:stdin, 1:stdout, 2:stderr}) @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @param gid: Group ID to spawn as; useful for dropping privilages @type gid: Integer @param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts. @type groups: List @param umask: An integer representing the umask for the process (see man chmod for umask details) @type umask: Integer @param cwd: Current working directory @type cwd: String @param logfile: name of a file to use for logging purposes @type logfile: String @param path_lookup: If the binary is not fully specified then look for it in PATH @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param close_fds: If True, then close all file descriptors except those referenced by fd_pipes (default is True for python3.3 and earlier, and False for python3.4 and later due to non-inheritable file descriptor behavior from PEP 446). @type close_fds: Boolean @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param unshare_mount: If True, mount namespace will be unshared and mounts will be private to the namespace @type unshare_mount: Boolean @param unshare_pid: If True, PID ns will be unshared from the spawned process @type unshare_pid: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) """ # mycommand is either a str or a list if isinstance(mycommand, str): mycommand = mycommand.split() env = os.environ if env is None else env # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and (not os.path.isabs(binary) or not os.path.isfile(binary) or not os.access(binary, os.X_OK)): binary = path_lookup and find_binary(binary) or None if not binary: raise CommandNotFound(mycommand[0]) # If we haven't been told what file descriptors to use # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { 0: portage._get_stdin().fileno(), 1: sys.__stdout__.fileno(), 2: sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. mypids = [] if logfile: # Using a log file requires that stdout and stderr # are assigned to the process we're running. if 1 not in fd_pipes or 2 not in fd_pipes: raise ValueError(fd_pipes) # Create a pipe (pr, pw) = os.pipe() # Create a tee process, giving it our stdout and stderr # as well as the read end of the pipe. mypids.extend( spawn( ("tee", "-i", "-a", logfile), returnpid=True, fd_pipes={ 0: pr, 1: fd_pipes[1], 2: fd_pipes[2] }, )) # We don't need the read end of the pipe, so close it. os.close(pr) # Assign the write end of the pipe to our stdout and stderr. fd_pipes[1] = pw fd_pipes[2] = pw # Cache _has_ipv6() result for use in child processes. _has_ipv6() # This caches the libc library lookup and _unshare_validator results # in the current process, so that results are cached for use in # child processes. unshare_flags = 0 if unshare_net or unshare_ipc or unshare_mount or unshare_pid: # from /usr/include/bits/sched.h CLONE_NEWNS = 0x00020000 CLONE_NEWUTS = 0x04000000 CLONE_NEWIPC = 0x08000000 CLONE_NEWPID = 0x20000000 CLONE_NEWNET = 0x40000000 if unshare_net: # UTS namespace to override hostname unshare_flags |= CLONE_NEWNET | CLONE_NEWUTS if unshare_ipc: unshare_flags |= CLONE_NEWIPC if unshare_mount: # NEWNS = mount namespace unshare_flags |= CLONE_NEWNS if unshare_pid: # we also need mount namespace for slave /proc unshare_flags |= CLONE_NEWPID | CLONE_NEWNS _unshare_validate(unshare_flags) # Force instantiation of portage.data.userpriv_groups before the # fork, so that the result is cached in the main process. bool(groups) parent_pid = portage.getpid() pid = None try: pid = os.fork() if pid == 0: portage._ForkWatcher.hook(portage._ForkWatcher) try: _exec( binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid, unshare_flags, cgroup, ) except SystemExit: raise except Exception as e: # We need to catch _any_ exception so that it doesn't # propagate out of this function and cause exiting # with anything other than os._exit() writemsg("%s:\n %s\n" % (e, " ".join(mycommand)), noiselevel=-1) traceback.print_exc() sys.stderr.flush() finally: # Don't used portage.getpid() here, due to a race with the above # portage._ForkWatcher cache update. if pid == 0 or (pid is None and _os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1) if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid), )) # Add the pid to our local and the global pid lists. mypids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. if logfile: os.close(pw) # If the caller wants to handle cleaning up the processes, we tell # it about all processes that were created. if returnpid: return mypids # Otherwise we clean them up. while mypids: # Pull the last reader in the pipe chain. If all processes # in the pipe are well behaved, it will die when the process # it is reading from dies. pid = mypids.pop(0) # and wait for it. retval = os.waitpid(pid, 0)[1] if retval: # If it failed, kill off anything else that # isn't dead yet. for pid in mypids: # With waitpid and WNOHANG, only check the # first element of the tuple since the second # element may vary (bug #337465). if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) # If it got a signal, return the signal that was sent. if retval & 0xFF: return (retval & 0xFF) << 8 # Otherwise, return its exit code. return retval >> 8 # Everything succeeded return 0
def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, logfile=None, path_lookup=True, pre_exec=None, close_fds=(sys.version_info < (3, 4)), unshare_net=False, unshare_ipc=False, cgroup=None): """ Spawns a given command. @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: A dict of Key=Value pairs for env variables @type env: Dictionary @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example (default is {0:stdin, 1:stdout, 2:stderr}) @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @param gid: Group ID to spawn as; useful for dropping privilages @type gid: Integer @param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts. @type groups: List @param umask: An integer representing the umask for the process (see man chmod for umask details) @type umask: Integer @param logfile: name of a file to use for logging purposes @type logfile: String @param path_lookup: If the binary is not fully specified then look for it in PATH @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param close_fds: If True, then close all file descriptors except those referenced by fd_pipes (default is True for python3.3 and earlier, and False for python3.4 and later due to non-inheritable file descriptor behavior from PEP 446). @type close_fds: Boolean @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) """ # mycommand is either a str or a list if isinstance(mycommand, basestring): mycommand = mycommand.split() if sys.hexversion < 0x3000000: # Avoid a potential UnicodeEncodeError from os.execve(). env_bytes = {} for k, v in env.items(): env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \ _unicode_encode(v, encoding=_encodings['content']) env = env_bytes del env_bytes # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \ (not os.path.isabs(binary) or not os.path.isfile(binary) or not os.access(binary, os.X_OK)): binary = path_lookup and find_binary(binary) or None if not binary: raise CommandNotFound(mycommand[0]) # If we haven't been told what file descriptors to use # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { 0: portage._get_stdin().fileno(), 1: sys.__stdout__.fileno(), 2: sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. mypids = [] if logfile: # Using a log file requires that stdout and stderr # are assigned to the process we're running. if 1 not in fd_pipes or 2 not in fd_pipes: raise ValueError(fd_pipes) # Create a pipe (pr, pw) = os.pipe() # Create a tee process, giving it our stdout and stderr # as well as the read end of the pipe. mypids.extend( spawn(('tee', '-i', '-a', logfile), returnpid=True, fd_pipes={ 0: pr, 1: fd_pipes[1], 2: fd_pipes[2] })) # We don't need the read end of the pipe, so close it. os.close(pr) # Assign the write end of the pipe to our stdout and stderr. fd_pipes[1] = pw fd_pipes[2] = pw # This caches the libc library lookup in the current # process, so that it's only done once rather than # for each child process. if unshare_net or unshare_ipc: find_library("c") # Force instantiation of portage.data.userpriv_groups before the # fork, so that the result is cached in the main process. bool(groups) parent_pid = os.getpid() pid = None try: pid = os.fork() if pid == 0: try: _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, pre_exec, close_fds, unshare_net, unshare_ipc, cgroup) except SystemExit: raise except Exception as e: # We need to catch _any_ exception so that it doesn't # propagate out of this function and cause exiting # with anything other than os._exit() writemsg("%s:\n %s\n" % (e, " ".join(mycommand)), noiselevel=-1) traceback.print_exc() sys.stderr.flush() finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1) if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid), )) # Add the pid to our local and the global pid lists. mypids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. if logfile: os.close(pw) # If the caller wants to handle cleaning up the processes, we tell # it about all processes that were created. if returnpid: return mypids # Otherwise we clean them up. while mypids: # Pull the last reader in the pipe chain. If all processes # in the pipe are well behaved, it will die when the process # it is reading from dies. pid = mypids.pop(0) # and wait for it. retval = os.waitpid(pid, 0)[1] if retval: # If it failed, kill off anything else that # isn't dead yet. for pid in mypids: # With waitpid and WNOHANG, only check the # first element of the tuple since the second # element may vary (bug #337465). if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) # If it got a signal, return the signal that was sent. if (retval & 0xff): return ((retval & 0xff) << 8) # Otherwise, return its exit code. return (retval >> 8) # Everything succeeded return 0
def _start(self): if self.fd_pipes is None: self.fd_pipes = {} else: self.fd_pipes = self.fd_pipes.copy() fd_pipes = self.fd_pipes master_fd, slave_fd = self._pipe(fd_pipes) can_log = self._can_log(slave_fd) if can_log: log_file_path = self.logfile else: log_file_path = None null_input = None if not self.background or 0 in fd_pipes: # Subclasses such as AbstractEbuildProcess may have already passed # in a null file descriptor in fd_pipes, so use that when given. pass else: # TODO: Use job control functions like tcsetpgrp() to control # access to stdin. Until then, use /dev/null so that any # attempts to read from stdin will immediately return EOF # instead of blocking indefinitely. null_input = os.open('/dev/null', os.O_RDWR) fd_pipes[0] = null_input fd_pipes.setdefault(0, portage._get_stdin().fileno()) fd_pipes.setdefault(1, sys.__stdout__.fileno()) fd_pipes.setdefault(2, sys.__stderr__.fileno()) # flush any pending output stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno()) for fd in fd_pipes.values(): if fd in stdout_filenos: sys.__stdout__.flush() sys.__stderr__.flush() break fd_pipes_orig = fd_pipes.copy() if log_file_path is not None or self.background: fd_pipes[1] = slave_fd fd_pipes[2] = slave_fd else: # Create a dummy pipe that PipeLogger uses to efficiently # monitor for process exit by listening for the EOF event. # Re-use of the allocated fd number for the key in fd_pipes # guarantees that the keys will not collide for similarly # allocated pipes which are used by callers such as # FileDigester and MergeProcess. See the _setup_pipes # docstring for more benefits of this allocation approach. self._dummy_pipe_fd = slave_fd fd_pipes[slave_fd] = slave_fd kwargs = {} for k in self._spawn_kwarg_names: v = getattr(self, k) if v is not None: kwargs[k] = v kwargs["fd_pipes"] = fd_pipes kwargs["returnpid"] = True kwargs.pop("logfile", None) retval = self._spawn(self.args, **kwargs) os.close(slave_fd) if null_input is not None: os.close(null_input) if isinstance(retval, int): # spawn failed self._unregister() self._set_returncode((self.pid, retval)) self._async_wait() return self.pid = retval[0] stdout_fd = None if can_log and not self.background: stdout_fd = os.dup(fd_pipes_orig[1]) # FD_CLOEXEC is enabled by default in Python >=3.4. if sys.hexversion < 0x3040000 and fcntl is not None: try: fcntl.FD_CLOEXEC except AttributeError: pass else: fcntl.fcntl(stdout_fd, fcntl.F_SETFD, fcntl.fcntl(stdout_fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) self._pipe_logger = PipeLogger(background=self.background, scheduler=self.scheduler, input_fd=master_fd, log_file_path=log_file_path, stdout_fd=stdout_fd) self._pipe_logger.addExitListener(self._pipe_logger_exit) self._pipe_logger.start() self._registered = True
def _start(self): if self.fd_pipes is None: self.fd_pipes = {} else: self.fd_pipes = self.fd_pipes.copy() fd_pipes = self.fd_pipes master_fd, slave_fd = self._pipe(fd_pipes) can_log = self._can_log(slave_fd) if can_log: log_file_path = self.logfile else: log_file_path = None null_input = None if not self.background or 0 in fd_pipes: # Subclasses such as AbstractEbuildProcess may have already passed # in a null file descriptor in fd_pipes, so use that when given. pass else: # TODO: Use job control functions like tcsetpgrp() to control # access to stdin. Until then, use /dev/null so that any # attempts to read from stdin will immediately return EOF # instead of blocking indefinitely. null_input = os.open('/dev/null', os.O_RDWR) fd_pipes[0] = null_input fd_pipes.setdefault(0, portage._get_stdin().fileno()) fd_pipes.setdefault(1, sys.__stdout__.fileno()) fd_pipes.setdefault(2, sys.__stderr__.fileno()) # flush any pending output stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno()) for fd in fd_pipes.values(): if fd in stdout_filenos: sys.__stdout__.flush() sys.__stderr__.flush() break fd_pipes_orig = fd_pipes.copy() if log_file_path is not None or self.background: fd_pipes[1] = slave_fd fd_pipes[2] = slave_fd else: # Create a dummy pipe that PipeLogger uses to efficiently # monitor for process exit by listening for the EOF event. # Re-use of the allocated fd number for the key in fd_pipes # guarantees that the keys will not collide for similarly # allocated pipes which are used by callers such as # FileDigester and MergeProcess. See the _setup_pipes # docstring for more benefits of this allocation approach. self._dummy_pipe_fd = slave_fd fd_pipes[slave_fd] = slave_fd kwargs = {} for k in self._spawn_kwarg_names: v = getattr(self, k) if v is not None: kwargs[k] = v kwargs["fd_pipes"] = fd_pipes kwargs["returnpid"] = True kwargs.pop("logfile", None) retval = self._spawn(self.args, **kwargs) os.close(slave_fd) if null_input is not None: os.close(null_input) if isinstance(retval, int): # spawn failed self.returncode = retval self._async_wait() return self.pid = retval[0] stdout_fd = None if can_log and not self.background: stdout_fd = os.dup(fd_pipes_orig[1]) build_logger = BuildLogger(env=self.env, log_path=log_file_path, log_filter_file=self.log_filter_file, scheduler=self.scheduler) build_logger.start() pipe_logger = PipeLogger(background=self.background, scheduler=self.scheduler, input_fd=master_fd, log_file_path=build_logger.stdin, stdout_fd=stdout_fd) pipe_logger.start() self._registered = True self._main_task = asyncio.ensure_future(self._main( build_logger, pipe_logger), loop=self.scheduler) self._main_task.add_done_callback(self._main_exit)
def _start(self): pkg = self.pkg pretend = self.pretend bintree = pkg.root_config.trees["bintree"] settings = bintree.settings use_locks = "distlocks" in settings.features pkg_path = self.pkg_path if not pretend: portage.util.ensure_dirs(os.path.dirname(pkg_path)) if use_locks: self.lock() exists = os.path.exists(pkg_path) resume = exists and os.path.basename(pkg_path) in bintree.invalids if not (pretend or resume): # Remove existing file or broken symlink. try: os.unlink(pkg_path) except OSError: pass # urljoin doesn't work correctly with # unrecognized protocols like sftp if bintree._remote_has_index: instance_key = bintree.dbapi._instance_key(pkg.cpv) rel_uri = bintree._remotepkgs[instance_key].get("PATH") if not rel_uri: rel_uri = pkg.cpv + ".tbz2" remote_base_uri = bintree._remotepkgs[ instance_key]["BASE_URI"] uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/") else: uri = settings["PORTAGE_BINHOST"].rstrip("/") + \ "/" + pkg.pf + ".tbz2" if pretend: portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1) self._set_returncode((self.pid, os.EX_OK << 8)) self._async_wait() return protocol = urllib_parse_urlparse(uri)[0] fcmd_prefix = "FETCHCOMMAND" if resume: fcmd_prefix = "RESUMECOMMAND" fcmd = settings.get(fcmd_prefix + "_" + protocol.upper()) if not fcmd: fcmd = settings.get(fcmd_prefix) fcmd_vars = { "DISTDIR" : os.path.dirname(pkg_path), "URI" : uri, "FILE" : os.path.basename(pkg_path) } for k in ("PORTAGE_SSH_OPTS",): try: fcmd_vars[k] = settings[k] except KeyError: pass fetch_env = dict(settings.items()) fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \ for x in portage.util.shlex_split(fcmd)] if self.fd_pipes is None: self.fd_pipes = {} fd_pipes = self.fd_pipes # Redirect all output to stdout since some fetchers like # wget pollute stderr (if portage detects a problem then it # can send it's own message to stderr). fd_pipes.setdefault(0, portage._get_stdin().fileno()) fd_pipes.setdefault(1, sys.__stdout__.fileno()) fd_pipes.setdefault(2, sys.__stdout__.fileno()) self.args = fetch_args self.env = fetch_env if settings.selinux_enabled(): self._selinux_type = settings["PORTAGE_FETCH_T"] SpawnProcess._start(self)
def pre_sync(self, repo): msg = ">>> Syncing repository '%s' into '%s'..." \ % (repo.name, repo.location) self.logger(self.xterm_titles, msg) writemsg_level(msg + "\n") try: st = os.stat(repo.location) except OSError: st = None self.usersync_uid = None spawn_kwargs = {} # Redirect command stderr to stdout, in order to prevent # spurious cron job emails (bug 566132). spawn_kwargs["fd_pipes"] = { 0: portage._get_stdin().fileno(), 1: sys.__stdout__.fileno(), 2: sys.__stdout__.fileno() } spawn_kwargs["env"] = self.settings.environ() if repo.sync_user is not None: def get_sync_user_data(sync_user): user = None group = None home = None logname = None spl = sync_user.split(':', 1) if spl[0]: username = spl[0] try: try: pw = pwd.getpwnam(username) except KeyError: pw = pwd.getpwuid(int(username)) except (ValueError, KeyError): writemsg("!!! User '%s' invalid or does not exist\n" % username, noiselevel=-1) return (logname, user, group, home) user = pw.pw_uid group = pw.pw_gid home = pw.pw_dir logname = pw.pw_name if len(spl) > 1: groupname = spl[1] try: try: gp = grp.getgrnam(groupname) except KeyError: pw = grp.getgrgid(int(groupname)) except (ValueError, KeyError): writemsg("!!! Group '%s' invalid or does not exist\n" % groupname, noiselevel=-1) return (logname, user, group, home) group = gp.gr_gid return (logname, user, group, home) # user or user:group (logname, uid, gid, home) = get_sync_user_data( repo.sync_user) if uid is not None: spawn_kwargs["uid"] = uid self.usersync_uid = uid if gid is not None: spawn_kwargs["gid"] = gid spawn_kwargs["groups"] = [gid] if home is not None: spawn_kwargs["env"]["HOME"] = home if logname is not None: spawn_kwargs["env"]["LOGNAME"] = logname if st is None: perms = {'mode': 0o755} # respect sync-user if set if 'umask' in spawn_kwargs: perms['mode'] &= ~spawn_kwargs['umask'] if 'uid' in spawn_kwargs: perms['uid'] = spawn_kwargs['uid'] if 'gid' in spawn_kwargs: perms['gid'] = spawn_kwargs['gid'] portage.util.ensure_dirs(repo.location, **perms) st = os.stat(repo.location) if (repo.sync_user is None and 'usersync' in self.settings.features and portage.data.secpass >= 2 and (st.st_uid != os.getuid() and st.st_mode & 0o700 or st.st_gid != os.getgid() and st.st_mode & 0o070)): try: pw = pwd.getpwuid(st.st_uid) except KeyError: pass else: # Drop privileges when syncing, in order to match # existing uid/gid settings. self.usersync_uid = st.st_uid spawn_kwargs["uid"] = st.st_uid spawn_kwargs["gid"] = st.st_gid spawn_kwargs["groups"] = [st.st_gid] spawn_kwargs["env"]["HOME"] = pw.pw_dir spawn_kwargs["env"]["LOGNAME"] = pw.pw_name umask = 0o002 if not st.st_mode & 0o020: umask = umask | 0o020 spawn_kwargs["umask"] = umask # override the defaults when sync_umask is set if repo.sync_umask is not None: spawn_kwargs["umask"] = int(repo.sync_umask, 8) spawn_kwargs.setdefault("umask", 0o022) self.spawn_kwargs = spawn_kwargs if self.usersync_uid is not None: # PORTAGE_TMPDIR is used below, so validate it and # bail out if necessary. rval = _check_temp_dir(self.settings) if rval != os.EX_OK: return rval os.umask(0o022) return os.EX_OK
def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, logfile=None, path_lookup=True, pre_exec=None, close_fds=True, unshare_net=False, unshare_ipc=False, cgroup=None): """ Spawns a given command. @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: A dict of Key=Value pairs for env variables @type env: Dictionary @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example (default is {0:stdin, 1:stdout, 2:stderr}) @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @param gid: Group ID to spawn as; useful for dropping privilages @type gid: Integer @param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts. @type groups: List @param umask: An integer representing the umask for the process (see man chmod for umask details) @type umask: Integer @param logfile: name of a file to use for logging purposes @type logfile: String @param path_lookup: If the binary is not fully specified then look for it in PATH @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param close_fds: If True, then close all file descriptors except those referenced by fd_pipes (default is True). @type close_fds: Boolean @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) """ # mycommand is either a str or a list if isinstance(mycommand, basestring): mycommand = mycommand.split() if sys.hexversion < 0x3000000: # Avoid a potential UnicodeEncodeError from os.execve(). env_bytes = {} for k, v in env.items(): env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \ _unicode_encode(v, encoding=_encodings['content']) env = env_bytes del env_bytes # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \ (not os.path.isabs(binary) or not os.path.isfile(binary) or not os.access(binary, os.X_OK)): binary = path_lookup and find_binary(binary) or None if not binary: raise CommandNotFound(mycommand[0]) # If we haven't been told what file descriptors to use # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { 0:portage._get_stdin().fileno(), 1:sys.__stdout__.fileno(), 2:sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. mypids = [] if logfile: # Using a log file requires that stdout and stderr # are assigned to the process we're running. if 1 not in fd_pipes or 2 not in fd_pipes: raise ValueError(fd_pipes) # Create a pipe (pr, pw) = os.pipe() # Create a tee process, giving it our stdout and stderr # as well as the read end of the pipe. mypids.extend(spawn(('tee', '-i', '-a', logfile), returnpid=True, fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]})) # We don't need the read end of the pipe, so close it. os.close(pr) # Assign the write end of the pipe to our stdout and stderr. fd_pipes[1] = pw fd_pipes[2] = pw # This caches the libc library lookup in the current # process, so that it's only done once rather than # for each child process. if unshare_net or unshare_ipc: find_library("c") parent_pid = os.getpid() pid = None try: pid = os.fork() if pid == 0: try: _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, pre_exec, close_fds, unshare_net, unshare_ipc, cgroup) except SystemExit: raise except Exception as e: # We need to catch _any_ exception so that it doesn't # propagate out of this function and cause exiting # with anything other than os._exit() writemsg("%s:\n %s\n" % (e, " ".join(mycommand)), noiselevel=-1) traceback.print_exc() sys.stderr.flush() finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1) if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid),)) # Add the pid to our local and the global pid lists. mypids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. if logfile: os.close(pw) # If the caller wants to handle cleaning up the processes, we tell # it about all processes that were created. if returnpid: return mypids # Otherwise we clean them up. while mypids: # Pull the last reader in the pipe chain. If all processes # in the pipe are well behaved, it will die when the process # it is reading from dies. pid = mypids.pop(0) # and wait for it. retval = os.waitpid(pid, 0)[1] if retval: # If it failed, kill off anything else that # isn't dead yet. for pid in mypids: # With waitpid and WNOHANG, only check the # first element of the tuple since the second # element may vary (bug #337465). if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) # If it got a signal, return the signal that was sent. if (retval & 0xff): return ((retval & 0xff) << 8) # Otherwise, return its exit code. return (retval >> 8) # Everything succeeded return 0