def _cmd_implementation_sanity_check(self, domain): pkg = self.pkg eapi = pkg.eapi_obj if eapi.options.has_required_use: use = pkg.use for node in pkg.required_use: if not node.match(use): print "REQUIRED_USE requirement weren't met\nFailed to match: %s\nfrom: %s\nfor USE: %s\npkg: %s" % \ (node, pkg.required_use, " ".join(use), pkg) return False if 'pretend' not in pkg.mandatory_phases: return True commands = {"request_inherit": partial(inherit_handler, self._eclass_cache)} env = expected_ebuild_env(pkg) env["ROOT"] = domain.root try: logger.debug("running ebuild pkg_pretend sanity check for %s", pkg) start = time.time() ret = run_generic_phase(pkg, "pretend", env, True, True, False, extra_handlers=commands) logger.debug("pkg_pretend sanity check for %s took %2.2f seconds", pkg, time.time() - start) return ret except format.GenericBuildError, e: logger.error("pkg_pretend sanity check for %s failed with exception %r" % (pkg, e)) return False
def _check_pkg_pretend(self, pkg, *, domain, **kwargs): """Run pkg_pretend phase.""" # pkg_pretend is not defined or required if 'pretend' not in pkg.mandatory_phases: return commands = None if not pkg.built: commands = { 'request_inherit': partial(inherit_handler, self._eclass_cache), 'has_version': ebd_ipc.Has_Version(self), 'best_version': ebd_ipc.Best_Version(self), } # Use base build tempdir for $T instead of full pkg specific path to # avoid having to create/remove directories -- pkg_pretend isn't # allowed to write to the filesystem anyway. self.env = expected_ebuild_env(pkg) self.env["T"] = domain.pm_tmpdir ebd.set_path_vars(self.env, pkg, domain) # avoid clipping eend() messages self.env["PKGCORE_RC_PREFIX"] = '2' with TemporaryFile() as f: # suppress bash output by default fd_pipes = {1: f.fileno(), 2: f.fileno()} try: run_generic_phase( pkg, "pretend", self.env, tmpdir=None, fd_pipes=fd_pipes, userpriv=True, sandbox=True, extra_handlers=commands) except ProcessorError as e: f.seek(0) output = f.read().decode().strip('\n') return errors.PkgPretendError(pkg, output, e)
def _cmd_implementation_sanity_check(self, domain): pkg = self.pkg eapi = pkg.eapi_obj if eapi.options.has_required_use: use = pkg.use for node in pkg.required_use: if not node.match(use): print( textwrap.dedent(""" REQUIRED_USE requirement wasn't met Failed to match: {} from: {} for USE: {} pkg: {} """.format(node, pkg.required_use, " ".join(use), pkg.cpvstr))) return False if 'pretend' not in pkg.mandatory_phases: return True commands = None if not pkg.built: commands = { "request_inherit": partial(inherit_handler, self._eclass_cache) } env = expected_ebuild_env(pkg) tmpdir = normpath(domain._get_tempspace()) builddir = pjoin(tmpdir, env["CATEGORY"], env["PF"]) pkg_tmpdir = normpath(pjoin(builddir, "temp")) ensure_dirs(pkg_tmpdir, mode=0770, gid=portage_gid, minimal=True) env["ROOT"] = domain.root env["T"] = pkg_tmpdir try: logger.debug("running ebuild pkg_pretend sanity check for %s", pkg.cpvstr) start = time.time() ret = run_generic_phase(pkg, "pretend", env, userpriv=True, sandbox=True, fakeroot=False, extra_handlers=commands) logger.debug("pkg_pretend sanity check for %s took %2.2f seconds", pkg.cpvstr, time.time() - start) return ret except format.GenericBuildError as e: logger.error( "pkg_pretend sanity check for %s failed with exception %r" % (pkg.cpvstr, e)) return False finally: shutil.rmtree(builddir) # try to wipe the cat dir; if not empty, ignore it try: os.rmdir(os.path.dirname(builddir)) except EnvironmentError as e: if e.errno != errno.ENOTEMPTY: raise
def run_generic_phase(pkg, phase, env, userpriv, sandbox, fakeroot, extra_handlers=None, failure_allowed=False, logging=None): """ :param phase: phase to execute :param env: environment mapping for the phase :param userpriv: will we drop to :obj:`pkgcore.os_data.portage_uid` and :obj:`pkgcore.os_data.portage_gid` access for this phase? :param sandbox: should this phase be sandboxed? :param fakeroot: should the phase be fakeroot'd? Only really useful for install phase, and is mutually exclusive with sandbox :param extra_handlers: extra command handlers :type extra_handlers: mapping from string to callable :param failure_allowed: allow failure without raising error :type failure_allowed: boolean :param logging: None or a filepath to log output to :return: True when the phase has finished execution """ userpriv = userpriv and is_userpriv_capable() sandbox = sandbox and is_sandbox_capable() fakeroot = fakeroot and is_fakeroot_capable() if env is None: env = expected_ebuild_env(pkg) ebd = request_ebuild_processor(userpriv=userpriv, sandbox=sandbox, fakeroot=fakeroot) # this is a bit of a hack; used until ebd accepts observers that handle # the output redirection on it's own. Primary relevance is when # stdout/stderr are pointed at a file; we leave buffering on, just # force the flush for synchronization. sys.stdout.flush() sys.stderr.flush() try: if not ebd.run_phase(phase, env, env.get('T'), sandbox=sandbox, logging=logging, additional_commands=extra_handlers): if not failure_allowed: raise format.GenericBuildError( phase + ": Failed building (False/0 return from handler)") logger.warning("executing phase %s: execution failed, ignoring" % (phase,)) except Exception as e: ebd.shutdown_processor() release_ebuild_processor(ebd) if isinstance(e, IGNORED_EXCEPTIONS + (format.GenericBuildError,)): raise raise_from( format.GenericBuildError("Executing phase %s: Caught exception: " "%s" % (phase, e))) release_ebuild_processor(ebd) return True
def _cmd_implementation_sanity_check(self, domain): pkg = self.pkg eapi = pkg.eapi_obj if eapi.options.has_required_use: use = pkg.use for node in pkg.required_use: if not node.match(use): print(textwrap.dedent( """ REQUIRED_USE requirement wasn't met Failed to match: {} from: {} for USE: {} pkg: {} """.format(node, pkg.required_use, " ".join(use), pkg.cpvstr) )) return False if 'pretend' not in pkg.mandatory_phases: return True commands = None if not pkg.built: commands = {"request_inherit": partial(inherit_handler, self._eclass_cache)} env = expected_ebuild_env(pkg) tmpdir = normpath(domain._get_tempspace()) builddir = pjoin(tmpdir, env["CATEGORY"], env["PF"]) pkg_tmpdir = normpath(pjoin(builddir, "temp")) ensure_dirs(pkg_tmpdir, mode=0770, gid=portage_gid, minimal=True) env["ROOT"] = domain.root env["T"] = pkg_tmpdir try: logger.debug("running ebuild pkg_pretend sanity check for %s", pkg.cpvstr) start = time.time() ret = run_generic_phase(pkg, "pretend", env, userpriv=True, sandbox=True, fakeroot=False, extra_handlers=commands) logger.debug("pkg_pretend sanity check for %s took %2.2f seconds", pkg.cpvstr, time.time() - start) return ret except format.GenericBuildError as e: logger.error("pkg_pretend sanity check for %s failed with exception %r" % (pkg.cpvstr, e)) return False finally: shutil.rmtree(builddir) # try to wipe the cat dir; if not empty, ignore it try: os.rmdir(os.path.dirname(builddir)) except EnvironmentError as e: if e.errno != errno.ENOTEMPTY: raise
def _check_pkg_pretend(self, pkg, *, domain, **kwargs): """Run pkg_pretend phase.""" # pkg_pretend is not defined or required if 'pretend' not in pkg.mandatory_phases: return True commands = None if not pkg.built: commands = { 'request_inherit': partial(inherit_handler, self._eclass_cache), 'has_version': ebd_ipc.Has_Version(self), 'best_version': ebd_ipc.Best_Version(self), } # Use base build tempdir for $T instead of full pkg specific path to # avoid having to create/remove directories -- pkg_pretend isn't # allowed to write to the filesystem anyway. env = expected_ebuild_env(pkg) env["T"] = domain.pm_tmpdir env["ROOT"] = domain.root # avoid clipping eend() messages env["PKGCORE_RC_PREFIX"] = '2' start = time.time() with TemporaryFile() as f: try: # suppress bash output by default fd_pipes = {1: f.fileno(), 2: f.fileno()} ret = run_generic_phase(pkg, "pretend", env, tmpdir=None, fd_pipes=fd_pipes, userpriv=True, sandbox=True, extra_handlers=commands) logger.debug( "pkg_pretend sanity check for %s took %2.2f seconds", pkg.cpvstr, time.time() - start) return ret except format.GenericBuildError as e: f.seek(0) msg = f.read().decode().strip('\n') raise errors.PkgPretendError(pkg, msg)
def _check_pkg_pretend(self, pkg, *, domain, **kwargs): """Run pkg_pretend phase.""" # pkg_pretend is not defined or required if 'pretend' not in pkg.mandatory_phases: return True commands = None if not pkg.built: commands = { 'request_inherit': partial(inherit_handler, self._eclass_cache), 'has_version': ebd_ipc.Has_Version(self), 'best_version': ebd_ipc.Best_Version(self), } # Use base build tempdir for $T instead of full pkg specific path to # avoid having to create/remove directories -- pkg_pretend isn't # allowed to write to the filesystem anyway. env = expected_ebuild_env(pkg) env["T"] = domain.pm_tmpdir env["ROOT"] = domain.root # avoid clipping eend() messages env["PKGCORE_RC_PREFIX"] = '2' start = time.time() with TemporaryFile() as f: try: # suppress bash output by default fd_pipes = {1: f.fileno(), 2: f.fileno()} ret = run_generic_phase( pkg, "pretend", env, tmpdir=None, fd_pipes=fd_pipes, userpriv=True, sandbox=True, extra_handlers=commands) logger.debug( "pkg_pretend sanity check for %s took %2.2f seconds", pkg.cpvstr, time.time() - start) return ret except format.GenericBuildError as e: f.seek(0) msg = f.read().decode().strip('\n') raise errors.PkgPretendError(pkg, msg)
def __init__(self, pkg, initial_env=None, env_data_source=None, features=None, observer=None, clean=True, tmp_offset=None, use_override=None, allow_fetching=False): """ :param pkg: :class:`pkgcore.ebuild.ebuild_src.package` instance this env is being setup for :param initial_env: initial environment to use for this ebuild :param env_data_source: a :obj:`snakeoil.data_source.base` instance to restore the environment from- used for restoring the state of an ebuild processing, whether for unmerging, or walking phases during building :param features: ebuild features, hold over from portage, will be broken down at some point """ if use_override is not None: use = use_override else: use = pkg.use self.allow_fetching = allow_fetching if not hasattr(self, "observer"): self.observer = observer if not pkg.eapi_obj.is_supported: raise TypeError("package %s uses an unsupported eapi: %s" % (pkg, pkg.eapi)) if initial_env is not None: # copy. self.env = dict(initial_env) for x in ("USE", "ACCEPT_LICENSE"): if x in self.env: del self.env[x] else: self.env = {} # temp hack. for x in ('chost', 'cbuild', 'ctarget'): val = getattr(pkg, x) if val is not None: self.env[x.upper()] = val # special note... if CTARGET is the same as CHOST, suppress it. # certain ebuilds (nano for example) will misbehave w/ it. if pkg.ctarget is not None and pkg.ctarget == pkg.chost: self.env.pop("CTARGET") if "PYTHONPATH" in os.environ: self.env["PYTHONPATH"] = os.environ["PYTHONPATH"] if "PKGCORE_DEBUG" in os.environ: self.env["PKGCORE_DEBUG"] = str(int(os.environ["PKGCORE_DEBUG"])) if features is None: features = self.env.get("FEATURES", ()) # XXX: note this is just eapi3 compatibility; not full prefix, soon.. self.env["ROOT"] = self.domain.root self.prefix_mode = pkg.eapi_obj.options.prefix_capable or 'force-prefix' in features self.env["PKGCORE_PREFIX_SUPPORT"] = 'false' self.prefix = '/' if self.prefix_mode: self.env['EROOT'] = normpath(self.domain.root) self.prefix = self.domain.prefix.lstrip("/") eprefix = normpath(pjoin(self.env["EROOT"], self.prefix)) if eprefix == '/': # Set eprefix to '' if it's basically empty; this keeps certain crappy builds # (cmake for example) from puking over //usr/blah pathways eprefix = '' self.env["EPREFIX"] = eprefix self.env["PKGCORE_PREFIX_SUPPORT"] = 'true' self.env.update(pkg.eapi_obj.get_ebd_env()) # generate a list of internally implemented EAPI specific functions that shouldn't be exported ret, eapi_funcs = spawn_get_output([ pjoin(const.EAPI_BIN_PATH, 'generate_eapi_func_list.bash'), str(pkg.eapi) ]) if ret != 0: raise Exception( "failed to generate list of EAPI %s specific functions" % str(pkg.eapi)) self.env["PKGCORE_EAPI_FUNCS"] = ' '.join(x.strip() for x in eapi_funcs) self.env_data_source = env_data_source if env_data_source is not None and \ not isinstance(env_data_source, data_source.base): raise TypeError( "env_data_source must be None, or a pkgcore.data_source.base " "derivative: %s: %s" % (env_data_source.__class__, env_data_source)) self.features = set(x.lower() for x in features) self.env["FEATURES"] = ' '.join(sorted(self.features)) iuse_effective_regex = (re.escape(x) for x in pkg.iuse_effective) iuse_effective_regex = "^(%s)$" % "|".join(iuse_effective_regex) iuse_effective_regex = iuse_effective_regex.replace("\\.\\*", ".*") self.env["PKGCORE_IUSE_EFFECTIVE"] = iuse_effective_regex expected_ebuild_env(pkg, self.env, env_source_override=self.env_data_source) self.env["PKGCORE_FINALIZED_RESTRICT"] = ' '.join( str(x) for x in pkg.restrict) self.restrict = pkg.restrict for x in ("sandbox", "userpriv", "fakeroot"): setattr(self, x, self.feat_or_bool(x) and not (x in self.restrict)) if self.fakeroot: logger.warning( "disabling fakeroot; unusable till coreutils/fakeroot" + " interaction is fixed") self.fakeroot = False if self.userpriv and os.getuid() != 0: self.userpriv = False if "PORT_LOGDIR" in self.env: self.logging = pjoin( self.env["PORT_LOGDIR"], "%s:%s:%s.log" % (pkg.cpvstr, self.__class__.__name__, time.strftime("%Y%m%d-%H%M%S", time.localtime()))) del self.env["PORT_LOGDIR"] else: self.logging = False self.env["XARGS"] = xargs self.bashrc = self.env.pop("bashrc", ()) self.pkg = pkg self.eapi = pkg.eapi self.eapi_obj = pkg.eapi_obj wipes = [ k for k, v in self.env.iteritems() if not isinstance(v, basestring) ] for k in wipes: del self.env[k] self.set_op_vars(tmp_offset) self.clean_at_start = clean self.clean_needed = False
def _cmd_implementation_sanity_check(self, domain, observer): """Various ebuild sanity checks (REQUIRED_USE, pkg_pretend).""" pkg = self.pkg eapi = pkg.eapi # perform REQUIRED_USE checks if eapi.options.has_required_use: use = pkg.use for node in pkg.required_use: if not node.match(use): observer.info(textwrap.dedent( """ REQUIRED_USE requirement wasn't met Failed to match: {} from: {} for USE: {} pkg: {} """.format(node, pkg.required_use, " ".join(use), pkg.cpvstr) )) return False # return if running pkg_pretend is not required if 'pretend' not in pkg.mandatory_phases: return True # run pkg_pretend phase commands = None if not pkg.built: commands = {"request_inherit": partial(inherit_handler, self._eclass_cache)} env = expected_ebuild_env(pkg) builddir = pjoin(domain.pm_tmpdir, env["CATEGORY"], env["PF"]) pkg_tmpdir = normpath(pjoin(builddir, "temp")) ensure_dirs(pkg_tmpdir, mode=0770, gid=portage_gid, minimal=True) env["ROOT"] = domain.root env["T"] = pkg_tmpdir # TODO: make colored output easier to achieve from observers msg = ['>>> Running pkg_pretend for ', observer._output._out.fg('green'), pkg.cpvstr, observer._output._out.reset] observer._output._out.write(*msg) try: start = time.time() ret = run_generic_phase( pkg, "pretend", env, userpriv=True, sandbox=True, extra_handlers=commands) logger.debug( "pkg_pretend sanity check for %s took %2.2f seconds", pkg.cpvstr, time.time() - start) return ret except format.GenericBuildError as e: return False finally: shutil.rmtree(builddir) # try to wipe the cat dir; if not empty, ignore it try: os.rmdir(os.path.dirname(builddir)) except EnvironmentError as e: # POSIX specifies either ENOTEMPTY or EEXIST for non-empty dir # in particular, Solaris uses EEXIST in that case. # https://github.com/pkgcore/pkgcore/pull/181 if e.errno not in (errno.ENOTEMPTY, errno.EEXIST): raise
def __init__(self, pkg, initial_env=None, env_data_source=None, features=None, observer=None, clean=True, tmp_offset=None, use_override=None, allow_fetching=False): """ :param pkg: :class:`pkgcore.ebuild.ebuild_src.package` instance this env is being setup for :param initial_env: initial environment to use for this ebuild :param env_data_source: a :obj:`snakeoil.data_source.base` instance to restore the environment from- used for restoring the state of an ebuild processing, whether for unmerging, or walking phases during building :param features: ebuild features, hold over from portage, will be broken down at some point """ if use_override is not None: use = use_override else: use = pkg.use self.allow_fetching = allow_fetching if not hasattr(self, "observer"): self.observer = observer if not pkg.eapi.is_supported: raise TypeError( "package %s uses an unsupported eapi: %s" % (pkg, pkg.eapi)) if initial_env is not None: # copy. self.env = dict(initial_env) for x in ("USE", "ACCEPT_LICENSE"): if x in self.env: del self.env[x] else: self.env = {} if "PYTHONPATH" in os.environ: self.env["PYTHONPATH"] = os.environ["PYTHONPATH"] if features is None: features = self.env.get("FEATURES", ()) # XXX: note this is just EAPI 3 compatibility; not full prefix, soon.. self.env["ROOT"] = self.domain.root self.prefix_mode = pkg.eapi.options.prefix_capable or 'force-prefix' in features self.env["PKGCORE_PREFIX_SUPPORT"] = 'false' self.prefix = '/' if self.prefix_mode: self.prefix = self.domain.prefix self.env['EPREFIX'] = self.prefix.rstrip('/') self.env['EROOT'] = abspath( pjoin(self.domain.root, self.prefix.lstrip('/'))).rstrip('/') + '/' self.env["PKGCORE_PREFIX_SUPPORT"] = 'true' # set the list of internally implemented EAPI specific functions that # shouldn't be exported if os.path.exists(pjoin(const.EBD_PATH, 'funcnames', str(pkg.eapi))): with open(pjoin(const.EBD_PATH, 'funcnames', str(pkg.eapi)), 'r') as f: eapi_funcs = f.readlines() else: ret, eapi_funcs = spawn_get_output( [pjoin(const.EBD_PATH, 'generate_eapi_func_list.bash'), str(pkg.eapi)]) if ret != 0: raise Exception("failed to generate list of EAPI %s specific functions" % str(pkg.eapi)) self.env["PKGCORE_EAPI_FUNCS"] = ' '.join(x.strip() for x in eapi_funcs) self.env_data_source = env_data_source if (env_data_source is not None and not isinstance(env_data_source, data_source.base)): raise TypeError( "env_data_source must be None, or a pkgcore.data_source.base " "derivative: %s: %s" % ( env_data_source.__class__, env_data_source)) self.features = set(x.lower() for x in features) self.env["FEATURES"] = ' '.join(sorted(self.features)) iuse_effective_regex = (re.escape(x) for x in pkg.iuse_effective) iuse_effective_regex = "^(%s)$" % "|".join(iuse_effective_regex) iuse_effective_regex = iuse_effective_regex.replace("\\.\\*", ".*") self.env["PKGCORE_IUSE_EFFECTIVE"] = iuse_effective_regex expected_ebuild_env(pkg, self.env, env_source_override=self.env_data_source) self.env["PKGCORE_FINALIZED_RESTRICT"] = ' '.join(str(x) for x in pkg.restrict) self.restrict = pkg.restrict for x in ("sandbox", "userpriv"): setattr(self, x, self.feat_or_bool(x) and not (x in self.restrict)) if self.userpriv and os.getuid() != 0: self.userpriv = False if "PORT_LOGDIR" in self.env: self.logging = pjoin( self.env["PORT_LOGDIR"], "%s:%s:%s.log" % ( pkg.cpvstr, self.__class__.__name__, time.strftime("%Y%m%d-%H%M%S", time.localtime()))) del self.env["PORT_LOGDIR"] else: self.logging = False self.env["XARGS"] = xargs self.bashrc = self.env.pop("bashrc", ()) self.pkg = pkg self.eapi = pkg.eapi wipes = [k for k, v in self.env.iteritems() if not isinstance(v, basestring)] for k in wipes: del self.env[k] self.set_op_vars(tmp_offset) self.clean_at_start = clean self.clean_needed = False
def __init__(self, pkg, initial_env=None, env_data_source=None, observer=None, clean=True, tmp_offset=None, allow_fetching=False): """ :param pkg: :class:`pkgcore.ebuild.ebuild_src.package` instance this env is being setup for :param initial_env: initial environment to use for this ebuild :param env_data_source: a :obj:`snakeoil.data_source.base` instance to restore the environment from- used for restoring the state of an ebuild processing, whether for unmerging, or walking phases during building """ self.allow_fetching = allow_fetching self.pkg = pkg self.eapi = pkg.eapi if not hasattr(self, "observer"): self.observer = observer if not self.eapi.is_supported: raise TypeError(f"package {pkg} uses unsupported EAPI: {str(self.eapi)!r}") if initial_env is not None: # copy. self.env = dict(initial_env) for x in ("USE", "ACCEPT_LICENSE"): self.env.pop(x, None) else: self.env = {} # Drop all USE_EXPAND variables from the exported environment. for u in self.domain.profile.use_expand: self.env.pop(u, None) # Only export USE_EXPAND variables for the package's enabled USE flags. d = defaultdict(list) for u in pkg.use: m = self.domain.use_expand_re.match(u) if m: use_expand, value = m.groups() d[use_expand.upper()].append(value) for k, v in d.items(): self.env[k] = ' '.join(sorted(v)) self.bashrc = self.env.pop("bashrc", ()) # TODO: drop this once we rewrite/remove calling python-based scripts # from the bash side if "PYTHONPATH" in os.environ: self.env["PYTHONPATH"] = os.environ["PYTHONPATH"] self.features = set(x.lower() for x in self.domain.features) self.env["FEATURES"] = ' '.join(sorted(self.features)) # XXX: note this is just EAPI 3 and EAPI 7 compatibility; not full prefix, soon.. trailing_slash = self.eapi.options.trailing_slash self.env['ROOT'] = self.domain.root.rstrip(trailing_slash) + trailing_slash self.prefix_mode = self.eapi.options.prefix_capable or 'force-prefix' in self.features self.env['PKGCORE_PREFIX_SUPPORT'] = 'false' self.prefix = '/' if self.prefix_mode: self.prefix = self.domain.prefix self.env['EPREFIX'] = self.prefix.rstrip(os.sep) self.env['EROOT'] = pjoin(self.env['ROOT'], self.env['EPREFIX']) + trailing_slash self.env['PKGCORE_PREFIX_SUPPORT'] = 'true' if self.eapi.options.has_sysroot: self.env['SYSROOT'] = self.env['ROOT'] self.env['ESYSROOT'] = pjoin(self.env['SYSROOT'], self.env['EPREFIX']) self.env['BROOT'] = self.env['EPREFIX'] # internally implemented EAPI specific functions to skip when exporting env self.env["PKGCORE_EAPI_FUNCS"] = ' '.join(self.eapi.bash_funcs) self.env_data_source = env_data_source if (env_data_source is not None and not isinstance(env_data_source, data_source.base)): raise TypeError( "env_data_source must be None, or a pkgcore.data_source.base " f"derivative: {env_data_source.__class__}: {env_data_source}") iuse_effective_regex = f"^({'|'.join(re.escape(x) for x in pkg.iuse_effective)})$" self.env["PKGCORE_IUSE_EFFECTIVE"] = iuse_effective_regex.replace("\\.\\*", ".*") expected_ebuild_env(pkg, self.env, env_source_override=self.env_data_source) self.env["PKGCORE_FINALIZED_RESTRICT"] = ' '.join(str(x) for x in pkg.restrict) self.restrict = pkg.restrict for x in ("sandbox", "userpriv"): setattr(self, x, self.feat_or_bool(x) and not (x in self.restrict)) if self.userpriv and os.getuid() != 0: self.userpriv = False if "PORT_LOGDIR" in self.env: self.logging = pjoin( self.env["PORT_LOGDIR"], "%s:%s:%s.log" % ( pkg.cpvstr, self.__class__.__name__, time.strftime("%Y%m%d-%H%M%S", time.localtime()))) del self.env["PORT_LOGDIR"] else: self.logging = False self.env["PKGCORE_PKG_REPO"] = pkg.source_repository self.env["XARGS"] = xargs # wipe variables listed in ENV_UNSET for supporting EAPIs if self.eapi.options.has_env_unset: for x in self.env.pop('ENV_UNSET', ()): self.env.pop(x, None) # wipe any remaining internal settings from the exported env wipes = [k for k, v in self.env.items() if not isinstance(v, str)] for k in wipes: del self.env[k] self._set_op_vars(tmp_offset) self.clean_at_start = clean self.clean_needed = False # various IPC command support self._ipc_helpers = { # bash helpers 'doins': ebd_ipc.Doins(self), 'dodoc': ebd_ipc.Dodoc(self), 'dohtml': ebd_ipc.Dohtml(self), 'doinfo': ebd_ipc.Doinfo(self), 'dodir': ebd_ipc.Dodir(self), 'doexe': ebd_ipc.Doexe(self), 'dobin': ebd_ipc.Dobin(self), 'dosbin': ebd_ipc.Dosbin(self), 'dolib': ebd_ipc.Dolib(self), 'dolib.so': ebd_ipc.Dolib_so(self), 'dolib.a': ebd_ipc.Dolib_a(self), 'doman': ebd_ipc.Doman(self), 'domo': ebd_ipc.Domo(self), 'dosym': ebd_ipc.Dosym(self), 'dohard': ebd_ipc.Dohard(self), 'keepdir': ebd_ipc.Keepdir(self), # bash functions 'has_version': ebd_ipc.Has_Version(self), 'best_version': ebd_ipc.Best_Version(self), 'unpack': ebd_ipc.Unpack(self), 'eapply': ebd_ipc.Eapply(self), 'eapply_user': ebd_ipc.Eapply_User(self), 'docompress': ebd_ipc.Docompress(self), 'dostrip': ebd_ipc.Dostrip(self), }
def __init__(self, pkg, initial_env=None, env_data_source=None, observer=None, clean=True, tmp_offset=None, allow_fetching=False): """ :param pkg: :class:`pkgcore.ebuild.ebuild_src.package` instance this env is being setup for :param initial_env: initial environment to use for this ebuild :param env_data_source: a :obj:`snakeoil.data_source.base` instance to restore the environment from- used for restoring the state of an ebuild processing, whether for unmerging, or walking phases during building """ self.allow_fetching = allow_fetching self.pkg = pkg self.eapi = pkg.eapi if not hasattr(self, "observer"): self.observer = observer if not self.eapi.is_supported: raise TypeError(f"package {pkg} uses unsupported EAPI: {str(self.eapi)!r}") if initial_env is not None: # copy. self.env = dict(initial_env) for x in ("USE", "ACCEPT_LICENSE"): self.env.pop(x, None) else: self.env = {} # Drop all USE_EXPAND variables from the exported environment. for u in self.domain.profile.use_expand: self.env.pop(u, None) # Only export USE_EXPAND variables for the package's enabled USE flags. d = defaultdict(list) for u in pkg.use: m = self.domain.use_expand_re.match(u) if m: use_expand, value = m.groups() d[use_expand.upper()].append(value) for k, v in d.items(): self.env[k] = ' '.join(sorted(v)) self.bashrc = self.env.pop("bashrc", ()) # TODO: drop this once we rewrite/remove calling python-based scripts # from the bash side if "PYTHONPATH" in os.environ: self.env["PYTHONPATH"] = os.environ["PYTHONPATH"] self.features = set(x.lower() for x in self.domain.features) self.env["FEATURES"] = ' '.join(sorted(self.features)) self.set_path_vars(self.env, self.pkg, self.domain) # internally implemented EAPI specific functions to skip when exporting env self.env["PKGCORE_EAPI_FUNCS"] = ' '.join(self.eapi.bash_funcs) self.env_data_source = env_data_source if (env_data_source is not None and not isinstance(env_data_source, data_source.base)): raise TypeError( "env_data_source must be None, or a pkgcore.data_source.base " f"derivative: {env_data_source.__class__}: {env_data_source}") iuse_effective_regex = f"^({'|'.join(re.escape(x) for x in pkg.iuse_effective)})$" self.env["PKGCORE_IUSE_EFFECTIVE"] = iuse_effective_regex.replace("\\.\\*", ".*") expected_ebuild_env(pkg, self.env, env_source_override=self.env_data_source) self.env["PKGCORE_FINALIZED_RESTRICT"] = ' '.join(str(x) for x in pkg.restrict) self.restrict = pkg.restrict for x in ("sandbox", "userpriv"): setattr(self, x, self.feat_or_bool(x) and not (x in self.restrict)) if self.userpriv and os.getuid() != 0: self.userpriv = False if "PORT_LOGDIR" in self.env: self.logging = pjoin( self.env["PORT_LOGDIR"], "%s:%s:%s.log" % ( pkg.cpvstr, self.__class__.__name__, time.strftime("%Y%m%d-%H%M%S", time.localtime()))) del self.env["PORT_LOGDIR"] else: self.logging = False self.env["PKGCORE_PKG_REPO"] = pkg.source_repository self.env["XARGS"] = xargs # wipe variables listed in ENV_UNSET for supporting EAPIs if self.eapi.options.has_env_unset: for x in self.env.pop('ENV_UNSET', ()): self.env.pop(x, None) # wipe any remaining internal settings from the exported env wipes = [k for k, v in self.env.items() if not isinstance(v, str)] for k in wipes: del self.env[k] self._set_op_vars(tmp_offset) self.clean_at_start = clean self.clean_needed = False # various IPC command support self._ipc_helpers = { # bash helpers 'doins': ebd_ipc.Doins(self), 'dodoc': ebd_ipc.Dodoc(self), 'dohtml': ebd_ipc.Dohtml(self), 'doinfo': ebd_ipc.Doinfo(self), 'dodir': ebd_ipc.Dodir(self), 'doexe': ebd_ipc.Doexe(self), 'dobin': ebd_ipc.Dobin(self), 'dosbin': ebd_ipc.Dosbin(self), 'dolib': ebd_ipc.Dolib(self), 'dolib.so': ebd_ipc.Dolib_so(self), 'dolib.a': ebd_ipc.Dolib_a(self), 'doman': ebd_ipc.Doman(self), 'domo': ebd_ipc.Domo(self), 'dosym': ebd_ipc.Dosym(self), 'dohard': ebd_ipc.Dohard(self), 'keepdir': ebd_ipc.Keepdir(self), # bash functions 'has_version': ebd_ipc.Has_Version(self), 'best_version': ebd_ipc.Best_Version(self), 'unpack': ebd_ipc.Unpack(self), 'eapply': ebd_ipc.Eapply(self), 'eapply_user': ebd_ipc.Eapply_User(self), 'docompress': ebd_ipc.Docompress(self), 'dostrip': ebd_ipc.Dostrip(self), # internals 'filter_env': ebd_ipc.FilterEnv(self), }
def run_generic_phase(pkg, phase, env, userpriv, sandbox, fd_pipes=None, extra_handlers=None, failure_allowed=False, logging=None, **kwargs): """ :param phase: phase to execute :param env: environment mapping for the phase :param userpriv: will we drop to :obj:`pkgcore.os_data.portage_uid` and :obj:`pkgcore.os_data.portage_gid` access for this phase? :param sandbox: should this phase be sandboxed? :param fd_pipes: use custom file descriptors for ebd instance :type fd_pipes: mapping between file descriptors :param extra_handlers: extra command handlers :type extra_handlers: mapping from string to callable :param failure_allowed: allow failure without raising error :type failure_allowed: boolean :param logging: None or a filepath to log output to :return: True when the phase has finished execution """ userpriv = userpriv and is_userpriv_capable() sandbox = sandbox and is_sandbox_capable() tmpdir = kwargs.get('tmpdir', env.get('T', None)) if env is None: env = expected_ebuild_env(pkg) ebd = request_ebuild_processor(userpriv=userpriv, sandbox=sandbox, fd_pipes=fd_pipes) # this is a bit of a hack; used until ebd accepts observers that handle # the output redirection on its own. Primary relevance is when # stdout/stderr are pointed at a file; we leave buffering on, just # force the flush for synchronization. sys.stdout.flush() sys.stderr.flush() try: if not ebd.run_phase(phase, env, tmpdir=tmpdir, sandbox=sandbox, logging=logging, additional_commands=extra_handlers): if not failure_allowed: raise format.GenericBuildError( phase + ": Failed building (False/0 return from handler)") logger.warning(f"executing phase {phase}: execution failed, ignoring") except Exception as e: if isinstance(e, ebd_ipc.IpcError): # notify bash side of IPC error ebd.write(e.ret) if isinstance(e, ebd_ipc.IpcInternalError): # show main exception cause for internal IPC errors ebd.shutdown_processor(force=True) raise e.__cause__ try: ebd.shutdown_processor() except ProcessorError as pe: # catch die errors during shutdown e = pe release_ebuild_processor(ebd) if isinstance(e, ProcessorError): # force verbose die output e._verbosity = 1 raise e elif isinstance(e, IGNORED_EXCEPTIONS + (format.GenericBuildError,)): raise raise format.GenericBuildError( f"Executing phase {phase}: Caught exception: {e}") from e release_ebuild_processor(ebd) return True
def __init__(self, pkg, initial_env=None, env_data_source=None, features=None, observer=None, clean=True, tmp_offset=None, use_override=None, allow_fetching=False): """ :param pkg: :class:`pkgcore.ebuild.ebuild_src.package` instance this env is being setup for :param initial_env: initial environment to use for this ebuild :param env_data_source: a :obj:`snakeoil.data_source.base` instance to restore the environment from- used for restoring the state of an ebuild processing, whether for unmerging, or walking phases during building :param features: ebuild features, hold over from portage, will be broken down at some point """ if use_override is not None: use = use_override else: use = pkg.use self.allow_fetching = allow_fetching if not hasattr(self, "observer"): self.observer = observer if not pkg.eapi_obj.is_supported: raise TypeError( "package %s uses an unsupported eapi: %s" % (pkg, pkg.eapi)) if initial_env is not None: # copy. self.env = dict(initial_env) for x in ("USE", "ACCEPT_LICENSE"): if x in self.env: del self.env[x] else: self.env = {} # temp hack. for x in ('chost', 'cbuild', 'ctarget'): val = getattr(pkg, x) if val is not None: self.env[x.upper()] = val # special note... if CTARGET is the same as CHOST, suppress it. # certain ebuilds (nano for example) will misbehave w/ it. if pkg.ctarget is not None and pkg.ctarget == pkg.chost: self.env.pop("CTARGET") if "PYTHONPATH" in os.environ: self.env["PYTHONPATH"] = os.environ["PYTHONPATH"] if "PKGCORE_DEBUG" in os.environ: self.env["PKGCORE_DEBUG"] = str(int(os.environ["PKGCORE_DEBUG"])) if features is None: features = self.env.get("FEATURES", ()) # XXX: note this is just eapi3 compatibility; not full prefix, soon.. self.env["ROOT"] = self.domain.root self.prefix_mode = pkg.eapi_obj.options.prefix_capable or 'force-prefix' in features self.env["PKGCORE_PREFIX_SUPPORT"] = 'false' self.prefix = '/' if self.prefix_mode: self.env['EROOT'] = normpath(self.domain.root) self.prefix = self.domain.prefix.lstrip("/") eprefix = normpath(pjoin(self.env["EROOT"], self.prefix)) if eprefix == '/': # Set eprefix to '' if it's basically empty; this keeps certain crappy builds # (cmake for example) from puking over //usr/blah pathways eprefix = '' self.env["EPREFIX"] = eprefix self.env["PKGCORE_PREFIX_SUPPORT"] = 'true' self.env.update(pkg.eapi_obj.get_ebd_env()) self.env_data_source = env_data_source if env_data_source is not None and \ not isinstance(env_data_source, data_source.base): raise TypeError( "env_data_source must be None, or a pkgcore.data_source.base " "derivative: %s: %s" % ( env_data_source.__class__, env_data_source)) self.features = set(x.lower() for x in features) self.env["FEATURES"] = ' '.join(sorted(self.features)) iuse_effective_regex = (re.escape(x) for x in pkg.iuse_effective) iuse_effective_regex = "^(%s)$" % "|".join(iuse_effective_regex) iuse_effective_regex = iuse_effective_regex.replace("\\.\\*", ".*") self.env["PKGCORE_IUSE_EFFECTIVE"] = iuse_effective_regex expected_ebuild_env(pkg, self.env, env_source_override=self.env_data_source) self.env["PKGCORE_FINALIZED_RESTRICT"] = ' '.join(str(x) for x in pkg.restrict) self.restrict = pkg.restrict for x in ("sandbox", "userpriv", "fakeroot"): setattr(self, x, self.feat_or_bool(x) and not (x in self.restrict)) if self.fakeroot: logger.warning("disabling fakeroot; unusable till coreutils/fakeroot" + " interaction is fixed") self.fakeroot = False if self.userpriv and os.getuid() != 0: self.userpriv = False if "PORT_LOGDIR" in self.env: self.logging = pjoin(self.env["PORT_LOGDIR"], "%s:%s:%s.log" % (pkg.cpvstr, self.__class__.__name__, time.strftime("%Y%m%d-%H%M%S", time.localtime()))) del self.env["PORT_LOGDIR"] else: self.logging = False self.env["XARGS"] = xargs self.bashrc = self.env.pop("bashrc", ()) self.pkg = pkg self.eapi = pkg.eapi self.eapi_obj = pkg.eapi_obj wipes = [k for k, v in self.env.iteritems() if not isinstance(v, basestring)] for k in wipes: del self.env[k] self.set_op_vars(tmp_offset) self.clean_at_start = clean self.clean_needed = False
def run_generic_phase(pkg, phase, env, userpriv, sandbox, fd_pipes=None, extra_handlers=None, failure_allowed=False, logging=None, **kwargs): """ :param phase: phase to execute :param env: environment mapping for the phase :param userpriv: will we drop to :obj:`pkgcore.os_data.portage_uid` and :obj:`pkgcore.os_data.portage_gid` access for this phase? :param sandbox: should this phase be sandboxed? :param fd_pipes: use custom file descriptors for ebd instance :type fd_pipes: mapping between file descriptors :param extra_handlers: extra command handlers :type extra_handlers: mapping from string to callable :param failure_allowed: allow failure without raising error :type failure_allowed: boolean :param logging: None or a filepath to log output to :return: True when the phase has finished execution """ userpriv = userpriv and is_userpriv_capable() sandbox = sandbox and is_sandbox_capable() tmpdir = kwargs.get('tmpdir', env.get('T', None)) if env is None: env = expected_ebuild_env(pkg) ebd = request_ebuild_processor(userpriv=userpriv, sandbox=sandbox, fd_pipes=fd_pipes) # this is a bit of a hack; used until ebd accepts observers that handle # the output redirection on its own. Primary relevance is when # stdout/stderr are pointed at a file; we leave buffering on, just # force the flush for synchronization. sys.stdout.flush() sys.stderr.flush() try: if not ebd.run_phase(phase, env, tmpdir=tmpdir, sandbox=sandbox, logging=logging, additional_commands=extra_handlers): if not failure_allowed: raise format.GenericBuildError( phase + ": Failed building (False/0 return from handler)") logger.warning(f"executing phase {phase}: execution failed, ignoring") except Exception as e: if isinstance(e, ebd_ipc.IpcError): # notify bash side of IPC error ebd.write(e.ret) if isinstance(e, ebd_ipc.IpcInternalError): # show main exception cause for internal IPC errors ebd.shutdown_processor(force=True) raise e.__cause__ try: ebd.shutdown_processor() except ProcessorError as pe: # catch die errors during shutdown e = pe release_ebuild_processor(ebd) if isinstance(e, ProcessorError): # force verbose die output e._verbosity = 1 raise e elif isinstance(e, IGNORED_EXCEPTIONS + (format.GenericBuildError,)): raise raise format.GenericBuildError( f"Executing phase {phase}: Caught exception: {e}") from e release_ebuild_processor(ebd) return True
def _cmd_implementation_sanity_check(self, domain, observer): """Various ebuild sanity checks (REQUIRED_USE, pkg_pretend).""" pkg = self.pkg eapi = pkg.eapi # perform REQUIRED_USE checks if eapi.options.has_required_use: use = pkg.use for node in pkg.required_use: if not node.match(use): observer.info( textwrap.dedent( """ REQUIRED_USE requirement wasn't met Failed to match: {} from: {} for USE: {} pkg: {} """.format( node, pkg.required_use, " ".join(use), pkg.cpvstr ) ) ) return False # return if running pkg_pretend is not required if "pretend" not in pkg.mandatory_phases: return True # run pkg_pretend phase commands = None if not pkg.built: commands = {"request_inherit": partial(inherit_handler, self._eclass_cache)} env = expected_ebuild_env(pkg) tmpdir = normpath(domain._get_tempspace()) builddir = pjoin(tmpdir, env["CATEGORY"], env["PF"]) pkg_tmpdir = normpath(pjoin(builddir, "temp")) ensure_dirs(pkg_tmpdir, mode=0770, gid=portage_gid, minimal=True) env["ROOT"] = domain.root env["T"] = pkg_tmpdir # TODO: make colored output easier to achieve from observers msg = [ ">>> Running pkg_pretend for ", observer._output._out.fg("green"), pkg.cpvstr, observer._output._out.reset, ] observer._output._out.write(*msg) try: start = time.time() ret = run_generic_phase(pkg, "pretend", env, userpriv=True, sandbox=True, extra_handlers=commands) logger.debug("pkg_pretend sanity check for %s took %2.2f seconds", pkg.cpvstr, time.time() - start) return ret except format.GenericBuildError as e: return False finally: shutil.rmtree(builddir) # try to wipe the cat dir; if not empty, ignore it try: os.rmdir(os.path.dirname(builddir)) except EnvironmentError as e: # POSIX specifies either ENOTEMPTY or EEXIST for non-empty dir # in particular, Solaris uses EEXIST in that case. # https://github.com/pkgcore/pkgcore/pull/181 if e.errno not in (errno.ENOTEMPTY, errno.EEXIST): raise
def __init__( self, pkg, initial_env=None, env_data_source=None, features=None, observer=None, clean=True, tmp_offset=None, use_override=None, allow_fetching=False, ): """ :param pkg: :class:`pkgcore.ebuild.ebuild_src.package` instance this env is being setup for :param initial_env: initial environment to use for this ebuild :param env_data_source: a :obj:`snakeoil.data_source.base` instance to restore the environment from- used for restoring the state of an ebuild processing, whether for unmerging, or walking phases during building :param features: ebuild features, hold over from portage, will be broken down at some point """ if use_override is not None: use = use_override else: use = pkg.use self.allow_fetching = allow_fetching if not hasattr(self, "observer"): self.observer = observer if not pkg.eapi.is_supported: raise TypeError("package %s uses an unsupported eapi: %s" % (pkg, pkg.eapi)) if initial_env is not None: # copy. self.env = dict(initial_env) for x in ("USE", "ACCEPT_LICENSE"): if x in self.env: del self.env[x] else: self.env = {} if "PYTHONPATH" in os.environ: self.env["PYTHONPATH"] = os.environ["PYTHONPATH"] if features is None: features = self.env.get("FEATURES", ()) # XXX: note this is just EAPI 3 compatibility; not full prefix, soon.. self.env["ROOT"] = self.domain.root self.prefix_mode = pkg.eapi.options.prefix_capable or "force-prefix" in features self.env["PKGCORE_PREFIX_SUPPORT"] = "false" self.prefix = "/" if self.prefix_mode: self.prefix = self.domain.prefix self.env["EPREFIX"] = self.prefix.rstrip("/") self.env["EROOT"] = abspath(pjoin(self.domain.root, self.prefix.lstrip("/"))).rstrip("/") + "/" self.env["PKGCORE_PREFIX_SUPPORT"] = "true" # set the list of internally implemented EAPI specific functions that # shouldn't be exported if os.path.exists(pjoin(const.EBD_PATH, "funcnames", str(pkg.eapi))): with open(pjoin(const.EBD_PATH, "funcnames", str(pkg.eapi)), "r") as f: eapi_funcs = f.readlines() else: ret, eapi_funcs = spawn_get_output([pjoin(const.EBD_PATH, "generate_eapi_func_list.bash"), str(pkg.eapi)]) if ret != 0: raise Exception("failed to generate list of EAPI %s specific functions" % str(pkg.eapi)) self.env["PKGCORE_EAPI_FUNCS"] = " ".join(x.strip() for x in eapi_funcs) self.env_data_source = env_data_source if env_data_source is not None and not isinstance(env_data_source, data_source.base): raise TypeError( "env_data_source must be None, or a pkgcore.data_source.base " "derivative: %s: %s" % (env_data_source.__class__, env_data_source) ) self.features = set(x.lower() for x in features) self.env["FEATURES"] = " ".join(sorted(self.features)) iuse_effective_regex = (re.escape(x) for x in pkg.iuse_effective) iuse_effective_regex = "^(%s)$" % "|".join(iuse_effective_regex) iuse_effective_regex = iuse_effective_regex.replace("\\.\\*", ".*") self.env["PKGCORE_IUSE_EFFECTIVE"] = iuse_effective_regex expected_ebuild_env(pkg, self.env, env_source_override=self.env_data_source) self.env["PKGCORE_FINALIZED_RESTRICT"] = " ".join(str(x) for x in pkg.restrict) self.restrict = pkg.restrict for x in ("sandbox", "userpriv"): setattr(self, x, self.feat_or_bool(x) and not (x in self.restrict)) if self.userpriv and os.getuid() != 0: self.userpriv = False if "PORT_LOGDIR" in self.env: self.logging = pjoin( self.env["PORT_LOGDIR"], "%s:%s:%s.log" % (pkg.cpvstr, self.__class__.__name__, time.strftime("%Y%m%d-%H%M%S", time.localtime())), ) del self.env["PORT_LOGDIR"] else: self.logging = False self.env["XARGS"] = xargs self.bashrc = self.env.pop("bashrc", ()) self.pkg = pkg self.eapi = pkg.eapi wipes = [k for k, v in self.env.iteritems() if not isinstance(v, basestring)] for k in wipes: del self.env[k] self.set_op_vars(tmp_offset) self.clean_at_start = clean self.clean_needed = False