def arches_desc(self):
"""Arch stability status (GLEP 72).
See https://www.gentoo.org/glep/glep-0072.html for more details.
"""
fp = pjoin(self.profiles_base, 'arches.desc')
d = {'stable': set(), 'transitional': set(), 'testing': set()}
try:
for lineno, line in iter_read_bash(fp, enum_line=True):
try:
arch, status = line.split()
except ValueError:
logger.error(f"{self.repo_id}::profiles/arches.desc, "
f"line {lineno}: invalid line format: "
"should be '<arch> <status>'")
continue
if arch not in self.known_arches:
logger.warning(f"{self.repo_id}::profiles/arches.desc, "
f"line {lineno}: unknown arch: {arch!r}")
continue
if status not in d:
logger.warning(
f"{self.repo_id}::profiles/arches.desc, "
f"line {lineno}: unknown status: {status!r}")
continue
d[status].add(arch)
except FileNotFoundError:
pass
return mappings.ImmutableDict(d)
def parse(profiles_base, repo_id, known_status=None, known_arch=None):
"""Return the mapping of arches to profiles for a repo."""
l = []
fp = pjoin(profiles_base, 'profiles.desc')
try:
for lineno, line in iter_read_bash(fp, enum_line=True):
try:
arch, profile, status = line.split()
except ValueError:
logger.error(
f"{repo_id}::profiles/profiles.desc, "
f"line {lineno}: invalid profile line format: "
"should be 'arch profile status'")
continue
if known_status is not None and status not in known_status:
logger.warning(
f"{repo_id}::profiles/profiles.desc, "
f"line {lineno}: unknown profile status: {status!r}")
if known_arch is not None and arch not in known_arch:
logger.warning(f"{repo_id}::profiles/profiles.desc, "
f"line {lineno}: unknown arch: {arch!r}")
# Normalize the profile name on the offchance someone slipped an extra /
# into it.
path = '/'.join(filter(None, profile.split('/')))
deprecated = os.path.exists(
os.path.join(profiles_base, path, 'deprecated'))
l.append(
_KnownProfile(profiles_base, arch, path, status,
deprecated))
except FileNotFoundError:
logger.debug(
f"No profile descriptions found at {repo_id}::profiles/profiles.desc"
)
return frozenset(l)
def iter_vulnerabilities(self):
"""generator yielding each GLSA restriction"""
for path in self.paths:
for fn in listdir_files(path):
# glsa-1234-12.xml
if not (fn.startswith("glsa-") and fn.endswith(".xml")):
continue
# This verifies the filename is of the correct syntax.
try:
[int(x) for x in fn[5:-4].split("-")]
except ValueError:
continue
root = etree.parse(pjoin(path, fn))
glsa_node = root.getroot()
if glsa_node.tag != 'glsa':
raise ValueError("glsa without glsa rootnode")
for affected in root.findall('affected'):
for pkg in affected.findall('package'):
try:
pkgname = str(pkg.get('name')).strip()
pkg_vuln_restrict = \
self.generate_intersects_from_pkg_node(
pkg, tag="glsa(%s)" % fn[5:-4])
if pkg_vuln_restrict is None:
continue
pkgatom = atom.atom(pkgname)
yield fn[5:-4], pkgname, pkgatom, pkg_vuln_restrict
except (TypeError, ValueError) as e:
# thrown from cpv.
logger.warning(
f"invalid glsa- {fn}, package {pkgname}: {e}")
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
logger.warning(f"invalid glsa- {fn}: error: {e}")
def package_keywords_splitter(iterable):
for line, lineno, path in iterable:
v = line.split()
try:
yield parse_match(v[0]), tuple(v[1:]), line, lineno, path
except ParseError as e:
logger.warning(f'{path!r}, line {lineno}: parsing error: {e}')
def is_supported(self):
"""Check if an EAPI is supported."""
if EAPI.known_eapis.get(self._magic) is not None:
if not self.options.is_supported:
logger.warning(f"EAPI '{self}' isn't fully supported")
sys.stderr.flush()
return True
return False
def installed_repos(self):
"""Group of configured, installed package repos."""
repos = []
for repo in self.installed_repos_raw:
try:
repos.append(self._wrap_repo(repo, filtered=False))
except repo_errors.RepoError as e:
logger.warning(f'skipping {repo.repo_id!r} repo: {e}')
return RepositoryGroup(repos)
def setup_workdir(self):
# ensure dirs.
for k in ("HOME", "T", "WORKDIR", "D"):
if not ensure_dirs(self.env[k], mode=0o4770, gid=portage_gid, minimal=True):
raise format.FailedDirectory(
self.env[k],
"%s doesn't fulfill minimum mode %o and gid %i" % (k, 0o770, portage_gid))
# XXX hack, just 'til pkgcore controls these directories
if (os.stat(self.env[k]).st_mode & 0o2000):
logger.warning(f"{self.env[k]} ( {k} ) is setgid")
def _update_metadata(self, pkg, ebp=None):
parsed_eapi = pkg.eapi
if not parsed_eapi.is_supported:
return {'EAPI': str(parsed_eapi)}
with processor.reuse_or_request(ebp) as my_proc:
try:
mydata = my_proc.get_keys(pkg, self._ecache)
except processor.ProcessorError as e:
raise metadata_errors.MetadataException(
pkg, 'data', 'failed sourcing ebuild', e)
inherited = mydata.pop("INHERITED", None)
# Rewrite defined_phases as needed, since we now know the EAPI.
eapi = get_eapi(mydata.get('EAPI', '0'))
if parsed_eapi != eapi:
raise metadata_errors.MetadataException(
pkg, 'eapi',
f"parsed EAPI '{parsed_eapi}' doesn't match sourced EAPI '{eapi}'"
)
wipes = set(mydata)
wipes.difference_update(eapi.metadata_keys)
if mydata["DEFINED_PHASES"] != '-':
phases = mydata["DEFINED_PHASES"].split()
d = eapi.phases_rev
phases = set(d.get(x) for x in phases)
# Discard is required should we have gotten
# a phase that isn't actually in this EAPI.
phases.discard(None)
mydata["DEFINED_PHASES"] = ' '.join(sorted(phases))
if inherited:
mydata["_eclasses_"] = self._ecache.get_eclass_data(
inherited.split())
mydata['_chf_'] = chksum.LazilyHashedPath(pkg.path)
for x in wipes:
del mydata[x]
if self._cache is not None:
for cache in self._cache:
if not cache.readonly:
try:
cache[pkg.cpvstr] = mydata
except cache_errors.CacheError as e:
logger.warning("caught cache error: %s", e)
del e
continue
break
return mydata
def tmpdir(self):
"""Temporary directory for the system.
Uses PORTAGE_TMPDIR setting and falls back to using the system's TMPDIR if unset.
"""
path = self.settings.get('PORTAGE_TMPDIR', '')
if not os.path.exists(path):
try:
os.mkdir(path)
except EnvironmentError:
path = tempfile.gettempdir()
logger.warning(f'nonexistent PORTAGE_TMPDIR path, defaulting to {path!r}')
return os.path.normpath(path)
def format(self, op):
out = self.out
origautoline = out.autoline
out.autoline = False
out.write('* ')
out.write(out.fg('blue'), op.pkg.key)
out.write(f"-{op.pkg.fullver}")
out.write(f"::{op.pkg.repo.repo_id} ")
out.write(out.fg('blue'), f"{{:{op.pkg.slot}}} ")
op_type = op.desc
if op.desc == 'add':
suffix = 'N'
if op.pkg.slot != '0':
op_type = 'slotted_add'
suffix = 'S'
out.write(out.fg('yellow'), f"[{suffix}]")
elif op.desc == 'replace':
if op.pkg != op.old_pkg:
if op.pkg > op.old_pkg:
op_type = 'upgrade'
else:
op_type = 'downgrade'
out.write(out.fg('yellow'),
f"[{op_type[0].upper()} {op.old_pkg.fullver}]")
else:
out.write(out.fg('yellow'), "[R]")
else:
# shouldn't reach here
logger.warning("unknown op type encountered: desc(%r), %r",
op.desc, op)
self.visit_op(op_type)
red = out.fg('red')
green = out.fg('green')
flags = []
use = set(op.pkg.use)
for flag in sorted(op.pkg.iuse_stripped):
if flag in use:
flags.extend((green, flag, ' '))
else:
flags.extend((red, '-', flag, ' '))
if flags:
out.write(' ')
# Throw away the final space.
out.write(*flags[:-1])
out.write('\n')
out.autoline = origautoline
def repo_id(self):
"""Main identifier for the repo.
The precedence order is as follows: repos.conf name, repo-name from
metadata/layout.conf, profiles/repo_name, and finally a fallback to the
repo's location for unlabeled repos.
"""
if self.config_name:
return self.config_name
if self.repo_name:
return self.repo_name
if self.pms_repo_name:
return self.pms_repo_name
if not self.is_empty:
logger.warning(f"repo lacks a defined name: {self.location!r}")
return self.location
def _read_cache_file(package, cache_path):
"""Read an existing cache file."""
stored_cache = {}
cache_data = list(readlines_ascii(cache_path, True, True, False))
if len(cache_data) >= 1:
if cache_data[0] != CACHE_HEADER:
logger.warning(
"plugin cache has a wrong header: %r, regenerating", cache_data[0])
cache_data = []
else:
cache_data = cache_data[1:]
if not cache_data:
return {}
try:
for line in cache_data:
module, mtime, entries = line.split(':', 2)
mtime = int(mtime)
# Needed because ''.split(':') == [''], not []
if not entries:
entries = set()
else:
entries = entries.replace(':', ',').split(',')
if not len(entries) % 3 == 0:
logger.error(
"failed reading cache %s; entries field isn't "
"divisable by 3: %r", cache_path, entries)
continue
entries = iter(entries)
def f(val):
if val.isdigit():
val = int(val)
return val
entries = set(
_plugin_data(
key, int(priority),
f'{package.__name__}.{module}', f(target))
for (key, priority, target) in zip(entries, entries, entries))
stored_cache[(module, mtime)] = entries
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
logger.warning("failed reading cache; exception %s, regenerating.", e)
stored_cache.clear()
return stored_cache
def source_repos_raw(self):
"""Group of package repos without filtering."""
repos = []
for r in self.__repos:
try:
repo = r.instantiate()
except config_errors.InstantiationError as e:
# roll back the exception chain to a meaningful error message
exc = find_user_exception(e)
if exc is None:
exc = e
logger.warning(f'skipping {r.name!r} repo: {exc}')
continue
if not repo.is_supported:
logger.warning(
f'skipping {r.name!r} repo: unsupported EAPI {str(repo.eapi)!r}')
continue
repos.append(repo)
return RepositoryGroup(repos)
def _get_metadata(self, pkg, ebp=None, force_regen=False):
caches = self._cache
if force_regen:
caches = ()
ebuild_hash = chksum.LazilyHashedPath(pkg.path)
for cache in caches:
if cache is not None:
try:
data = cache[pkg.cpvstr]
if cache.validate_entry(data, ebuild_hash, self._ecache):
return data
if not cache.readonly:
del cache[pkg.cpvstr]
except KeyError:
continue
except cache_errors.CacheError as e:
logger.warning("caught cache error: %s", e)
del e
continue
# no cache entries, regen
return self._update_metadata(pkg, ebp=ebp)
def _atoms(self):
try:
s = set()
for x in readlines_ascii(self.path, True):
if not x or x.startswith("#"):
continue
elif x.startswith("@"):
if self.error_on_subsets:
raise ValueError(
"set %s isn't a valid atom in pkgset %r" %
(x, self.path))
logger.warning(
"set item %r found in pkgset %r: it will be "
"wiped on update since portage/pkgcore store set items "
"in a separate way", x[1:], self.path)
continue
s.add(atom(x))
except InvalidDependency as e:
raise errors.ParsingError("parsing %r" % self.path,
exception=e) from e
return s
def _add_sets(self):
self["world"] = basics.AutoConfigSection({
"class":
"pkgcore.pkgsets.filelist.WorldFile",
"location":
pjoin(self.root, econst.WORLD_FILE.lstrip('/'))
})
self["system"] = basics.AutoConfigSection({
"class": "pkgcore.pkgsets.system.SystemSet",
"profile": "profile"
})
self["installed"] = basics.AutoConfigSection({
"class": "pkgcore.pkgsets.installed.Installed",
"vdb": "vdb"
})
self["versioned-installed"] = basics.AutoConfigSection({
"class": "pkgcore.pkgsets.installed.VersionedInstalled",
"vdb": "vdb"
})
set_fp = pjoin(self.dir, "sets")
try:
for setname in listdir_files(set_fp):
# Potential for name clashes here, those will just make
# the set not show up in config.
if setname in ("system", "world"):
logger.warning(
"user defined set %r is disallowed; ignoring",
pjoin(set_fp, setname))
continue
self[setname] = basics.AutoConfigSection({
"class":
"pkgcore.pkgsets.filelist.FileList",
"location":
pjoin(set_fp, setname)
})
except FileNotFoundError:
pass
def add_data(self):
if self.observer is None:
end = start = lambda x: None
else:
start = self.observer.phase_start
end = self.observer.phase_end
pkg = self.new_pkg
final_path = discern_loc(self.repo.base, pkg, self.repo.extension)
tmp_path = pjoin(
os.path.dirname(final_path),
".tmp.%i.%s" % (os.getpid(), os.path.basename(final_path)))
self.tmp_path, self.final_path = tmp_path, final_path
if not ensure_dirs(os.path.dirname(tmp_path), mode=0o755):
raise repo_interfaces.Failure(
f"failed creating directory: {os.path.dirname(tmp_path)!r}")
try:
start(f"generating tarball: {tmp_path}")
tar.write_set(pkg.contents,
tmp_path,
compressor='bzip2',
parallelize=True)
end("tarball created", True)
start("writing Xpak")
# ok... got a tarball. now add xpak.
xpak.Xpak.write_xpak(tmp_path, generate_attr_dict(pkg))
end("wrote Xpak", True)
# ok... we tagged the xpak on.
os.chmod(tmp_path, 0o644)
except Exception as e:
try:
unlink_if_exists(tmp_path)
except EnvironmentError as e:
logger.warning(f"failed removing {tmp_path!r}: {e}")
raise
return True
def package_env_splitter(basedir, iterable):
for line, lineno, path in iterable:
val = line.split()
if len(val) == 1:
logger.warning(f"{path!r}, line {lineno}: missing file reference: {line!r}")
continue
paths = []
for env_file in val[1:]:
fp = pjoin(basedir, env_file)
if os.path.exists(fp):
paths.append(fp)
else:
logger.warning(f"{path!r}, line {lineno}: nonexistent file: {fp!r}")
try:
yield parse_match(val[0]), tuple(paths), line, lineno, path
except ParseError as e:
logger.warning(f'{path!r}, line {lineno}: parsing error: {e}')
def __init__(self, domain, pkg, verified_files, eclass_cache,
observer=None, force_test=False, **kwargs):
"""
:param pkg: :obj:`pkgcore.ebuild.ebuild_src.package` instance we'll be
building
:param eclass_cache: the :class:`pkgcore.ebuild.eclass_cache`
we'll be using
:param verified_files: mapping of fetchables mapped to their disk location
"""
self._built_class = ebuild_built.fresh_built_package
format.build.__init__(self, domain, pkg, verified_files, observer)
domain_settings = self.domain.settings
ebd.__init__(self, pkg, initial_env=domain_settings, **kwargs)
self.env["FILESDIR"] = pjoin(os.path.dirname(pkg.ebuild.path), "files")
self.eclass_cache = eclass_cache
self.run_test = force_test or self.feat_or_bool("test", domain_settings)
self.allow_failed_test = self.feat_or_bool("test-fail-continue", domain_settings)
if "test" in self.restrict:
self.run_test = False
elif not force_test and "test" not in pkg.use:
if self.run_test:
logger.warning(f"disabling test for {pkg} due to test use flag being disabled")
self.run_test = False
# XXX minor hack
path = self.env["PATH"].split(os.pathsep)
for s, default in (("DISTCC", ".distcc"), ("CCACHE", "ccache")):
b = (self.feat_or_bool(s, domain_settings) and
s not in self.restrict)
setattr(self, s.lower(), b)
if b:
# looks weird I realize, but
# pjoin("/foor/bar", "/barr/foo") == "/barr/foo"
# and pjoin("/foo/bar", ".asdf") == "/foo/bar/.asdf"
self.env.setdefault(s + "_DIR", pjoin(self.domain.tmpdir, default))
# gentoo bug 355283
libdir = self.env.get("ABI")
if libdir is not None:
libdir = self.env.get(f"LIBDIR_{libdir}")
if libdir is not None:
libdir = self.env.get(libdir)
if libdir is None:
libdir = "lib"
path.insert(0, f"/usr/{libdir}/{s.lower()}/bin")
else:
for y in ("_PATH", "_DIR"):
if s + y in self.env:
del self.env[s+y]
self.env["PATH"] = os.pathsep.join(path)
# ordering must match appearance order in SRC_URI per PMS
self.env["A"] = ' '.join(iter_stable_unique(pkg.distfiles))
if self.eapi.options.has_AA:
pkg = self.pkg
while hasattr(pkg, '_raw_pkg'):
pkg = getattr(pkg, '_raw_pkg')
self.env["AA"] = ' '.join(set(iflatten_instance(pkg.distfiles)))
if self.eapi.options.has_KV:
self.env["KV"] = domain.KV
if self.eapi.options.has_merge_type:
self.env["MERGE_TYPE"] = "source"
if self.eapi.options.has_portdir:
self.env["PORTDIR"] = pkg.repo.location
self.env["ECLASSDIR"] = eclass_cache.eclassdir
if self.setup_is_for_src:
self._init_distfiles_env()
def __del__(self):
if getattr(self, 'underway', False):
logger.warning(
f"{self.old_pkg} -> {self.new_pkg} replacement was underway, but "
"wasn't completed")
self.lock.release_write_lock()
def run_generic_phase(pkg, phase, env, userpriv, sandbox, fd_pipes=None,
extra_handlers=None, failure_allowed=False, logging=None, **kwargs):
"""
:param phase: phase to execute
:param env: environment mapping for the phase
:param userpriv: will we drop to
:obj:`pkgcore.os_data.portage_uid` and
:obj:`pkgcore.os_data.portage_gid` access for this phase?
:param sandbox: should this phase be sandboxed?
:param fd_pipes: use custom file descriptors for ebd instance
:type fd_pipes: mapping between file descriptors
:param extra_handlers: extra command handlers
:type extra_handlers: mapping from string to callable
:param failure_allowed: allow failure without raising error
:type failure_allowed: boolean
:param logging: None or a filepath to log output to
:return: True when the phase has finished execution
"""
userpriv = userpriv and is_userpriv_capable()
sandbox = sandbox and is_sandbox_capable()
tmpdir = kwargs.get('tmpdir', env.get('T', None))
if env is None:
env = expected_ebuild_env(pkg)
ebd = request_ebuild_processor(userpriv=userpriv, sandbox=sandbox, fd_pipes=fd_pipes)
# this is a bit of a hack; used until ebd accepts observers that handle
# the output redirection on its own. Primary relevance is when
# stdout/stderr are pointed at a file; we leave buffering on, just
# force the flush for synchronization.
sys.stdout.flush()
sys.stderr.flush()
try:
if not ebd.run_phase(phase, env, tmpdir=tmpdir, sandbox=sandbox,
logging=logging, additional_commands=extra_handlers):
if not failure_allowed:
raise format.GenericBuildError(
phase + ": Failed building (False/0 return from handler)")
logger.warning(f"executing phase {phase}: execution failed, ignoring")
except Exception as e:
if isinstance(e, ebd_ipc.IpcError):
# notify bash side of IPC error
ebd.write(e.ret)
if isinstance(e, ebd_ipc.IpcInternalError):
# show main exception cause for internal IPC errors
ebd.shutdown_processor(force=True)
raise e.__cause__
try:
ebd.shutdown_processor()
except ProcessorError as pe:
# catch die errors during shutdown
e = pe
release_ebuild_processor(ebd)
if isinstance(e, ProcessorError):
# force verbose die output
e._verbosity = 1
raise e
elif isinstance(e, IGNORED_EXCEPTIONS + (format.GenericBuildError,)):
raise
raise format.GenericBuildError(
f"Executing phase {phase}: Caught exception: {e}") from e
release_ebuild_processor(ebd)
return True
def __del__(self):
if getattr(self, 'underway', False):
logger.warning(f"{self} merge was underway, but wasn't completed")
self.lock.release_write_lock()
self.clean_tempdir()
def format(self, op):
# <type> - ebuild, block or nomerge (for --tree)
# N - new package
# R - rebuild package
# F - fetch restricted
# f - fetch restricted already downloaded
# D - downgrade
# U - updating to another version
# # - masked
# * - missing keyword
# ~ - unstable keyword
# Caveats:
# - U and D are both displayed to show a downgrade - this is kept
# in order to be consistent with existing portage behaviour
out = self.out
origautoline = out.autoline
out.autoline = False
self.pkg_disabled_use = self.pkg_forced_use = set()
if hasattr(self, 'pkg_get_use'):
self.pkg_forced_use, _, self.pkg_disabled_use = self.pkg_get_use(
op.pkg)
# This is for the summary at the end
if self.quiet_repo_display:
self.repos.setdefault(op.pkg.repo, len(self.repos) + 1)
pkg_is_bold = any(
x.match(op.pkg) for x in getattr(self, 'world_list', ()))
# We don't do blockers or --tree stuff yet
data = ['[']
pkg_coloring = []
if pkg_is_bold:
pkg_coloring.append(out.bold)
if op.desc == 'remove':
pkg_coloring.insert(0, out.fg('red'))
data += pkg_coloring + ['uninstall']
elif getattr(op.pkg, 'built', False):
pkg_coloring.insert(0, out.fg('magenta'))
data += pkg_coloring + ['binary']
else:
pkg_coloring.insert(0, out.fg('green'))
data += pkg_coloring + ['ebuild']
data += [out.reset, ' ']
out.write(*data)
# Order is important here - look at the above diagram
op_type = op.desc
op_chars = [[' '] for x in range(7)]
if 'fetch' in op.pkg.restrict:
if all(
os.path.isfile(pjoin(self.distdir, f))
for f in op.pkg.distfiles):
fetched = [out.fg('green'), out.bold, 'f', out.reset]
else:
fetched = [out.fg('red'), out.bold, 'F', out.reset]
op_chars[3] = fetched
if op.desc == "add":
op_chars[1] = [out.fg('green'), out.bold, 'N', out.reset]
if op.pkg.slot != '0' and self.installed_repos.match(
op.pkg.unversioned_atom):
op_chars[2] = [out.fg('green'), out.bold, 'S', out.reset]
op_type = 'slotted_add'
elif op.desc == "replace":
if op.pkg == op.old_pkg:
op_chars[2] = [out.fg('yellow'), out.bold, 'R', out.reset]
else:
op_chars[4] = [out.fg('cyan'), out.bold, 'U', out.reset]
if op.pkg > op.old_pkg:
op_type = 'upgrade'
else:
op_chars[5] = [out.fg('blue'), out.bold, 'D', out.reset]
op_type = 'downgrade'
elif op.desc == 'remove':
pass
else:
logger.warning("unformattable op type: desc(%r), %r", op.desc, op)
if self.verbosity > 0:
if (self.unstable_arch in op.pkg.keywords and self.unstable_arch
not in op.pkg.repo.domain_settings['ACCEPT_KEYWORDS']):
op_chars[6] = [out.fg('yellow'), out.bold, '~', out.reset]
elif not op.pkg.keywords:
op_chars[6] = [out.fg('red'), out.bold, '*', out.reset]
else:
if op.pkg.repo.masked.match(op.pkg.versioned_atom):
op_chars[6] = [out.fg('red'), out.bold, '#', out.reset]
out.write(*(iflatten_instance(op_chars)))
out.write('] ')
self.visit_op(op_type)
pkg = [op.pkg.cpvstr]
if self.verbosity > 0:
if op.pkg.subslot != op.pkg.slot:
pkg.append(f":{op.pkg.slot}/{op.pkg.subslot}")
elif op.pkg.slot != '0':
pkg.append(f":{op.pkg.slot}")
if not self.quiet_repo_display and op.pkg.source_repository:
pkg.append(f"::{op.pkg.source_repository}")
out.write(*(pkg_coloring + pkg + [out.reset]))
installed = []
if op.desc == 'replace':
old_pkg = [op.old_pkg.fullver]
if self.verbosity > 0:
if op.old_pkg.subslot != op.old_pkg.slot:
old_pkg.append(f":{op.old_pkg.slot}/{op.old_pkg.subslot}")
elif op.old_pkg.slot != '0':
old_pkg.append(f":{op.old_pkg.slot}")
if not self.quiet_repo_display and op.old_pkg.source_repository:
old_pkg.append(f"::{op.old_pkg.source_repository}")
if op_type != 'replace' or op.pkg.source_repository != op.old_pkg.source_repository:
installed = ''.join(old_pkg)
elif op_type == 'slotted_add':
if self.verbosity > 0:
pkgs = sorted(f"{x.fullver}:{x.slot}"
for x in self.installed_repos.match(
op.pkg.unversioned_atom))
else:
pkgs = sorted(x.fullver for x in self.installed_repos.match(
op.pkg.unversioned_atom))
installed = ', '.join(pkgs)
# output currently installed versions
if installed:
out.write(' ', out.fg('blue'), out.bold, f'[{installed}]',
out.reset)
# Build a list of (useflags, use_expand_dicts) tuples.
# HACK: if we are in "replace" mode we build a list of length
# 4, else this is a list of length 2. We then pass this to
# format_use which can take either 2 or 4 arguments.
uses = ((), ())
if op.desc == 'replace':
uses = (op.pkg.iuse_stripped, op.pkg.use, op.old_pkg.iuse_stripped,
op.old_pkg.use)
elif op.desc == 'add':
uses = (op.pkg.iuse_stripped, op.pkg.use)
stuff = list(map(self.use_splitter, uses))
# Convert the list of tuples to a list of lists and a list of
# dicts (both length 2 or 4).
uselists, usedicts = list(zip(*stuff))
# output USE flags
self.format_use('use', *uselists)
# output USE_EXPAND flags
for expand in sorted(self.use_expand - self.use_expand_hidden):
flaglists = [d.get(expand, ()) for d in usedicts]
self.format_use(expand, *flaglists)
# output download size
if self.verbosity > 0:
if not op.pkg.built:
downloads = set(f for f in op.pkg.distfiles
if not os.path.isfile(pjoin(self.distdir, f)))
if downloads.difference(self.downloads):
self.downloads.update(downloads)
size = sum(
v['size']
for dist, v in op.pkg.manifest.distfiles.items()
if dist in downloads)
if size:
self.download_size += size
out.write(' ', sizeof_fmt(size))
if self.quiet_repo_display:
out.write(out.fg('cyan'), f" [{self.repos[op.pkg.repo]}]")
out.write('\n')
out.autoline = origautoline
def _parse_config(self):
"""Load data from the repo's metadata/layout.conf file."""
path = pjoin(self.location, self.layout_offset)
data = read_dict(iter_read_bash(
readlines(path, strip_whitespace=True, swallow_missing=True)),
source_isiter=True,
strip=True,
filename=path,
ignore_errors=True)
sf = object.__setattr__
sf(self, 'repo_name', data.get('repo-name', None))
hashes = data.get('manifest-hashes', '').lower().split()
if hashes:
hashes = ['size'] + hashes
hashes = tuple(iter_stable_unique(hashes))
else:
hashes = self.default_hashes
required_hashes = data.get('manifest-required-hashes',
'').lower().split()
if required_hashes:
required_hashes = ['size'] + required_hashes
required_hashes = tuple(iter_stable_unique(required_hashes))
else:
required_hashes = self.default_required_hashes
manifest_policy = data.get('use-manifests', 'strict').lower()
d = {
'disabled': (manifest_policy == 'false'),
'strict': (manifest_policy == 'strict'),
'thin': (data.get('thin-manifests', '').lower() == 'true'),
'signed': (data.get('sign-manifests', 'true').lower() == 'true'),
'hashes': hashes,
'required_hashes': required_hashes,
}
sf(self, 'manifests', _immutable_attr_dict(d))
masters = data.get('masters')
_missing_masters = False
if masters is None:
if not self.is_empty:
logger.warning(
f"{self.repo_id} repo at {self.location!r}, doesn't "
"specify masters in metadata/layout.conf. Please explicitly "
"set masters (use \"masters =\" if the repo is standalone)."
)
_missing_masters = True
masters = ()
else:
masters = tuple(iter_stable_unique(masters.split()))
sf(self, '_missing_masters', _missing_masters)
sf(self, 'masters', masters)
aliases = data.get('aliases', '').split() + [
self.config_name, self.repo_name, self.pms_repo_name, self.location
]
sf(self, 'aliases', tuple(filter(None, iter_stable_unique(aliases))))
sf(self, 'eapis_deprecated',
tuple(iter_stable_unique(data.get('eapis-deprecated', '').split())))
sf(self, 'eapis_banned',
tuple(iter_stable_unique(data.get('eapis-banned', '').split())))
sf(
self, 'properties_allowed',
tuple(
iter_stable_unique(data.get('properties-allowed',
'').split())))
sf(self, 'restrict_allowed',
tuple(iter_stable_unique(data.get('restrict-allowed', '').split())))
v = set(data.get('cache-formats', 'md5-dict').lower().split())
if not v:
v = [None]
else:
# sort into favored order
v = [f for f in self.supported_cache_formats if f in v]
if not v:
logger.warning(
f'unknown cache format: falling back to md5-dict format')
v = ['md5-dict']
sf(self, 'cache_format', list(v)[0])
profile_formats = set(
data.get('profile-formats', 'pms').lower().split())
if not profile_formats:
logger.info(
f"{self.repo_id!r} repo at {self.location!r} has explicitly "
"unset profile-formats, defaulting to pms")
profile_formats = {'pms'}
unknown = profile_formats.difference(self.supported_profile_formats)
if unknown:
logger.info("%r repo at %r has unsupported profile format%s: %s",
self.repo_id, self.location, pluralism(unknown),
', '.join(sorted(unknown)))
profile_formats.difference_update(unknown)
profile_formats.add('pms')
sf(self, 'profile_formats', profile_formats)
def finish(self):
ret = self.format_op.finalize()
self.format_op.cleanup(disable_observer=True)
if not ret:
logger.warning(f"ignoring unexpected result from uninstall finalize- {ret!r}")
return base.finish(self)
def __del__(self):
if getattr(self, 'underway', False):
logger.warning(f"{self.old_pkg} unmerge was underway, but wasn't completed")
self.lock.release_write_lock()
def package_masks(iterable):
for line, lineno, path in iterable:
try:
yield parse_match(line), line, lineno, path
except ParseError as e:
logger.warning(f'{path!r}, line {lineno}: parsing error: {e}')
def __init__(self, location=None, profile_override=None, **kwargs):
"""
Args:
location (optional[str]): path to the portage config directory,
(defaults to /etc/portage)
profile_override (optional[str]): profile to use instead of the current system
profile, i.e. the target of the /etc/portage/make.profile symlink
configroot (optional[str]): location for various portage config files (defaults to /)
root (optional[str]): target root filesystem (defaults to /)
buildpkg (optional[bool]): forcibly disable/enable building binpkgs, otherwise
FEATURES=buildpkg from make.conf is used
Returns:
dict: config settings
"""
self._config = {}
location = location if location is not None else '/etc/portage'
self.dir = pjoin(
os.environ.get('PORTAGE_CONFIGROOT', kwargs.pop('configroot',
'/')),
location.lstrip('/'))
# this actually differs from portage parsing- we allow
# make.globals to provide vars used in make.conf, portage keeps
# them separate (kind of annoying)
#
# this isn't preserving incremental behaviour for features/use unfortunately
make_conf = {}
try:
self.load_make_conf(make_conf,
pjoin(const.CONFIG_PATH, 'make.globals'))
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
raise config_errors.ParsingError(
"failed to load make.globals") from e
self.load_make_conf(make_conf,
pjoin(self.dir, 'make.conf'),
required=False,
allow_sourcing=True,
incrementals=True)
self.root = os.environ.get(
"ROOT", kwargs.pop('root', make_conf.get("ROOT", "/")))
gentoo_mirrors = [
x.rstrip("/") + "/distfiles"
for x in make_conf.pop("GENTOO_MIRRORS", "").split()
]
self.features = frozenset(
optimize_incrementals(make_conf.get('FEATURES', '').split()))
self._add_sets()
self._add_profile(profile_override)
self['vdb'] = basics.AutoConfigSection({
'class':
'pkgcore.vdb.ondisk.tree',
'location':
pjoin(self.root, 'var', 'db', 'pkg'),
'cache_location':
'/var/cache/edb/dep/var/db/pkg',
})
try:
repos_conf_defaults, repos_conf = self.load_repos_conf(
pjoin(self.dir, 'repos.conf'))
except config_errors.ParsingError as e:
if not getattr(getattr(e, 'exc', None), 'errno',
None) == errno.ENOENT:
raise
try:
# fallback to defaults provided by pkgcore
repos_conf_defaults, repos_conf = self.load_repos_conf(
pjoin(const.CONFIG_PATH, 'repos.conf'))
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
raise config_errors.ParsingError(
'failed to find a usable repos.conf') from e
self['ebuild-repo-common'] = basics.AutoConfigSection({
'class':
'pkgcore.ebuild.repository.tree',
'default_mirrors':
gentoo_mirrors,
'inherit-only':
True,
})
repo_map = {}
for repo_name, repo_opts in list(repos_conf.items()):
repo_cls = repo_opts.pop('repo-type')
try:
repo = repo_cls(self,
repo_name=repo_name,
repo_opts=repo_opts,
repo_map=repo_map,
defaults=repos_conf_defaults)
except repo_errors.UnsupportedRepo as e:
logger.warning(
f'skipping {repo_name!r} repo: unsupported EAPI {str(e.repo.eapi)!r}'
)
del repos_conf[repo_name]
continue
self[repo_name] = basics.AutoConfigSection(repo)
# XXX: Hack for portage-2 profile format support. We need to figure out how
# to dynamically create this from the config at runtime on attr access.
profiles.ProfileNode._repo_map = ImmutableDict(repo_map)
self._make_repo_syncers(repos_conf, make_conf)
repos = [name for name in repos_conf.keys()]
if repos:
self['repo-stack'] = basics.FakeIncrementalDictConfigSection(
my_convert_hybrid, {
'class': 'pkgcore.repository.multiplex.config_tree',
'repos': tuple(repos)
})
self['vuln'] = basics.AutoConfigSection({
'class': SecurityUpgradesViaProfile,
'ebuild_repo': 'repo-stack',
'vdb': 'vdb',
'profile': 'profile',
})
# check if package building was forced on by the user
forced_buildpkg = kwargs.pop('buildpkg', False)
if forced_buildpkg:
make_conf['FEATURES'] += ' buildpkg'
# now add the fetcher- we delay it till here to clean out the environ
# it passes to the command.
# *everything* in make_conf must be str values also.
self._add_fetcher(make_conf)
# finally... domain.
make_conf.update({
'class': 'pkgcore.ebuild.domain.domain',
'repos': tuple(repos),
'fetcher': 'fetcher',
'default': True,
'vdb': ('vdb', ),
'profile': 'profile',
'name': 'livefs',
'root': self.root,
'config_dir': self.dir,
})
self['livefs'] = basics.FakeIncrementalDictConfigSection(
my_convert_hybrid, make_conf)
def finish(self):
ret = self.format_op.finalize()
if not ret:
logger.warning(f"ignoring unexpected result from replace finalize- {ret!r}")
return base.finish(self)
def load_repos_conf(cls, path):
"""parse repos.conf files
Args:
path (str): path to the repos.conf which can be a regular file or
directory, if a directory is passed all the non-hidden files within
that directory are parsed in alphabetical order.
Returns:
dict: global repo settings
dict: repo settings
"""
main_defaults = {}
repos = {}
parser = ParseConfig()
for fp in sorted_scan(os.path.realpath(path),
follow_symlinks=True,
nonexistent=True,
hidden=False,
backup=False):
try:
with open(fp) as f:
defaults, repo_confs = parser.parse_file(f)
except PermissionError as e:
raise base_errors.PermissionDenied(fp, write=False) from e
except EnvironmentError as e:
raise config_errors.ParsingError(f"parsing {fp!r}",
exception=e) from e
except configparser.Error as e:
raise config_errors.ParsingError(f"repos.conf: {fp!r}",
exception=e) from e
if defaults and main_defaults:
logger.warning(
f"repos.conf: parsing {fp!r}: overriding DEFAULT section")
main_defaults.update(defaults)
for name, repo_conf in repo_confs.items():
if name in repos:
logger.warning(
f"repos.conf: parsing {fp!r}: overriding {name!r} repo"
)
# ignore repo if location is unset
location = repo_conf.get('location', None)
if location is None:
logger.warning(
f"repos.conf: parsing {fp!r}: "
f"{name!r} repo missing location setting, ignoring repo"
)
continue
repo_conf['location'] = os.path.abspath(location)
# repo type defaults to ebuild for compat with portage
repo_type = repo_conf.get('repo-type', 'ebuild-v1')
try:
repo_conf['repo-type'] = cls._supported_repo_types[
repo_type]
except KeyError:
logger.warning(
f"repos.conf: parsing {fp!r}: "
f"{name!r} repo has unsupported repo-type {repo_type!r}, "
"ignoring repo")
continue
# Priority defaults to zero if unset or invalid for ebuild repos
# while binpkg repos have the lowest priority by default.
priority = repo_conf.get('priority', None)
if priority is None:
if repo_type.startswith('binpkg'):
priority = -10000
else:
priority = 0
try:
priority = int(priority)
except ValueError:
logger.warning(
f"repos.conf: parsing {fp!r}: {name!r} repo has invalid priority "
f"setting: {priority!r} (defaulting to 0)")
priority = 0
finally:
repo_conf['priority'] = priority
# register repo
repos[name] = repo_conf
if repos:
# the default repo is gentoo if unset and gentoo exists
default_repo = main_defaults.get('main-repo', 'gentoo')
if default_repo not in repos:
raise config_errors.UserConfigError(
f"repos.conf: default repo {default_repo!r} is undefined or invalid"
)
if 'main-repo' not in main_defaults:
main_defaults['main-repo'] = default_repo
# the default repo has a low priority if unset or zero
if repos[default_repo]['priority'] == 0:
repos[default_repo]['priority'] = -1000
# sort repos via priority, in this case high values map to high priorities
repos = OrderedDict((k, v) for k, v in sorted(
repos.items(), key=lambda d: d[1]['priority'], reverse=True))
return main_defaults, repos
