def test_builtin_full_override(self): # check our assumptions... # and yes, the signatures below are for file rather than # dict; we need a cpy class for the test, the ConfigHint doesn't # have to be accurate however class cls(dict): __slots__ = () self.assertRaises(TypeError, basics.ConfigType, cls) raw_hint = ConfigHint(types={ "filename": "str", "mode": "r", "buffering": "int" }, typename='file', required=['filename'], positional=['filename']) # make sure it still tries to introspect, and throws typeerror. # introspection is generally wanted- if it must be skipped, the # ConfigHint must make it explicit cls.pkgcore_config_type = raw_hint self.assertRaises(TypeError, basics.ConfigType, cls) cls.pkgcore_config_type = raw_hint.clone(authorative=True) conf = basics.ConfigType(cls) self.assertEqual(conf.name, 'file') self.assertEqual(list(conf.required), ['filename']) self.assertEqual(list(conf.positional), ['filename']) self.assertEqual(sorted(conf.types), ['buffering', 'filename', 'mode'])
def test_clone(self): c = ConfigHint(types={'foo':'list', 'one':'str'}, positional=['one'], required=['one'], typename='barn', doc='orig doc') c2 = c.clone(types={'foo':'list', 'one':'str', 'two':'str'}, required=['one', 'two']) self.assertEqual(c2.types, {'foo':'list', 'one':'str', 'two':'str'}) self.assertEqual(c2.positional, c.positional) self.assertEqual(c2.required, ['one', 'two']) self.assertEqual(c2.typename, c.typename) self.assertEqual(c2.allow_unknowns, c.allow_unknowns) self.assertEqual(c2.doc, c.doc)
class EclassConsumerSet(VersionedInstalled): pkgcore_config_type = ConfigHint( typename='pkgset', types={ 'vdb': 'refs:repo', 'repos': 'refs:repo', 'eclasses': 'list' }, ) def __init__(self, vdb, repos, eclasses): VersionedInstalled.__init__(self, vdb) self.repos = RepositoryGroup(repos) self.eclasses = frozenset(eclasses) def __iter__(self): for atom in VersionedInstalled.__iter__(self): pkgs = self.repos.match(atom) if not pkgs: # pkg is installed but no longer in any repo, just ignore it. continue assert len( pkgs) == 1, 'I do not know what I am doing: %r' % (pkgs, ) pkg = pkgs[0] if self.eclasses.isdisjoint(pkg.data.get('_eclasses_', ())): yield atom
class StackedCaches(base): """ collapse multiple eclass caches into one. Does L->R searching for eclass matches. """ pkgcore_config_type = ConfigHint( { 'caches': 'refs:eclass_cache', 'location': 'str', 'eclassdir': 'str' }, typename='eclass_cache') def __init__(self, caches, **kwds): """ :param caches: :obj:`cache` instances to stack; ordering should be desired lookup order :keyword eclassdir: override for the master eclass dir, required for eapi0 and idiot eclass usage. defaults to pulling from the first cache. """ if len(caches) < 2: raise TypeError("%s requires at least two eclass_caches" % self.__class__) kwds.setdefault("eclassdir", caches[0].eclassdir) kwds.setdefault("location", os.path.dirname(kwds["eclassdir"].rstrip(os.path.sep))) self._caches = caches base.__init__(self, **kwds) def _load_eclasses(self): return StackedDict(*[ec.eclasses for ec in self._caches])
class cache(base): pkgcore_config_type = ConfigHint({ "path": "str", "location": "str" }, typename='eclass_cache') def __init__(self, path, location=None): """ :param location: ondisk location of the tree we're working with """ base.__init__(self, location=location, eclassdir=normpath(path)) def _load_eclasses(self): """Force an update of the internal view of on disk/remote eclasses.""" ec = {} eclass_len = len(".eclass") try: files = listdir_files(self.eclassdir) except (FileNotFoundError, NotADirectoryError): return ImmutableDict() for y in files: if not y.endswith(".eclass"): continue ys = y[:-eclass_len] ec[intern(ys)] = LazilyHashedPath(pjoin(self.eclassdir, y), eclassdir=self.eclassdir) return ImmutableDict(ec)
class WorldFile(FileList): """Set of packages contained in the world file.""" pkgcore_config_type = ConfigHint(typename='pkgset') error_on_subsets = False def __init__(self, location=const.WORLD_FILE, gid=os_data.portage_gid, mode=0o644): FileList.__init__(self, location, gid=gid, mode=mode) def add(self, atom_inst): self._modify(atom_inst, FileList.add) def remove(self, atom_inst): self._modify(atom_inst, FileList.remove) def _modify(self, atom_inst, func): if atom_inst.slot: for slot in atom_inst.slot: if slot == '0': new_atom_inst = atom(atom_inst.key) else: new_atom_inst = atom(atom_inst.key + ":" + slot) func(self, new_atom_inst) else: atom_inst = atom(atom_inst.key) func(self, atom_inst)
class FakeDomain: pkgcore_config_type = ConfigHint({'repo': 'ref:repo'}, typename='domain') def __init__(self, repo): object.__init__(self) self.ebuild_repos_unfiltered = repo
class FileList: pkgcore_config_type = ConfigHint({'location': 'str'}, typename='pkgset') error_on_subsets = True def __init__(self, location, gid=os_data.portage_gid, mode=0o644): self.path = location self.gid = gid self.mode = mode # note that _atoms is generated on the fly. @klass.jit_attr def _atoms(self): try: s = set() for x in readlines_ascii(self.path, True): if not x or x.startswith("#"): continue elif x.startswith("@"): if self.error_on_subsets: raise ValueError( "set %s isn't a valid atom in pkgset %r" % (x, self.path)) logger.warning( "set item %r found in pkgset %r: it will be " "wiped on update since portage/pkgcore store set items " "in a separate way", x[1:], self.path) continue s.add(atom(x)) except InvalidDependency as e: raise errors.ParsingError("parsing %r" % self.path, exception=e) from e return s def __iter__(self): return iter(self._atoms) def __len__(self): return len(self._atoms) def __contains__(self, key): return key in self._atoms def add(self, atom_inst): self._atoms.add(atom_inst) def remove(self, atom_inst): self._atoms.remove(atom_inst) def flush(self): f = None try: f = AtomicWriteFile(self.path, gid=self.gid, perms=self.mode) f.write("\n".join(str(x) for x in sorted(self._atoms))) f.close() except: if f is not None: f.discard() raise
class SyncableRepo(syncable.tree, util.SimpleTree): pkgcore_config_type = ConfigHint(typename='repo_config') def __init__(self, succeed=True): util.SimpleTree.__init__(self, {}) syncer = FakeSyncer('/fake', 'fake', succeed=succeed) syncable.tree.__init__(self, syncer)
def make_repo_config(repo_data, livefs=False, frozen=False, repo_id=None): def repo(): return fake_repo(repo_data, livefs=livefs, frozen=frozen, repo_id=repo_id) repo.pkgcore_config_type = ConfigHint(typename='repo') return basics.HardCodedConfigSection({'class': repo})
class UserProfile(OnDiskProfile): pkgcore_config_type = ConfigHint( {'user_path': 'str', 'parent_path': 'str', 'parent_profile': 'str'}, required=('user_path', 'parent_path', 'parent_profile'), typename='profile', ) def __init__(self, user_path, parent_path, parent_profile, load_profile_base=True): super().__init__(parent_path, parent_profile, load_profile_base) self.node = UserProfileNode(user_path, pjoin(parent_path, parent_profile))
class OnDiskProfile(ProfileStack): pkgcore_config_type = ConfigHint( {'basepath': 'str', 'profile': 'str'}, required=('basepath', 'profile'), typename='profile', ) def __init__(self, basepath, profile, load_profile_base=True): super().__init__(pjoin(basepath, profile)) self.basepath = basepath self.load_profile_base = load_profile_base @staticmethod def split_abspath(path): path = abspath(path) # filter's heavy, but it handles '/' while also # suppressing the leading '/' chunks = [x for x in path.split("/") if x] try: # poor mans rindex. pbase = max(idx for idx, x in enumerate(chunks) if x == 'profiles') except ValueError: # no base found. return None return pjoin("/", *chunks[:pbase+1]), '/'.join(chunks[pbase+1:]) @classmethod def from_abspath(cls, path): vals = cls.split_abspath(path) if vals is not None: vals = cls(load_profile_base=True, *vals) return vals @klass.jit_attr def stack(self): l = ProfileStack.stack.function(self) if self.load_profile_base: l = (EmptyRootNode._autodetect_and_create(self.basepath),) + l return l @klass.jit_attr def _incremental_masks(self): stack = self.stack if self.load_profile_base: stack = stack[1:] return ProfileStack._incremental_masks(self, stack_override=stack) @klass.jit_attr def _incremental_unmasks(self): stack = self.stack if self.load_profile_base: stack = stack[1:] return ProfileStack._incremental_unmasks(self, stack_override=stack)
class SystemSet: """Set of packages defined by the selected profile.""" pkgcore_config_type = ConfigHint({'profile': 'ref:profile'}, typename='pkgset') def __init__(self, profile): self.system = frozenset(profile.system) def __iter__(self): for pkg in self.system: yield pkg
class FakeDomain: pkgcore_config_type = ConfigHint({ 'repos': 'refs:repo', 'vdb': 'refs:repo' }, typename='domain') def __init__(self, repos, vdb): object.__init__(self) self.source_repos = repos self.installed_repos = vdb
class FakeDomain: pkgcore_config_type = ConfigHint({'repos': 'refs:repo'}, typename='domain') def __init__(self, repos): self.all_ebuild_repos_raw = multiplex.tree(*repos) self.root = None def add_repo(self, *args, **kwargs): """stubbed""" def find_repo(self, *args, **kwargs): """stubbed"""
class FakeDomain: pkgcore_config_type = ConfigHint({'repos': 'refs:repo', 'binpkg': 'refs:repo', 'vdb': 'refs:repo'}, typename='domain') def __init__(self, repos, binpkg, vdb): object.__init__(self) self.repos = repos self.source_repos_raw = util.RepositoryGroup(repos) self.installed_repos = util.RepositoryGroup(vdb) self.binary_repos_raw = util.RepositoryGroup(binpkg) self.vdb = vdb
class FakeRepo(FakeEbuildRepo): pkgcore_config_type = ConfigHint({}, typename='repo') def __init__(self, repo_id='faker', arches=('amd64', 'x86', 'arm', 'arm64')): config = RepoConfig('nonexistent') object.__setattr__(config, 'known_arches', frozenset(arches)) pkgs = [ FakePkg('app-arch/bzip2-1.0.1-r1', repo=self, data={'SLOT': '0'}, keywords=('x86',)), FakePkg('app-arch/bzip2-1.0.5-r2', repo=self, data={'SLOT': '0'}, keywords=('x86',)), FakePkg('sys-apps/coreutils-8.25', repo=self, data={'SLOT': '0'}), FakePkg('x11-libs/gtk+-2.24', repo=self, data={'SLOT': '2'}, keywords=('amd64',)), FakePkg('x11-libs/gtk+-3.20', repo=self, data={'SLOT': '3'}, keywords=('amd64', 'x86')), ] super().__init__(repo_id=repo_id, pkgs=pkgs, config=config)
class Formatter: """Base Formatter class: All formatters should be subclasses of this.""" pkgcore_config_type = ConfigHint(typename='pmerge_formatter', raw_class=True) def __init__(self, **kwargs): self.__dict__.update(kwargs) def format(self, op): """Formats an op. Subclasses must define this method""" raise NotImplementedError(self.format) def ask(self, question, responses=None, default_answer=None, limit=3): return userquery(question, self.out, self.err, responses, default_answer, limit) def end(self): """Called at the end, normally for summary information"""
class SecurityUpgrades(metaclass=generic_equality): """Set of packages for available security upgrades.""" pkgcore_config_type = ConfigHint( { 'ebuild_repo': 'ref:repo', 'vdb': 'ref:vdb' }, typename='pkgset') __attr_comparison__ = ('arch', 'glsa_src', 'vdb') def __init__(self, ebuild_repo, vdb, arch): self.glsa_src = GlsaDirSet(ebuild_repo) self.vdb = vdb self.arch = arch def __iter__(self): for glsa, matches in find_vulnerable_repo_pkgs(self.glsa_src, self.vdb, grouped=True, arch=self.arch): yield packages.KeyedAndRestriction(glsa[0], restriction.Negate(glsa[1]))
class GlsaDirSet(metaclass=generic_equality): """generate a pkgset based on GLSA's distributed via a directory. (rsync tree is the usual source.) """ pkgcore_config_type = ConfigHint({'src': 'ref:repo'}, typename='pkgset') op_translate = {"ge": ">=", "gt": ">", "lt": "<", "le": "<=", "eq": "="} __attr_comparison__ = ('paths', ) def __init__(self, src): """ :param src: where to get the glsa from :type src: must be either full path to glsa dir, or a repo object to pull it from """ if not isinstance(src, str): src = tuple( sorted( filter(os.path.isdir, (pjoin(repo.base, 'metadata', 'glsa') for repo in get_virtual_repos(src, False) if hasattr(repo, 'base'))))) else: src = [src] self.paths = src def __iter__(self): for glsa, catpkg, pkgatom, vuln in self.iter_vulnerabilities(): yield packages.KeyedAndRestriction(pkgatom, vuln, key=catpkg, tag="GLSA vulnerable:") def pkg_grouped_iter(self, sorter=None): """yield GLSA restrictions grouped by package key :param sorter: must be either None, or a comparison function """ if sorter is None: sorter = iter pkgs = {} pkgatoms = {} for glsa, pkg, pkgatom, vuln in self.iter_vulnerabilities(): pkgatoms[pkg] = pkgatom pkgs.setdefault(pkg, []).append(vuln) for pkgname in sorter(pkgs): yield packages.KeyedAndRestriction( pkgatoms[pkgname], packages.OrRestriction(*pkgs[pkgname]), key=pkgname) def iter_vulnerabilities(self): """generator yielding each GLSA restriction""" for path in self.paths: for fn in listdir_files(path): # glsa-1234-12.xml if not (fn.startswith("glsa-") and fn.endswith(".xml")): continue # This verifies the filename is of the correct syntax. try: [int(x) for x in fn[5:-4].split("-")] except ValueError: continue root = etree.parse(pjoin(path, fn)) glsa_node = root.getroot() if glsa_node.tag != 'glsa': raise ValueError("glsa without glsa rootnode") for affected in root.findall('affected'): for pkg in affected.findall('package'): try: pkgname = str(pkg.get('name')).strip() pkg_vuln_restrict = \ self.generate_intersects_from_pkg_node( pkg, tag="glsa(%s)" % fn[5:-4]) if pkg_vuln_restrict is None: continue pkgatom = atom.atom(pkgname) yield fn[5:-4], pkgname, pkgatom, pkg_vuln_restrict except (TypeError, ValueError) as e: # thrown from cpv. logger.warning( f"invalid glsa- {fn}, package {pkgname}: {e}") except IGNORED_EXCEPTIONS: raise except Exception as e: logger.warning(f"invalid glsa- {fn}: error: {e}") def generate_intersects_from_pkg_node(self, pkg_node, tag=None): arch = pkg_node.get("arch") if arch is not None: arch = tuple(str(arch.strip()).split()) if not arch or "*" in arch: arch = None vuln = list(pkg_node.findall("vulnerable")) if not vuln: return None elif len(vuln) > 1: vuln_list = [self.generate_restrict_from_range(x) for x in vuln] vuln = packages.OrRestriction(*vuln_list) else: vuln_list = [self.generate_restrict_from_range(vuln[0])] vuln = vuln_list[0] if arch is not None: vuln = packages.AndRestriction( vuln, packages.PackageRestriction( "keywords", values.ContainmentMatch2(arch, match_all=False))) invuln = (pkg_node.findall("unaffected")) if not invuln: # wrap it. return packages.KeyedAndRestriction(vuln, tag=tag) invuln_list = [ self.generate_restrict_from_range(x, negate=True) for x in invuln ] invuln = [x for x in invuln_list if x not in vuln_list] if not invuln: if tag is None: return packages.KeyedAndRestriction(vuln, tag=tag) return packages.KeyedAndRestriction(vuln, tag=tag) return packages.KeyedAndRestriction(vuln, tag=tag, *invuln) def generate_restrict_from_range(self, node, negate=False): op = str(node.get("range").strip()) slot = str(node.get("slot", "").strip()) try: restrict = self.op_translate[op.lstrip("r")] except KeyError: raise ValueError(f'unknown operator: {op!r}') if node.text is None: raise ValueError(f"{op!r} node missing version") base = str(node.text.strip()) glob = base.endswith("*") if glob: base = base[:-1] base = cpv.VersionedCPV(f"cat/pkg-{base}") if glob: if op != "eq": raise ValueError(f"glob cannot be used with {op} ops") return packages.PackageRestriction( "fullver", values.StrGlobMatch(base.fullver)) restrictions = [] if op.startswith("r"): if not base.revision: if op == "rlt": # rlt -r0 can never match # this is a non-range. raise ValueError( "range %s version %s is a guaranteed empty set" % (op, str(node.text.strip()))) elif op == "rle": # rle -r0 -> = -r0 return atom_restricts.VersionMatch("=", base.version, negate=negate) elif op == "rge": # rge -r0 -> ~ return atom_restricts.VersionMatch("~", base.version, negate=negate) # rgt -r0 passes through to regular ~ + > restrictions.append(atom_restricts.VersionMatch("~", base.version)) restrictions.append( atom_restricts.VersionMatch(restrict, base.version, rev=base.revision), ) if slot: restrictions.append(atom_restricts.SlotDep(slot)) return packages.AndRestriction(*restrictions, negate=negate)
class Class(NewStyleClass): pkgcore_config_type = ConfigHint(types={'two': 'bool'}, doc='interesting')
class tree(prototype.tree): """Repository for packages installed on the filesystem.""" livefs = True configured = False configurables = ("domain", "settings") configure = None package_factory = staticmethod(ebuild_built.generate_new_factory) operations_kls = repo_ops.operations pkgcore_config_type = ConfigHint( {'location': 'str', 'cache_location': 'str', 'repo_id': 'str', 'disable_cache': 'bool'}, typename='repo') def __init__(self, location, cache_location=None, repo_id='vdb', disable_cache=False): super().__init__(frozen=False) self.repo_id = repo_id self.location = location if disable_cache: cache_location = None elif cache_location is None: cache_location = pjoin("/var/cache/edb/dep", location.lstrip("/")) self.cache_location = cache_location self._versions_tmp_cache = {} try: st = os.stat(self.location) if not stat.S_ISDIR(st.st_mode): raise errors.InitializationError( f"base not a dir: {self.location!r}") elif not st.st_mode & (os.X_OK|os.R_OK): raise errors.InitializationError( f"base lacks read/executable: {self.location!r}") except FileNotFoundError: pass except OSError as e: raise errors.InitializationError(f'lstat failed on base: {self.location!r}') from e self.package_class = self.package_factory(self) def _get_categories(self, *optional_category): # return if optional_category is passed... cause it's not yet supported if optional_category: return {} try: try: return tuple(x for x in listdir_dirs(self.location) if not x.startswith('.')) except EnvironmentError as e: raise KeyError(f"failed fetching categories: {e}") from e finally: pass def _get_packages(self, category): cpath = pjoin(self.location, category.lstrip(os.path.sep)) l = set() d = {} bad = False try: for x in listdir_dirs(cpath): if x.startswith(".tmp.") or x.endswith(".lockfile") \ or x.startswith("-MERGING-"): continue try: pkg = VersionedCPV(f'{category}/{x}') except InvalidCPV: bad = True if bad or not pkg.fullver: if '-scm' in x: bad = 'scm' elif '-try' in x: bad = 'try' else: raise InvalidCPV(f'{category}/{x}', 'no version component') logger.error( f'merged -{bad} pkg detected: {category}/{x}. ' f'throwing exception due to -{bad} not being a valid' ' version component. Silently ignoring that ' 'specific version is not viable either since it ' 'would result in pkgcore stomping whatever it was ' f'that -{bad} version merged. ' 'Use the offending pkg manager that merged it to ' 'unmerge it.') raise InvalidCPV( f'{category}/{x}', f'{bad} version component is not standard.') l.add(pkg.package) d.setdefault((category, pkg.package), []).append(pkg.fullver) except EnvironmentError as e: category = pjoin(self.location, category.lstrip(os.path.sep)) raise KeyError(f'failed fetching packages for category {category}: {e}') from e self._versions_tmp_cache.update(d) return tuple(l) def _get_versions(self, catpkg): return tuple(self._versions_tmp_cache.pop(catpkg)) def _get_ebuild_path(self, pkg): s = f"{pkg.package}-{pkg.fullver}" return pjoin(self.location, pkg.category, s, s + ".ebuild") def _get_path(self, pkg): s = f"{pkg.package}-{pkg.fullver}" return pjoin(self.location, pkg.category, s) _metadata_rewrites = { "bdepend": "BDEPEND", "depend": "DEPEND", "rdepend": "RDEPEND", "pdepend": "PDEPEND", "use": "USE", "eapi": "EAPI", "CONTENTS": "contents", "source_repository": "repository", "fullslot": "SLOT", } def _get_metadata(self, pkg): return IndeterminantDict( partial(self._internal_load_key, pjoin( self.location, pkg.category, f"{pkg.package}-{pkg.fullver}"))) def _internal_load_key(self, path, key): key = self._metadata_rewrites.get(key, key) if key == "contents": data = ContentsFile(pjoin(path, "CONTENTS"), mutable=True) elif key == "environment": fp = pjoin(path, key) if not os.path.exists(f'{fp}.bz2'): if not os.path.exists(fp): # icky. raise KeyError("environment: no environment file found") data = data_source.local_source(fp) else: data = data_source.bz2_source(f'{fp}.bz2') elif key == 'ebuild': fp = pjoin(path, os.path.basename(path.rstrip(os.path.sep)) + '.ebuild') data = data_source.local_source(fp) elif key == 'repo': # try both, for portage/paludis compatibility. data = readfile(pjoin(path, 'repository'), True) if data is None: data = readfile(pjoin(path, 'REPOSITORY'), True) if data is None: raise KeyError(key) else: data = readfile(pjoin(path, key), True) if data is None: raise KeyError((path, key)) data = data.rstrip('\n') return data def notify_remove_package(self, pkg): remove_it = len(self.packages[pkg.category]) == 1 prototype.tree.notify_remove_package(self, pkg) if remove_it: try: os.rmdir(pjoin(self.location, pkg.category)) except OSError as oe: # POSIX specifies either ENOTEMPTY or EEXIST for non-empty dir # in particular, Solaris uses EEXIST in that case. # https://github.com/pkgcore/pkgcore/pull/181 if oe.errno not in (errno.ENOTEMPTY, errno.EEXIST): raise # silently swallow it; del oe def __str__(self): return f"{self.repo_id}: location {self.location}"
class database(flat_hash.database): """Compatibility with (older) portage-generated caches. Autodetects per entry if it is a :class:`flat_hash.database` or PMS compliant cache entry, and converts old (and incomplete) INHERITED field to _eclasses_ as required. """ pkgcore_config_type = ConfigHint( { 'readonly': 'bool', 'location': 'str', 'label': 'str', 'eclasses': 'ref:eclass_cache' }, required=['location'], positional=['location'], typename='cache') # No eclass validation data is stored. eclass_chf_types = [] eclass_splitter = ' ' chf_type = 'mtime' complete_eclass_entries = True auxdbkeys_order = ( 'DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', 'KEYWORDS', '_eclasses_', 'IUSE', 'REQUIRED_USE', 'PDEPEND', 'BDEPEND', 'EAPI', 'PROPERTIES', 'DEFINED_PHASES', ) # this is the old cache format, flat_list. hardcoded, and must # remain that way. magic_line_count = 22 autocommits = True def __init__(self, location, *args, **config): self.ec = config.pop("eclasses", None) if self.ec is None: self.ec = eclass_cache.cache(pjoin(location, "eclass"), location) config.pop('label', None) self.mtime_in_entry = config.pop('mtime_in_entry', True) location = pjoin(location, 'metadata', 'cache') super().__init__(location, *args, **config) self.hardcoded_auxdbkeys_order = tuple( (idx, key) for idx, key in enumerate(self.auxdbkeys_order) if key in self._known_keys) self.hardcoded_auxdbkeys_processing = tuple( (key in self._known_keys and key or None) for key in self.auxdbkeys_order) __init__.__doc__ = flat_hash.database.__init__.__doc__.replace( "@keyword location", "@param location") def _parse_data(self, data, mtime): i = iter(self.hardcoded_auxdbkeys_processing) d = self._cdict_kls([(key, val) for (key, val) in zip(i, data) if key]) # sadly, this is faster then doing a .next() and snagging the # exception for x in i: # if we reach here, then bad things occurred. raise errors.GeneralCacheCorruption( "wrong line count, requires %i" % (self.magic_line_count, )) if self._mtime_used: # and not self.mtime_in_entry: d["_mtime_"] = int(mtime) return d def _setitem(self, cpv, values): values = ProtectedDict(values) # hack. proper solution is to make this a __setitem__ override, since # template.__setitem__ serializes _eclasses_, then we reconstruct it. eclasses = values.pop('_eclasses_', None) if eclasses is not None: eclasses = self.reconstruct_eclasses(cpv, eclasses) values["INHERITED"] = ' '.join(eclasses) s = cpv.rfind('/') fp = pjoin(self.location, cpv[:s], f'.update.{os.getpid()}.{cpv[s+1:]}') try: myf = open(fp, "w") except FileNotFoundError: try: self._ensure_dirs(cpv) myf = open(fp, "w") except EnvironmentError as e: raise errors.CacheCorruption(cpv, e) from e except EnvironmentError as e: raise errors.CacheCorruption(cpv, e) from e count = 0 for idx, key in self.hardcoded_auxdbkeys_order: myf.write("%s%s" % ("\n" * (idx - count), values.get(key, ""))) count = idx myf.write("\n" * (self.magic_line_count - count)) myf.close() self._set_mtime(fp, values, eclasses) # update written, now we move it new_fp = pjoin(self.location, cpv) try: os.rename(fp, new_fp) except EnvironmentError as e: os.remove(fp) raise errors.CacheCorruption(cpv, e) from e def _set_mtime(self, fp, values, eclasses): if self._mtime_used: self._ensure_access(fp, mtime=values["_mtime_"])
class Cache: pkgcore_config_type = ConfigHint(typename='cache') def __init__(self, readonly=True): self.readonly = self.frozen = readonly
class RepoConfig(syncable.tree, klass.ImmutableInstance, metaclass=WeakInstMeta): """Configuration data for an ebuild repository.""" layout_offset = "metadata/layout.conf" default_hashes = ('size', 'blake2b', 'sha512') default_required_hashes = ('size', 'blake2b') supported_profile_formats = ('pms', 'portage-1', 'portage-2', 'profile-set') supported_cache_formats = ('md5-dict', 'pms') __inst_caching__ = True pkgcore_config_type = ConfigHint(typename='repo_config', types={ 'config_name': 'str', 'syncer': 'lazy_ref:syncer', }) def __init__(self, location, config_name=None, syncer=None, profiles_base='profiles'): object.__setattr__(self, 'config_name', config_name) object.__setattr__(self, 'location', location) object.__setattr__(self, 'profiles_base', pjoin(self.location, profiles_base)) if not self.eapi.is_supported: raise repo_errors.UnsupportedRepo(self) super().__init__(syncer) self._parse_config() def _parse_config(self): """Load data from the repo's metadata/layout.conf file.""" path = pjoin(self.location, self.layout_offset) data = read_dict(iter_read_bash( readlines(path, strip_whitespace=True, swallow_missing=True)), source_isiter=True, strip=True, filename=path, ignore_errors=True) sf = object.__setattr__ sf(self, 'repo_name', data.get('repo-name', None)) hashes = data.get('manifest-hashes', '').lower().split() if hashes: hashes = ['size'] + hashes hashes = tuple(iter_stable_unique(hashes)) else: hashes = self.default_hashes required_hashes = data.get('manifest-required-hashes', '').lower().split() if required_hashes: required_hashes = ['size'] + required_hashes required_hashes = tuple(iter_stable_unique(required_hashes)) else: required_hashes = self.default_required_hashes manifest_policy = data.get('use-manifests', 'strict').lower() d = { 'disabled': (manifest_policy == 'false'), 'strict': (manifest_policy == 'strict'), 'thin': (data.get('thin-manifests', '').lower() == 'true'), 'signed': (data.get('sign-manifests', 'true').lower() == 'true'), 'hashes': hashes, 'required_hashes': required_hashes, } sf(self, 'manifests', _immutable_attr_dict(d)) masters = data.get('masters') _missing_masters = False if masters is None: if not self.is_empty: logger.warning( f"{self.repo_id} repo at {self.location!r}, doesn't " "specify masters in metadata/layout.conf. Please explicitly " "set masters (use \"masters =\" if the repo is standalone)." ) _missing_masters = True masters = () else: masters = tuple(iter_stable_unique(masters.split())) sf(self, '_missing_masters', _missing_masters) sf(self, 'masters', masters) aliases = data.get('aliases', '').split() + [ self.config_name, self.repo_name, self.pms_repo_name, self.location ] sf(self, 'aliases', tuple(filter(None, iter_stable_unique(aliases)))) sf(self, 'eapis_deprecated', tuple(iter_stable_unique(data.get('eapis-deprecated', '').split()))) sf(self, 'eapis_banned', tuple(iter_stable_unique(data.get('eapis-banned', '').split()))) sf( self, 'properties_allowed', tuple( iter_stable_unique(data.get('properties-allowed', '').split()))) sf(self, 'restrict_allowed', tuple(iter_stable_unique(data.get('restrict-allowed', '').split()))) v = set(data.get('cache-formats', 'md5-dict').lower().split()) if not v: v = [None] else: # sort into favored order v = [f for f in self.supported_cache_formats if f in v] if not v: logger.warning( f'unknown cache format: falling back to md5-dict format') v = ['md5-dict'] sf(self, 'cache_format', list(v)[0]) profile_formats = set( data.get('profile-formats', 'pms').lower().split()) if not profile_formats: logger.info( f"{self.repo_id!r} repo at {self.location!r} has explicitly " "unset profile-formats, defaulting to pms") profile_formats = {'pms'} unknown = profile_formats.difference(self.supported_profile_formats) if unknown: logger.info("%r repo at %r has unsupported profile format%s: %s", self.repo_id, self.location, pluralism(unknown), ', '.join(sorted(unknown))) profile_formats.difference_update(unknown) profile_formats.add('pms') sf(self, 'profile_formats', profile_formats) @klass.jit_attr def known_arches(self): """All valid KEYWORDS for the repo.""" try: return frozenset( iter_read_bash(pjoin(self.profiles_base, 'arch.list'))) except FileNotFoundError: return frozenset() @klass.jit_attr def arches_desc(self): """Arch stability status (GLEP 72). See https://www.gentoo.org/glep/glep-0072.html for more details. """ fp = pjoin(self.profiles_base, 'arches.desc') d = {'stable': set(), 'transitional': set(), 'testing': set()} try: for lineno, line in iter_read_bash(fp, enum_line=True): try: arch, status = line.split() except ValueError: logger.error(f"{self.repo_id}::profiles/arches.desc, " f"line {lineno}: invalid line format: " "should be '<arch> <status>'") continue if arch not in self.known_arches: logger.warning(f"{self.repo_id}::profiles/arches.desc, " f"line {lineno}: unknown arch: {arch!r}") continue if status not in d: logger.warning( f"{self.repo_id}::profiles/arches.desc, " f"line {lineno}: unknown status: {status!r}") continue d[status].add(arch) except FileNotFoundError: pass return mappings.ImmutableDict(d) @klass.jit_attr def use_desc(self): """Global USE flags for the repo.""" # todo: convert this to using a common exception base, with # conversion of ValueErrors... def converter(key): return (packages.AlwaysTrue, key) return tuple(self._split_use_desc_file('use.desc', converter)) @klass.jit_attr def use_local_desc(self): """Local USE flags for the repo.""" def converter(key): # todo: convert this to using a common exception base, with # conversion of ValueErrors/atom exceptions... chunks = key.split(':', 1) return (atom.atom(chunks[0]), chunks[1]) return tuple(self._split_use_desc_file('use.local.desc', converter)) @klass.jit_attr def use_expand_desc(self): """USE_EXPAND settings for the repo.""" base = pjoin(self.profiles_base, 'desc') d = dict() try: targets = listdir_files(base) except FileNotFoundError: targets = [] for use_group in targets: group = use_group.split('.', 1)[0] d[group] = tuple( self._split_use_desc_file(f'desc/{use_group}', lambda k: f'{group}_{k}', matcher=False)) return mappings.ImmutableDict(d) def _split_use_desc_file(self, name, converter, matcher=True): line = None fp = pjoin(self.profiles_base, name) try: for line in iter_read_bash(fp): try: key, val = line.split(None, 1) key = converter(key) if matcher: yield key[0], (key[1], val.split('-', 1)[1].strip()) else: yield key, val.split('-', 1)[1].strip() except ValueError as e: logger.error(f'failed parsing {fp!r}, line {line!r}: {e}') except FileNotFoundError: pass except ValueError as e: logger.error(f'failed parsing {fp!r}: {e}') @klass.jit_attr def is_empty(self): """Return boolean related to if the repo has files in it.""" result = True try: # any files existing means it's not empty result = not listdir(self.location) if result: logger.debug(f"repo is empty: {self.location!r}") except FileNotFoundError: pass return result @klass.jit_attr def pms_repo_name(self): """Repository name from profiles/repo_name (as defined by PMS). We're more lenient than the spec and don't verify it conforms to the specified format. """ name = readfile(pjoin(self.profiles_base, 'repo_name'), none_on_missing=True) if name is not None: name = name.split('\n', 1)[0].strip() return name @klass.jit_attr def repo_id(self): """Main identifier for the repo. The precedence order is as follows: repos.conf name, repo-name from metadata/layout.conf, profiles/repo_name, and finally a fallback to the repo's location for unlabeled repos. """ if self.config_name: return self.config_name if self.repo_name: return self.repo_name if self.pms_repo_name: return self.pms_repo_name if not self.is_empty: logger.warning(f"repo lacks a defined name: {self.location!r}") return self.location @klass.jit_attr def updates(self): """Package updates for the repo defined in profiles/updates/*.""" updates_dir = pjoin(self.profiles_base, 'updates') d = pkg_updates.read_updates(updates_dir) return mappings.ImmutableDict(d) @klass.jit_attr def categories(self): categories = readlines(pjoin(self.profiles_base, 'categories'), True, True, True) if categories is not None: return tuple(map(intern, categories)) return () @klass.jit_attr def profiles(self): return Profiles(self) @klass.jit_attr def base_profile(self): return profiles.EmptyRootNode(self.profiles_base) @klass.jit_attr def eapi(self): try: return self.base_profile.eapi except profiles.NonexistentProfile: return get_eapi('0') @klass.jit_attr def pkg_masks(self): """Package masks from profiles/package.mask.""" return frozenset(self.base_profile.masks[1]) @klass.jit_attr def pkg_deprecated(self): """Deprecated packages from profiles/package.deprecated.""" return frozenset(self.base_profile.pkg_deprecated[1])
class UnconfiguredTree(prototype.tree): """Raw implementation supporting standard ebuild tree. Return packages don't have USE configuration bound to them. """ false_packages = frozenset(["CVS", ".svn"]) false_categories = frozenset([ "eclass", "profiles", "packages", "distfiles", "metadata", "licenses", "scripts", "CVS", "local" ]) configured = False configurables = ("domain", "settings") configure = None package_factory = staticmethod(ebuild_src.generate_new_factory) enable_gpg = False extension = '.ebuild' operations_kls = repo_operations pkgcore_config_type = ConfigHint( { 'location': 'str', 'eclass_cache': 'ref:eclass_cache', 'masters': 'refs:repo', 'cache': 'refs:cache', 'default_mirrors': 'list', 'allow_missing_manifests': 'bool', 'repo_config': 'ref:repo_config', }, typename='repo') def __init__(self, location, eclass_cache=None, masters=(), cache=(), default_mirrors=None, allow_missing_manifests=False, repo_config=None): """ :param location: on disk location of the tree :param cache: sequence of :obj:`pkgcore.cache.template.database` instances to use for storing metadata :param masters: repo masters this repo inherits from :param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache` instance representing the eclasses available, if None, generates the eclass_cache itself :param default_mirrors: Either None, or sequence of mirrors to try fetching from first, then falling back to other uri """ super().__init__() self.base = self.location = location try: if not stat.S_ISDIR(os.stat(self.base).st_mode): raise errors.InitializationError( f"base not a dir: {self.base}") except OSError as e: raise errors.InitializationError( f"lstat failed: {self.base}") from e if repo_config is None: repo_config = repo_objs.RepoConfig(location) self.config = repo_config # profiles dir is required by PMS if not os.path.isdir(self.config.profiles_base): raise errors.InvalidRepo( f'missing required profiles dir: {self.location!r}') # verify we support the repo's EAPI if not self.is_supported: raise errors.UnsupportedRepo(self) if eclass_cache is None: eclass_cache = eclass_cache_mod.cache(pjoin( self.location, 'eclass'), location=self.location) self.eclass_cache = eclass_cache self.masters = masters self.trees = tuple(masters) + (self, ) self.licenses = repo_objs.Licenses(self.location) self.profiles = self.config.profiles if masters: self.licenses = repo_objs.OverlayedLicenses(*self.trees) self.profiles = repo_objs.OverlayedProfiles(*self.trees) # use mirrors from masters if not defined in the repo mirrors = dict(self.thirdpartymirrors) for master in masters: for k, v in master.mirrors.items(): if k not in mirrors: mirrors[k] = v if isinstance(cache, (tuple, list)): cache = tuple(cache) else: cache = (cache, ) self.mirrors = mirrors self.default_mirrors = default_mirrors self.cache = cache self._allow_missing_chksums = allow_missing_manifests self.package_class = self.package_factory(self, cache, self.eclass_cache, self.mirrors, self.default_mirrors) self._shared_pkg_cache = WeakValCache() self._bad_masked = RestrictionRepo(repo_id='bad_masked') self.projects_xml = repo_objs.LocalProjectsXml( pjoin(self.location, 'metadata', 'projects.xml')) repo_id = klass.alias_attr("config.repo_id") repo_name = klass.alias_attr("config.repo_name") aliases = klass.alias_attr("config.aliases") eapi = klass.alias_attr('config.eapi') is_supported = klass.alias_attr('config.eapi.is_supported') pkg_masks = klass.alias_attr('config.pkg_masks') @klass.jit_attr def known_arches(self): """Return all known arches for a repo (including masters).""" return frozenset( chain.from_iterable(r.config.known_arches for r in self.trees)) def path_restrict(self, path): """Return a restriction from a given path in a repo. :param path: full or partial path to an ebuild :return: a package restriction matching the given path if possible :raises ValueError: if the repo doesn't contain the given path, the path relates to a file that isn't an ebuild, or the ebuild isn't in the proper directory layout """ if path not in self: raise ValueError( f"{self.repo_id!r} repo doesn't contain: {path!r}") if not path.startswith(os.sep) and os.path.exists( pjoin(self.location, path)): path_chunks = path.split(os.path.sep) else: path = os.path.realpath(os.path.abspath(path)) relpath = path[len(os.path.realpath(self.location)):].strip('/') path_chunks = relpath.split(os.path.sep) if os.path.isfile(path): if not path.endswith('.ebuild'): raise ValueError(f"file is not an ebuild: {path!r}") elif len(path_chunks) != 3: # ebuild isn't in a category/PN directory raise ValueError( f"ebuild not in the correct directory layout: {path!r}") restrictions = [] # add restrictions until path components run out try: restrictions.append(restricts.RepositoryDep(self.repo_id)) if path_chunks[0] in self.categories: restrictions.append(restricts.CategoryDep(path_chunks[0])) restrictions.append(restricts.PackageDep(path_chunks[1])) base = cpv.VersionedCPV( f"{path_chunks[0]}/{os.path.splitext(path_chunks[2])[0]}") restrictions.append( restricts.VersionMatch('=', base.version, rev=base.revision)) except IndexError: pass return packages.AndRestriction(*restrictions) def __getitem__(self, cpv): cpv_inst = self.package_class(*cpv) if cpv_inst.fullver not in self.versions[(cpv_inst.category, cpv_inst.package)]: raise KeyError(cpv) return cpv_inst def rebind(self, **kwds): """Generate a new tree instance with the same location using new keywords. :param kwds: see __init__ for valid values """ o = self.__class__(self.location, **kwds) o.categories = self.categories o.packages = self.packages o.versions = self.versions return o @klass.jit_attr def thirdpartymirrors(self): mirrors = {} fp = pjoin(self.location, 'profiles', 'thirdpartymirrors') try: for k, v in read_dict(fp, splitter=None).items(): v = v.split() # shuffle mirrors so the same ones aren't used every time shuffle(v) mirrors[k] = v except FileNotFoundError: pass return ImmutableDict(mirrors) @klass.jit_attr def category_dirs(self): try: return frozenset( map( intern, filterfalse(self.false_categories.__contains__, (x for x in listdir_dirs(self.base) if not x.startswith('.'))))) except EnvironmentError as e: logger.error(f"failed listing categories: {e}") return () def _get_categories(self, *optional_category): # why the auto return? current porttrees don't allow/support # categories deeper then one dir. if optional_category: # raise KeyError return () categories = frozenset( chain.from_iterable(repo.config.categories for repo in self.trees)) if categories: return categories return self.category_dirs def _get_packages(self, category): cpath = pjoin(self.base, category.lstrip(os.path.sep)) try: return tuple( filterfalse(self.false_packages.__contains__, listdir_dirs(cpath))) except FileNotFoundError: if category in self.categories: # ignore it, since it's PMS mandated that it be allowed. return () except EnvironmentError as e: category = pjoin(self.base, category.lstrip(os.path.sep)) raise KeyError( f'failed fetching packages for category {category}: {e}' ) from e def _get_versions(self, catpkg): """Determine available versions for a given package. Ebuilds with mismatched or invalid package names are ignored. """ cppath = pjoin(self.base, catpkg[0], catpkg[1]) pkg = f'{catpkg[-1]}-' lp = len(pkg) extension = self.extension ext_len = -len(extension) try: return tuple(x[lp:ext_len] for x in listdir_files(cppath) if x[ext_len:] == extension and x[:lp] == pkg) except EnvironmentError as e: raise KeyError("failed fetching versions for package %s: %s" % (pjoin(self.base, '/'.join(catpkg)), str(e))) from e def _pkg_filter(self, raw, error_callback, pkgs): """Filter packages with bad metadata.""" while True: try: pkg = next(pkgs) except pkg_errors.PackageError as e: # ignore pkgs with invalid CPVs continue except StopIteration: return if raw: yield pkg elif self._bad_masked.has_match( pkg.versioned_atom) and error_callback is not None: error_callback(self._bad_masked[pkg.versioned_atom]) else: # check pkgs for unsupported/invalid EAPIs and bad metadata try: if not pkg.is_supported: exc = pkg_errors.MetadataException( pkg, 'eapi', f"EAPI '{pkg.eapi}' is not supported") self._bad_masked[pkg.versioned_atom] = exc if error_callback is not None: error_callback(exc) continue # TODO: add a generic metadata validation method to avoid slow metadata checks? pkg.data pkg.slot pkg.required_use except pkg_errors.MetadataException as e: self._bad_masked[e.pkg.versioned_atom] = e if error_callback is not None: error_callback(e) continue yield pkg def itermatch(self, *args, **kwargs): raw = 'raw_pkg_cls' in kwargs or not kwargs.get('versioned', True) error_callback = kwargs.pop('error_callback', None) kwargs.setdefault('pkg_filter', partial(self._pkg_filter, raw, error_callback)) return super().itermatch(*args, **kwargs) def _get_ebuild_path(self, pkg): return pjoin(self.base, pkg.category, pkg.package, f"{pkg.package}-{pkg.fullver}{self.extension}") def _get_ebuild_src(self, pkg): return local_source(self._get_ebuild_path(pkg), encoding='utf8') def _get_shared_pkg_data(self, category, package): key = (category, package) o = self._shared_pkg_cache.get(key) if o is None: mxml = self._get_metadata_xml(category, package) manifest = self._get_manifest(category, package) o = repo_objs.SharedPkgData(mxml, manifest) self._shared_pkg_cache[key] = o return o def _get_metadata_xml(self, category, package): return repo_objs.LocalMetadataXml( pjoin(self.base, category, package, "metadata.xml")) def _get_manifest(self, category, package): return digest.Manifest(pjoin(self.base, category, package, "Manifest"), thin=self.config.manifests.thin, enforce_gpg=self.enable_gpg) def _get_digests(self, pkg, allow_missing=False): if self.config.manifests.disabled: return True, {} try: manifest = pkg._shared_pkg_data.manifest manifest.allow_missing = allow_missing return allow_missing, manifest.distfiles except pkg_errors.ParseChksumError as e: if e.missing and allow_missing: return allow_missing, {} raise def __repr__(self): return "<ebuild %s location=%r @%#8x>" % (self.__class__.__name__, self.base, id(self)) @klass.jit_attr def deprecated(self): """Base deprecated packages restriction from profiles/package.deprecated.""" return packages.OrRestriction(*self.config.pkg_deprecated) def _regen_operation_helper(self, **kwds): return _RegenOpHelper(self, force=bool(kwds.get('force', False)), eclass_caching=bool( kwds.get('eclass_caching', True))) def __getstate__(self): d = self.__dict__.copy() del d['_shared_pkg_cache'] return d def __setstate__(self, state): self.__dict__ = state.copy() self.__dict__['_shared_pkg_cache'] = WeakValCache()
class VersionedInstalled(_Base): """Set of packages holding versioned atoms of all installed packages.""" pkgcore_config_type = ConfigHint({'vdb': 'refs:repo'}, typename='pkgset') getter = operator.attrgetter('versioned_atom')
class fetcher(base.fetcher): pkgcore_config_type = ConfigHint( { 'userpriv': 'bool', 'required_chksums': 'list', 'distdir': 'str', 'command': 'str', 'resume_command': 'str' }, allow_unknowns=True) def __init__(self, distdir, command, resume_command=None, required_chksums=None, userpriv=True, attempts=10, readonly=False, **extra_env): """ :param distdir: directory to download files to :type distdir: string :param command: shell command to execute to fetch a file :type command: string :param resume_command: if not None, command to use for resuming- if None, command is reused :param required_chksums: if None, all chksums must be verified, else only chksums listed :type required_chksums: None or sequence :param userpriv: depriv for fetching? :param attempts: max number of attempts before failing the fetch :param readonly: controls whether fetching is allowed """ super().__init__() self.distdir = distdir if required_chksums is not None: required_chksums = [x.lower() for x in required_chksums] else: required_chksums = [] if len(required_chksums) == 1 and required_chksums[0] == "all": self.required_chksums = None else: self.required_chksums = required_chksums def rewrite_command(string): new_command = string.replace("${DISTDIR}", self.distdir) new_command = new_command.replace("$DISTDIR", self.distdir) new_command = new_command.replace("${URI}", "%(URI)s") new_command = new_command.replace("$URI", "%(URI)s") new_command = new_command.replace("${FILE}", "%(FILE)s") new_command = new_command.replace("$FILE", "%(FILE)s") if new_command == string: raise MalformedCommand(string) try: new_command % {"URI": "blah", "FILE": "blah"} except KeyError as k: raise MalformedCommand( f"{command}: unexpected key {k.args[0]}") return new_command self.command = rewrite_command(command) if resume_command is None: self.resume_command = self.command else: self.resume_command = rewrite_command(resume_command) self.attempts = attempts self.userpriv = userpriv self.readonly = readonly self.extra_env = extra_env def fetch(self, target): """Fetch a file. :type target: :obj:`pkgcore.fetch.fetchable` instance :return: None if fetching failed, else on disk location of the copied file """ if not isinstance(target, fetchable): raise TypeError( f"target must be fetchable instance/derivative: {target}") kw = {"mode": 0o775} if self.readonly: kw["mode"] = 0o555 if self.userpriv: kw["gid"] = portage_gid kw["minimal"] = True if not ensure_dirs(self.distdir, **kw): raise errors.DistdirPerms( self.distdir, "if userpriv, uid must be %i, gid must be %i. " "if not readonly, directory must be 0775, else 0555" % (portage_uid, portage_gid)) path = pjoin(self.distdir, target.filename) uris = iter(target.uri) last_exc = RuntimeError("fetching failed for an unknown reason") spawn_opts = {'umask': 0o002, 'env': self.extra_env} if self.userpriv and is_userpriv_capable(): spawn_opts.update({"uid": portage_uid, "gid": portage_gid}) for _attempt in range(self.attempts): try: self._verify(path, target) return path except errors.MissingDistfile as e: command = self.command last_exc = e except errors.ChksumFailure: raise except errors.FetchFailed as e: last_exc = e if not e.resumable: try: os.unlink(path) command = self.command except OSError as e: raise errors.UnmodifiableFile(path, e) from e else: command = self.resume_command # Note we're not even checking the results, the verify portion of # the loop handles this. In other words, don't trust the external # fetcher's exit code, trust our chksums instead. try: spawn_bash( command % { "URI": next(uris), "FILE": target.filename }, **spawn_opts) except StopIteration: raise errors.FetchFailed(target.filename, "ran out of urls to fetch from") else: raise last_exc def get_path(self, fetchable): path = pjoin(self.distdir, fetchable.filename) if self._verify(path, fetchable) is None: return path return None def get_storage_path(self): return self.distdir
class rsync_syncer(base.ExternalSyncer): default_excludes = ['/distfiles', '/local', '/packages'] default_includes = [] default_conn_timeout = 15 default_opts = [ '--recursive', '--delete', '--delete-delay', '--perms', '--times', '--compress', '--force', '--links', '--safe-links', '--stats', '--human-readable', '--timeout=180', '--whole-file', # this one probably shouldn't be a default ] default_retries = 5 binary = "rsync" @classmethod def _parse_uri(cls, raw_uri): if not raw_uri.startswith("rsync://") and \ not raw_uri.startswith("rsync+"): raise base.UriError(raw_uri, "doesn't start with rsync:// nor rsync+") if raw_uri.startswith("rsync://"): return None, raw_uri proto = raw_uri.split(":", 1) proto[0] = proto[0].split("+", 1)[1] cls.require_binary(proto[0]) return proto[0], f"rsync:{proto[1]}" pkgcore_config_type = ConfigHint( { 'basedir': 'str', 'uri': 'str', 'conn_timeout': 'str', 'usersync': 'bool', 'compress': 'bool', 'excludes': 'list', 'includes': 'list', 'retries': 'str', 'opts': 'list', 'extra_opts': 'list', 'proxy': 'str' }, typename='syncer') def __init__(self, basedir, uri, conn_timeout=default_conn_timeout, usersync=False, compress=False, excludes=(), includes=(), retries=default_retries, proxy=None, opts=(), extra_opts=()): uri = uri.rstrip(os.path.sep) + os.path.sep self.rsh, uri = self._parse_uri(uri) super().__init__(basedir, uri, default_verbosity=1, usersync=usersync) self.hostname = self.parse_hostname(self.uri) if self.rsh: self.rsh = self.require_binary(self.rsh) self.opts = list(opts) if opts else list(self.default_opts) self.opts.extend(extra_opts) if compress: self.opts.append("--compress") self.opts.append("--contimeout=%i" % int(conn_timeout)) self.excludes = list(self.default_excludes) + list(excludes) self.includes = list(self.default_includes) + list(includes) self.retries = int(retries) self.use_proxy = proxy is not None if self.use_proxy: self.env['RSYNC_PROXY'] = proxy self.is_ipv6 = "--ipv6" in self.opts or "-6" in self.opts self.is_ipv6 = self.is_ipv6 and socket.has_ipv6 @staticmethod def parse_hostname(uri): return uri[len("rsync://"):].split("@", 1)[-1].split("/", 1)[0] def _get_ips(self): if self.use_proxy: # If we're using a proxy, name resolution is best left to the proxy. yield self.hostname return af_fam = socket.AF_INET if self.is_ipv6: af_fam = socket.AF_INET6 try: for ipaddr in socket.getaddrinfo(self.hostname, None, af_fam, socket.SOCK_STREAM): if ipaddr[0] == socket.AF_INET6: yield f"[{ipaddr[4][0]}]" else: yield ipaddr[4][0] except OSError as e: raise base.SyncError( f"DNS resolution failed for {self.hostname!r}: {e.strerror}") def _sync(self, verbosity, output_fd): fd_pipes = {1: output_fd, 2: output_fd} opts = list(self.opts) if self.rsh: opts.append("-e") opts.append(self.rsh) opts.extend(f"--exclude={x}" for x in self.excludes) opts.extend(f"--include={x}" for x in self.includes) if verbosity < 0: opts.append("--quiet") elif verbosity > 0: opts.extend('-v' for x in range(verbosity)) # zip limits to the shortest iterable ret = None for count, ip in zip(range(self.retries), self._get_ips()): cmd = [ self.binary_path, self.uri.replace(self.hostname, ip, 1), self.basedir ] + opts ret = self._spawn(cmd, fd_pipes) if ret == 0: return True elif ret == 1: raise base.SyncError( "rsync command syntax error: {' '.join(cmd)}") elif ret == 11: raise base.SyncError("rsync ran out of disk space") # need to do something here instead of just restarting... # else: # print(ret) raise base.SyncError("all attempts failed")
class domain(config_domain): # XXX ouch, verify this crap and add defaults and stuff _types = { 'profile': 'ref:profile', 'fetcher': 'ref:fetcher', 'repos': 'lazy_refs:repo', 'vdb': 'lazy_refs:repo', 'name': 'str', } for _thing in ('root', 'config_dir', 'CHOST', 'CBUILD', 'CTARGET', 'CFLAGS', 'PATH', 'PORTAGE_TMPDIR', 'DISTCC_PATH', 'DISTCC_DIR', 'CCACHE_DIR'): _types[_thing] = 'str' # TODO this is missing defaults pkgcore_config_type = ConfigHint( _types, typename='domain', required=['repos', 'profile', 'vdb', 'fetcher', 'name'], allow_unknowns=True) del _types, _thing def __init__(self, profile, repos, vdb, name=None, root='/', config_dir='/etc/portage', prefix='/', *, fetcher, **settings): self.name = name self.root = settings["ROOT"] = root self.config_dir = config_dir self.prefix = prefix self.ebuild_hook_dir = pjoin(self.config_dir, 'env') self.profile = profile self.fetcher = fetcher self.__repos = repos self.__vdb = vdb # prevent critical variables from being changed in make.conf for k in self.profile.profile_only_variables.intersection(settings.keys()): del settings[k] # Protect original settings from being overridden so matching # package.env settings can be overlaid properly. self._settings = ProtectedDict(settings) @load_property("/etc/profile.env", read_func=read_bash_dict) def system_profile(self, data): # prepend system profile $PATH if it exists if 'PATH' in data: path = stable_unique( data['PATH'].split(os.pathsep) + os.environ['PATH'].split(os.pathsep)) os.environ['PATH'] = os.pathsep.join(path) return ImmutableDict(data) @klass.jit_attr_named('_jit_reset_settings', uncached_val=None) def settings(self): settings = self._settings if 'CHOST' in settings and 'CBUILD' not in settings: settings['CBUILD'] = settings['CHOST'] # if unset, MAKEOPTS defaults to CPU thread count if 'MAKEOPTS' not in settings: settings['MAKEOPTS'] = '-j%i' % cpu_count() # reformat env.d and make.conf incrementals system_profile_settings = {} for x in const.incrementals: system_profile_val = self.system_profile.get(x, ()) make_conf_val = settings.get(x, ()) if isinstance(system_profile_val, str): system_profile_val = tuple(system_profile_val.split()) if isinstance(make_conf_val, str): make_conf_val = tuple(make_conf_val.split()) system_profile_settings[x] = system_profile_val settings[x] = make_conf_val # roughly... all incremental stacks should be interpreted left -> right # as such we start with the env.d settings, append profile settings, # and finally append make.conf settings onto that. for k, v in self.profile.default_env.items(): if k not in settings: settings[k] = v continue if k in const.incrementals: settings[k] = system_profile_settings[k] + v + settings[k] # next we finalize incrementals. for incremental in const.incrementals: # Skip USE/ACCEPT_LICENSE for the time being; hack; we need the # negations currently so that pkg iuse induced enablings can be # disabled by negations. For example, think of the profile doing # USE=-cdr for brasero w/ IUSE=+cdr. Similarly, ACCEPT_LICENSE is # skipped because negations are required for license filtering. if incremental not in settings or incremental in ("USE", "ACCEPT_LICENSE"): continue s = set() incremental_expansion( s, settings[incremental], f'while expanding {incremental}') settings[incremental] = tuple(s) if 'ACCEPT_KEYWORDS' not in settings: raise Failure("No ACCEPT_KEYWORDS setting detected from profile, " "or user config") s = set() default_keywords = [] incremental_expansion( s, settings['ACCEPT_KEYWORDS'], 'while expanding ACCEPT_KEYWORDS') default_keywords.extend(s) settings['ACCEPT_KEYWORDS'] = set(default_keywords) # pull trigger options from the env self._triggers = GenerateTriggers(self, settings) return ImmutableDict(settings) @property def arch(self): if "ARCH" not in self.settings: raise Failure("No ARCH setting detected from profile, or user config") return self.settings['ARCH'] @property def stable_arch(self): return self.arch @property def unstable_arch(self): return f"~{self.arch}" @klass.jit_attr_named('_jit_reset_features', uncached_val=None) def features(self): conf_features = list(self.settings.get('FEATURES', ())) env_features = os.environ.get('FEATURES', '').split() return frozenset(optimize_incrementals(conf_features + env_features)) @klass.jit_attr_named('_jit_reset_use', uncached_val=None) def use(self): # append expanded use, FEATURES, and environment defined USE flags use = list(self.settings.get('USE', ())) + list(self.profile.expand_use(self.settings)) # hackish implementation; if test is on, flip on the flag if "test" in self.features: use.append("test") if "prefix" in self.features: use.append("prefix") return frozenset(optimize_incrementals(use + os.environ.get('USE', '').split())) @klass.jit_attr_named('_jit_reset_enabled_use', uncached_val=None) def enabled_use(self): use = ChunkedDataDict() use.add_bare_global(*split_negations(self.use)) use.merge(self.profile.pkg_use) use.update_from_stream(chunked_data(k, *v) for k, v in self.pkg_use) use.freeze() return use @klass.jit_attr_none def forced_use(self): use = ChunkedDataDict() use.merge(getattr(self.profile, 'forced_use')) use.add_bare_global((), (self.arch,)) use.freeze() return use @klass.jit_attr_none def stable_forced_use(self): use = ChunkedDataDict() use.merge(getattr(self.profile, 'stable_forced_use')) use.add_bare_global((), (self.arch,)) use.freeze() return use @load_property("package.mask", parse_func=package_masks) def pkg_masks(self, data, debug=False): if debug: return tuple(data) return tuple(x[0] for x in data) @load_property("package.unmask", parse_func=package_masks) def pkg_unmasks(self, data, debug=False): if debug: return tuple(data) return tuple(x[0] for x in data) # TODO: deprecated, remove in 0.11 @load_property("package.keywords", parse_func=package_keywords_splitter) def pkg_keywords(self, data, debug=False): if debug: return tuple(data) return tuple((x[0], stable_unique(x[1])) for x in data) @load_property("package.accept_keywords", parse_func=package_keywords_splitter) def pkg_accept_keywords(self, data, debug=False): if debug: return tuple(data) return tuple((x[0], stable_unique(x[1])) for x in data) @load_property("package.license", parse_func=package_keywords_splitter) def pkg_licenses(self, data, debug=False): if debug: return tuple(data) return tuple((x[0], stable_unique(x[1])) for x in data) @load_property("package.use", parse_func=package_keywords_splitter) def pkg_use(self, data, debug=False): if debug: return tuple(data) return tuple((x[0], split_negations(stable_unique(x[1]))) for x in data) @load_property("package.env") def pkg_env(self, data, debug=False): func = partial(package_env_splitter, self.ebuild_hook_dir) data = func(data) if debug: return tuple(data) return tuple((x[0], x[1]) for x in data) @klass.jit_attr def bashrcs(self): files = sorted_scan(pjoin(self.config_dir, 'bashrc'), follow_symlinks=True) return tuple(local_source(x) for x in files) def _pkg_filters(self, pkg_accept_keywords=None, pkg_keywords=None): if pkg_accept_keywords is None: pkg_accept_keywords = self.pkg_accept_keywords if pkg_keywords is None: pkg_keywords = self.pkg_keywords # ~amd64 -> [amd64, ~amd64] default_keywords = set([self.arch]) default_keywords.update(self.settings['ACCEPT_KEYWORDS']) for x in self.settings['ACCEPT_KEYWORDS']: if x.startswith("~"): default_keywords.add(x.lstrip("~")) # create keyword filters accept_keywords = ( pkg_keywords + pkg_accept_keywords + self.profile.accept_keywords) filters = [self._make_keywords_filter( default_keywords, accept_keywords, incremental="package.keywords" in const.incrementals)] # add license filters master_license = [] master_license.extend(self.settings.get('ACCEPT_LICENSE', ())) if master_license or self.pkg_licenses: # restrict that matches iff the licenses are allowed restrict = delegate(partial(self._apply_license_filter, master_license)) filters.append(restrict) return tuple(filters) @klass.jit_attr_none def _default_licenses_manager(self): return OverlayedLicenses(*self.source_repos_raw) def _apply_license_filter(self, master_licenses, pkg, mode): """Determine if a package's license is allowed.""" # note we're not honoring mode; it's always match. # reason is that of not turning on use flags to get acceptable license # pairs, maybe change this down the line? matched_pkg_licenses = [] for atom, licenses in self.pkg_licenses: if atom.match(pkg): matched_pkg_licenses += licenses raw_accepted_licenses = master_licenses + matched_pkg_licenses license_manager = getattr(pkg.repo, 'licenses', self._default_licenses_manager) for and_pair in pkg.license.dnf_solutions(): accepted = incremental_expansion_license( pkg, and_pair, license_manager.groups, raw_accepted_licenses, msg_prefix=f"while checking ACCEPT_LICENSE ") if accepted.issuperset(and_pair): return True return False def _make_keywords_filter(self, default_keys, accept_keywords, incremental=False): """Generates a restrict that matches iff the keywords are allowed.""" if not accept_keywords and not self.profile.keywords: return packages.PackageRestriction( "keywords", values.ContainmentMatch2(frozenset(default_keys))) if self.unstable_arch not in default_keys: # stable; thus empty entries == ~arch def f(r, v): if not v: return r, self.unstable_arch return r, v data = collapsed_restrict_to_data( ((packages.AlwaysTrue, default_keys),), (f(*i) for i in accept_keywords)) else: if incremental: f = collapsed_restrict_to_data else: f = non_incremental_collapsed_restrict_to_data data = f(((packages.AlwaysTrue, default_keys),), accept_keywords) if incremental: raise NotImplementedError(self._incremental_apply_keywords_filter) #f = self._incremental_apply_keywords_filter else: f = self._apply_keywords_filter return delegate(partial(f, data)) @staticmethod def _incremental_apply_keywords_filter(data, pkg, mode): # note we ignore mode; keywords aren't influenced by conditionals. # note also, we're not using a restriction here. this is faster. allowed = data.pull_data(pkg) return any(True for x in pkg.keywords if x in allowed) def _apply_keywords_filter(self, data, pkg, mode): # note we ignore mode; keywords aren't influenced by conditionals. # note also, we're not using a restriction here. this is faster. pkg_keywords = pkg.keywords for atom, keywords in self.profile.keywords: if atom.match(pkg): pkg_keywords += keywords allowed = data.pull_data(pkg) if '**' in allowed: return True if "*" in allowed: for k in pkg_keywords: if k[0] not in "-~": return True if "~*" in allowed: for k in pkg_keywords: if k[0] == "~": return True return any(True for x in pkg_keywords if x in allowed) @klass.jit_attr_none def use_expand_re(self): return re.compile( "^(?:[+-])?(%s)_(.*)$" % "|".join(x.lower() for x in self.profile.use_expand)) def _split_use_expand_flags(self, use_stream): stream = ((self.use_expand_re.match(x), x) for x in use_stream) flags, ue_flags = predicate_split(bool, stream, itemgetter(0)) return list(map(itemgetter(1), flags)), [(x[0].groups(), x[1]) for x in ue_flags] def get_package_use_unconfigured(self, pkg, for_metadata=True): """Determine use flags for a given package. Roughly, this should result in the following, evaluated l->r: non USE_EXPAND; profiles, pkg iuse, global configuration, package.use configuration, commandline? stack profiles + pkg iuse; split it into use and use_expanded use; do global configuration + package.use configuration overriding of non-use_expand use if global configuration has a setting for use_expand. Args: pkg: package object for_metadata (bool): if True, we're doing use flag retrieval for metadata generation; otherwise, we're just requesting the raw use flags Returns: Three groups of use flags for the package in the following order: immutable flags, enabled flags, and disabled flags. """ pre_defaults = [x[1:] for x in pkg.iuse if x[0] == '+'] if pre_defaults: pre_defaults, ue_flags = self._split_use_expand_flags(pre_defaults) pre_defaults.extend( x[1] for x in ue_flags if x[0][0].upper() not in self.settings) attr = 'stable_' if self.stable_arch in pkg.keywords \ and self.unstable_arch not in self.settings['ACCEPT_KEYWORDS'] else '' disabled = getattr(self.profile, attr + 'masked_use').pull_data(pkg) immutable = getattr(self, attr + 'forced_use').pull_data(pkg) # lock the configurable use flags to only what's in IUSE, and what's forced # from the profiles (things like userland_GNU and arch) enabled = self.enabled_use.pull_data(pkg, pre_defaults=pre_defaults) # support globs for USE_EXPAND vars use_globs = [u for u in enabled if u.endswith('*')] enabled_use_globs = [] for glob in use_globs: for u in pkg.iuse_stripped: if u.startswith(glob[:-1]): enabled_use_globs.append(u) enabled.difference_update(use_globs) enabled.update(enabled_use_globs) if for_metadata: preserves = pkg.iuse_stripped enabled.intersection_update(preserves) enabled.update(immutable) enabled.difference_update(disabled) return immutable, enabled, disabled def get_package_domain(self, pkg): """Get domain object with altered settings from matching package.env entries.""" if getattr(pkg, '_domain', None) is not None: return pkg._domain files = [] for restrict, paths in self.pkg_env: if restrict.match(pkg): files.extend(paths) if files: pkg_settings = dict(self._settings.orig.items()) for path in files: PortageConfig.load_make_conf( pkg_settings, path, allow_sourcing=True, allow_recurse=False, incrementals=True) # TODO: Improve pkg domain vs main domain proxying, e.g. static # jitted attrs should always be generated and pulled from the main # domain obj; however, currently each pkg domain instance gets its # own copy so values collapsed on the pkg domain instance aren't # propagated back to the main domain leading to regen per pkg if # requested. pkg_domain = copy.copy(self) pkg_domain._settings = ProtectedDict(pkg_settings) # reset jitted attrs that can pull updated settings for attr in (x for x in dir(self) if x.startswith('_jit_reset_')): setattr(pkg_domain, attr, None) # store altered domain on the pkg obj to avoid recreating pkg domain object.__setattr__(pkg, "_domain", pkg_domain) return pkg_domain return self def get_package_bashrcs(self, pkg): for source in self.profile.bashrcs: yield source for source in self.bashrcs: yield source if not os.path.exists(self.ebuild_hook_dir): return # matching portage behavior... it's whacked. base = pjoin(self.ebuild_hook_dir, pkg.category) dirs = ( pkg.package, f"{pkg.package}:{pkg.slot}", getattr(pkg, "P", None), getattr(pkg, "PF", None), ) for fp in filter(None, dirs): fp = pjoin(base, fp) if os.path.exists(fp): yield local_source(fp) def _wrap_repo(self, repo, filtered=True): """Create a filtered, wrapped repo object for the domain.""" wrapped_repo = self._configure_repo(repo) if filtered: wrapped_repo = self.filter_repo(wrapped_repo) return wrapped_repo def add_repo(self, path, config, name=None, configure=True): """Add an external repo to the domain.""" path = os.path.abspath(path) if name is None: # parse repo id from the given path name = RepoConfig(path).repo_id if name in self.source_repos_raw: # fallback to using path for repo id in case of duplicate repos name = path if name in self.source_repos_raw: raise ValueError(f'{name!r} repo already configured') repo_config = RepoConfig(path, config_name=name) kwargs = {} if repo_config.cache_format is not None: # default to using md5 cache kwargs['cache'] = (md5_cache(path),) repo_obj = ebuild_repo.tree(config, repo_config, **kwargs) # TODO: reset related jit attrs self.source_repos_raw += repo_obj if configure: return self._wrap_repo(repo_obj) return repo_obj def find_repo(self, path, config, configure=True): """Find and add an external repo to the domain given a path.""" repo = None path = os.path.abspath(path) with suppress_logging(): while path != self.root: try: repo = self.add_repo(path, config=config, configure=configure) break except repo_errors.InitializationError: path = os.path.dirname(path) return repo def _configure_repo(self, repo): """Configure a raw repo.""" configured_repo = repo if not repo.configured: pargs = [repo] try: for x in repo.configurables: if x == "domain": pargs.append(self) elif x == "settings": pargs.append(self.settings) elif x == "profile": pargs.append(self.profile) else: pargs.append(getattr(self, x)) except AttributeError as e: raise Failure( f"failed configuring repo {repo!r}: " f"configurable missing: {e}") from e configured_repo = repo.configure(*pargs) return configured_repo def filter_repo(self, repo, pkg_masks=None, pkg_unmasks=None, pkg_filters=None, pkg_accept_keywords=None, pkg_keywords=None, profile=True): """Filter a configured repo.""" if pkg_masks is None: pkg_masks = self.pkg_masks if pkg_unmasks is None: pkg_unmasks = self.pkg_unmasks if pkg_filters is None: pkg_filters = self._pkg_filters(pkg_accept_keywords, pkg_keywords) global_masks = [((), repo.pkg_masks)] if profile: global_masks.extend(self.profile._incremental_masks) masks = set() for neg, pos in global_masks: masks.difference_update(neg) masks.update(pos) masks.update(pkg_masks) unmasks = set() if profile: for neg, pos in self.profile._incremental_unmasks: unmasks.difference_update(neg) unmasks.update(pos) unmasks.update(pkg_unmasks) filters = generate_filter(masks, unmasks, *pkg_filters) return filtered.tree(repo, filters, True) @klass.jit_attr_named('_jit_reset_tmpdir', uncached_val=None) def tmpdir(self): """Temporary directory for the system. Uses PORTAGE_TMPDIR setting and falls back to using the system's TMPDIR if unset. """ path = self.settings.get('PORTAGE_TMPDIR', '') if not os.path.exists(path): try: os.mkdir(path) except EnvironmentError: path = tempfile.gettempdir() logger.warning(f'nonexistent PORTAGE_TMPDIR path, defaulting to {path!r}') return os.path.normpath(path) @property def pm_tmpdir(self): """Temporary directory for the package manager.""" return pjoin(self.tmpdir, 'portage') @property def repo_configs(self): """All defined repo configs.""" return tuple(r.config for r in self.repos if hasattr(r, 'config')) @klass.jit_attr def KV(self): """The version of the running kernel.""" ret, version = spawn_get_output(['uname', '-r']) if ret == 0: return version[0].strip() raise ValueError('unknown kernel version') @klass.jit_attr_none def source_repos_raw(self): """Group of package repos without filtering.""" repos = [] for r in self.__repos: try: repo = r.instantiate() except config_errors.InstantiationError as e: # roll back the exception chain to a meaningful error message exc = find_user_exception(e) if exc is None: exc = e logger.warning(f'skipping {r.name!r} repo: {exc}') continue if not repo.is_supported: logger.warning( f'skipping {r.name!r} repo: unsupported EAPI {str(repo.eapi)!r}') continue repos.append(repo) return RepositoryGroup(repos) @klass.jit_attr_none def installed_repos_raw(self): """Group of installed repos without filtering.""" repos = [r.instantiate() for r in self.__vdb] if self.profile.provides_repo is not None: repos.append(self.profile.provides_repo) return RepositoryGroup(repos) @klass.jit_attr_none def repos_raw(self): """Group of all repos without filtering.""" return RepositoryGroup( chain(self.source_repos_raw, self.installed_repos_raw)) @klass.jit_attr_none def source_repos(self): """Group of configured, filtered package repos.""" repos = [] for repo in self.source_repos_raw: try: repos.append(self._wrap_repo(repo, filtered=True)) except repo_errors.RepoError as e: logger.warning(f'skipping {repo.repo_id!r} repo: {e}') return RepositoryGroup(repos) @klass.jit_attr_none def installed_repos(self): """Group of configured, installed package repos.""" repos = [] for repo in self.installed_repos_raw: try: repos.append(self._wrap_repo(repo, filtered=False)) except repo_errors.RepoError as e: logger.warning(f'skipping {repo.repo_id!r} repo: {e}') return RepositoryGroup(repos) @klass.jit_attr_none def unfiltered_repos(self): """Group of all configured repos without filtering.""" repos = chain(self.source_repos, self.installed_repos) return RepositoryGroup( (r.raw_repo if r.raw_repo is not None else r) for r in repos) @klass.jit_attr_none def repos(self): """Group of all repos.""" return RepositoryGroup( chain(self.source_repos, self.installed_repos)) @klass.jit_attr_none def ebuild_repos(self): """Group of all ebuild repos bound with configuration data.""" return RepositoryGroup( x for x in self.source_repos if isinstance(x.raw_repo, ebuild_repo.ConfiguredTree)) @klass.jit_attr_none def ebuild_repos_unfiltered(self): """Group of all ebuild repos without package filtering.""" return RepositoryGroup( x for x in self.unfiltered_repos if isinstance(x, ebuild_repo.ConfiguredTree)) @klass.jit_attr_none def ebuild_repos_raw(self): """Group of all ebuild repos without filtering.""" return RepositoryGroup( x for x in self.source_repos_raw if isinstance(x, ebuild_repo.UnconfiguredTree)) @klass.jit_attr_none def binary_repos(self): """Group of all binary repos bound with configuration data.""" return RepositoryGroup( x for x in self.source_repos if isinstance(x.raw_repo, binary_repo.ConfiguredTree)) @klass.jit_attr_none def binary_repos_unfiltered(self): """Group of all binary repos without package filtering.""" return RepositoryGroup( x for x in self.unfiltered_repos if isinstance(x, binary_repo.ConfiguredTree)) @klass.jit_attr_none def binary_repos_raw(self): """Group of all binary repos without filtering.""" return RepositoryGroup( x for x in self.source_repos_raw if isinstance(x, binary_repo.tree)) # multiplexed repos all_repos = klass.alias_attr("repos.combined") all_repos_raw = klass.alias_attr("repos_raw.combined") all_source_repos = klass.alias_attr("source_repos.combined") all_source_repos_raw = klass.alias_attr("source_repos_raw.combined") all_installed_repos = klass.alias_attr("installed_repos.combined") all_installed_repos_raw = klass.alias_attr("installed_repos_raw.combined") all_unfiltered_repos = klass.alias_attr("unfiltered_repos.combined") all_ebuild_repos = klass.alias_attr("ebuild_repos.combined") all_ebuild_repos_unfiltered = klass.alias_attr("ebuild_repos_unfiltered.combined") all_ebuild_repos_raw = klass.alias_attr("ebuild_repos_raw.combined") all_binary_repos = klass.alias_attr("binary_repos.combined") all_binary_repos_unfiltered = klass.alias_attr("binary_repos_unfiltered.combined") all_binary_repos_raw = klass.alias_attr("binary_repos_raw.combined")