class TestWeakValCache(TestCase): if WeakValueDictionary is WeakValCache: skip = "WeakValCache is weakref.WeakValueDictionary; indicates " \ "snakeoil._caching isn't compiled" def setUp(self): self.o = RefObj() self.w = WeakValCache() def test_setitem(self): s = "asdf" self.w[s] = self.o self.w["fds"] = self.o self.w[s] = self.o def test_getitem(self): s = "asdf" self.w[s] = self.o self.assertIdentical(self.w[s], self.o) def test_expiring(self): s = "asdf" self.w[s] = self.o self.assertTrue(self.w[s]) del self.o self.assertRaises(KeyError, self.w.__getitem__, s) def test_get(self): s = "asdf" self.assertRaises(KeyError, self.w.__getitem__, s) self.w[s] = self.o self.assertIdentical(self.w.get(s), self.o) def test_keys(self): self.assertEqual(list(self.w.keys()), []) self.w['a'] = self.o self.w['b'] = self.o self.w['c'] = self.o self.assertEqual(sorted(self.w.keys()), ['a', 'b', 'c']) del self.o self.assertEqual(sorted(self.w.keys()), []) def test_values(self): self.assertEqual(self.w.values(), []) self.w['a'] = self.o self.w['b'] = self.o self.w['c'] = self.o self.assertEqual(len(self.w.values()), 3) del self.o self.assertEqual(sorted(self.w.values()), []) def test_items(self): self.assertEqual(self.w.items(), []) self.w['a'] = self.o self.w['b'] = self.o self.w['c'] = self.o self.assertEqual(len(self.w.items()), 3) del self.o self.assertEqual(sorted(self.w.items()), [])
class TestWeakValCache(TestCase): if WeakValueDictionary is WeakValCache: skip = "WeakValCache is weakref.WeakValueDictionary; indicates " \ "snakeoil._caching isn't compiled" def setUp(self): self.o = RefObj() self.w = WeakValCache() def test_setitem(self): s = "asdf" self.w[s] = self.o self.w["fds"] = self.o self.w[s] = self.o def test_getitem(self): s = "asdf" self.w[s] = self.o self.assertIdentical(self.w[s], self.o) def test_expiring(self): s = "asdf" self.w[s] = self.o self.assertTrue(self.w[s]) del self.o self.assertRaises(KeyError, self.w.__getitem__, s) def test_get(self): s = "asdf" self.assertRaises(KeyError, self.w.__getitem__, s) self.w[s] = self.o self.assertIdentical(self.w.get(s), self.o)
def __init__(self, location, eclass_cache, cache=(), default_mirrors=None, ignore_paludis_versioning=False, allow_missing_manifests=False, repo_config=None): """ :param location: on disk location of the tree :param cache: sequence of :obj:`pkgcore.cache.template.database` instances to use for storing metadata :param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache` instance representing the eclasses available, if None, generates the eclass_cache itself :param default_mirrors: Either None, or sequence of mirrors to try fetching from first, then falling back to other uri :param ignore_paludis_versioning: If False, fail when -scm is encountred. if True, silently ignore -scm ebuilds. """ prototype.tree.__init__(self) if repo_config is None: repo_config = repo_objs.RepoConfig(location) self.config = repo_config self.base = self.location = location try: if not stat.S_ISDIR(os.stat(self.base).st_mode): raise errors.InitializationError( "base not a dir: %s" % self.base) except OSError: raise_from(errors.InitializationError( "lstat failed on base %s" % (self.base,))) self.eclass_cache = eclass_cache self.licenses = repo_objs.Licenses(location) fp = pjoin(self.base, metadata_offset, "thirdpartymirrors") mirrors = {} try: for k, v in read_dict(fp, splitter=None).iteritems(): v = v.split() shuffle(v) mirrors[k] = v except EnvironmentError as ee: if ee.errno != errno.ENOENT: raise if isinstance(cache, (tuple, list)): cache = tuple(cache) else: cache = (cache,) self.mirrors = mirrors self.default_mirrors = default_mirrors self.cache = cache self.ignore_paludis_versioning = ignore_paludis_versioning self._allow_missing_chksums = allow_missing_manifests self.package_class = self.package_factory( self, cache, self.eclass_cache, self.mirrors, self.default_mirrors) self._shared_pkg_cache = WeakValCache()
class base(object): """ Maintains the cache information about eclasses available to an ebuild. """ def __init__(self, location=None, eclassdir=None): self._eclass_data_inst_cache = WeakValCache() # generate this. # self.eclasses = {} # {"Name": ("location", "_mtime_")} self.location = location self.eclassdir = eclassdir def get_eclass_data(self, inherits): """Return the cachable entries from a list of inherited eclasses. Only make get_eclass_data calls for data you know came from this eclass_cache, otherwise be ready to catch a KeyError exception for any eclass that was requested, but not known to this cache. """ keys = tuple(sorted(inherits)) o = self._eclass_data_inst_cache.get(keys) if o is None: o = ImmutableDict((k, self.eclasses[k]) for k in keys) self._eclass_data_inst_cache[keys] = o return o def get_eclass(self, eclass): o = self.eclasses.get(eclass) if o is None: return None return local_source(o.path) eclasses = jit_attr_ext_method("_load_eclasses", "_eclasses") def rebuild_cache_entry(self, entry_eclasses): """Check if eclass data is still valid. Given a dict as returned by get_eclass_data, walk it comparing it to internal eclass view. :return: a boolean representing whether that eclass data is still up to date, or not """ ec = self.eclasses d = {} for eclass, chksums in entry_eclasses: data = ec.get(eclass) if any(val != getattr(data, chf, None) for chf, val in chksums): return None d[eclass] = data return d
class base(object): """ Maintains the cache information about eclasses available to an ebuild. """ def __init__(self, portdir=None, eclassdir=None): self._eclass_data_inst_cache = WeakValCache() # generate this. # self.eclasses = {} # {"Name": ("location", "_mtime_")} self.portdir = portdir self.eclassdir = eclassdir def get_eclass_data(self, inherits): """Return the cachable entries from a list of inherited eclasses. Only make get_eclass_data calls for data you know came from this eclass_cache, otherwise be ready to catch a KeyError exception for any eclass that was requested, but not known to this cache. """ keys = tuple(sorted(inherits)) o = self._eclass_data_inst_cache.get(keys) if o is None: o = ImmutableDict((k, self.eclasses[k]) for k in keys) self._eclass_data_inst_cache[keys] = o return o def get_eclass(self, eclass): o = self.eclasses.get(eclass) if o is None: return None return local_source(o.path) eclasses = jit_attr_ext_method("_load_eclasses", "_eclasses") def rebuild_cache_entry(self, entry_eclasses): """Check if eclass data is still valid. Given a dict as returned by get_eclass_data, walk it comparing it to internal eclass view. :return: a boolean representing whether that eclass data is still up to date, or not """ ec = self.eclasses d = {} for eclass, chksums in entry_eclasses: data = ec.get(eclass) if any(val != getattr(data, chf, None) for chf, val in chksums): return None d[eclass] = data return d
class _UnconfiguredTree(prototype.tree): """ raw implementation supporting standard ebuild tree. return packages don't have USE configuration bound to them. """ false_packages = frozenset(["CVS", ".svn"]) false_categories = frozenset([ "eclass", "profiles", "packages", "distfiles", "metadata", "licenses", "scripts", "CVS", "local" ]) configured = False configurables = ("domain", "settings") configure = None package_factory = staticmethod(ebuild_src.generate_new_factory) # This attributes needs to be replaced/removed; it's a hack for pmerge. repository_type = 'source' enable_gpg = False extension = '.ebuild' operations_kls = repo_operations pkgcore_config_type = ConfigHint( { 'location': 'str', 'cache': 'refs:cache', 'eclass_cache': 'ref:eclass_cache', 'default_mirrors': 'list', 'override_repo_id': 'str', 'ignore_paludis_versioning': 'bool', 'allow_missing_manifests': 'bool', 'repo_config': 'ref:raw_repo', }, typename='repo') def __init__(self, location, eclass_cache, cache=(), default_mirrors=None, override_repo_id=None, ignore_paludis_versioning=False, allow_missing_manifests=False, repo_config=None): """ :param location: on disk location of the tree :param cache: sequence of :obj:`pkgcore.cache.template.database` instances to use for storing metadata :param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache` instance representing the eclasses available, if None, generates the eclass_cache itself :param default_mirrors: Either None, or sequence of mirrors to try fetching from first, then falling back to other uri :param override_repo_id: Either None, or string to force as the repository unique id :param ignore_paludis_versioning: If False, fail when -scm is encountred. if True, silently ignore -scm ebuilds. """ prototype.tree.__init__(self) if repo_config is None: repo_config = repo_objs.RepoConfig(location) self.config = repo_config self._repo_id = override_repo_id self.base = self.location = location try: if not stat.S_ISDIR(os.stat(self.base).st_mode): raise errors.InitializationError("base not a dir: %s" % self.base) except OSError: raise_from( errors.InitializationError("lstat failed on base %s" % (self.base, ))) self.eclass_cache = eclass_cache self.licenses = repo_objs.Licenses(location) fp = pjoin(self.base, metadata_offset, "thirdpartymirrors") mirrors = {} try: for k, v in read_dict(fp, splitter=None).iteritems(): v = v.split() shuffle(v) mirrors[k] = v except EnvironmentError as ee: if ee.errno != errno.ENOENT: raise if isinstance(cache, (tuple, list)): cache = tuple(cache) else: cache = (cache, ) self.mirrors = mirrors self.default_mirrors = default_mirrors self.cache = cache self.ignore_paludis_versioning = ignore_paludis_versioning self._allow_missing_chksums = allow_missing_manifests self.package_class = self.package_factory(self, cache, self.eclass_cache, self.mirrors, self.default_mirrors) self._shared_pkg_cache = WeakValCache() repo_id = klass.alias_attr("config.repo_id") def __getitem__(self, cpv): cpv_inst = self.package_class(*cpv) if cpv_inst.fullver not in self.versions[(cpv_inst.category, cpv_inst.package)]: if cpv_inst.revision is None: if '%s-r0' % cpv_inst.fullver in \ self.versions[(cpv_inst.category, cpv_inst.package)]: # ebuild on disk has an explicit -r0 in it's name return cpv_inst raise KeyError(cpv) return cpv_inst def rebind(self, **kwds): """ generate a new tree instance with the same location using new keywords. :param kwds: see __init__ for valid values """ o = self.__class__(self.location, **kwds) o.categories = self.categories o.packages = self.packages o.versions = self.versions return o @klass.jit_attr def hardcoded_categories(self): # try reading $LOC/profiles/categories if it's available. cats = readlines(pjoin(self.base, 'profiles', 'categories'), True, True, True) if cats is not None: cats = tuple(imap(intern, cats)) return cats def _get_categories(self, *optional_category): # why the auto return? current porttrees don't allow/support # categories deeper then one dir. if optional_category: #raise KeyError return () cats = self.hardcoded_categories if cats is not None: return cats try: return tuple( imap( intern, ifilterfalse( self.false_categories.__contains__, (x for x in listdir_dirs(self.base) if x[0:1] != ".")))) except EnvironmentError as e: raise_from(KeyError("failed fetching categories: %s" % str(e))) def _get_packages(self, category): cpath = pjoin(self.base, category.lstrip(os.path.sep)) try: return tuple( ifilterfalse(self.false_packages.__contains__, listdir_dirs(cpath))) except EnvironmentError as e: if e.errno == errno.ENOENT: if self.hardcoded_categories and category in self.hardcoded_categories or \ isinstance(self, _SlavedTree) and category in self.parent_repo.categories: # ignore it, since it's PMS mandated that it be allowed. return () raise_from(KeyError("failed fetching packages for category %s: %s" % \ (pjoin(self.base, category.lstrip(os.path.sep)), \ str(e)))) def _get_versions(self, catpkg): cppath = pjoin(self.base, catpkg[0], catpkg[1]) pkg = catpkg[-1] + "-" lp = len(pkg) extension = self.extension ext_len = -len(extension) try: ret = tuple(x[lp:ext_len] for x in listdir_files(cppath) if x[ext_len:] == extension and x[:lp] == pkg) if any(('scm' in x or '-try' in x) for x in ret): if not self.ignore_paludis_versioning: for x in ret: if 'scm' in x: raise ebuild_errors.InvalidCPV( "%s/%s-%s has nonstandard -scm " "version component" % (catpkg + (x, ))) elif 'try' in x: raise ebuild_errors.InvalidCPV( "%s/%s-%s has nonstandard -try " "version component" % (catpkg + (x, ))) raise AssertionError('unreachable codepoint was reached') return tuple(x for x in ret if ('scm' not in x and 'try' not in x)) return ret except EnvironmentError as e: raise_from(KeyError("failed fetching versions for package %s: %s" % \ (pjoin(self.base, catpkg.lstrip(os.path.sep)), str(e)))) def _get_ebuild_path(self, pkg): if pkg.revision is None: if pkg.fullver not in self.versions[(pkg.category, pkg.package)]: # daft explicit -r0 on disk. return pjoin( self.base, pkg.category, pkg.package, "%s-%s-r0%s" % (pkg.package, pkg.fullver, self.extension)) return pjoin(self.base, pkg.category, pkg.package, \ "%s-%s%s" % (pkg.package, pkg.fullver, self.extension)) def _get_ebuild_src(self, pkg): return local_source(self._get_ebuild_path(pkg), encoding='utf8') def _get_shared_pkg_data(self, category, package): key = (category, package) o = self._shared_pkg_cache.get(key) if o is None: mxml = self._get_metadata_xml(category, package) manifest = self._get_manifest(category, package) o = repo_objs.SharedPkgData(mxml, manifest) self._shared_pkg_cache[key] = o return o def _get_metadata_xml(self, category, package): return repo_objs.LocalMetadataXml( pjoin(self.base, category, package, "metadata.xml")) def _get_manifest(self, category, package): return digest.Manifest(pjoin(self.base, category, package, "Manifest"), thin=self.config.manifests.thin, enforce_gpg=self.enable_gpg) def _get_digests(self, pkg, allow_missing=False): if self.config.manifests.disabled: return True, {} try: manifest = pkg._shared_pkg_data.manifest return allow_missing, manifest.distfiles except pkg_errors.ParseChksumError as e: if e.missing and allow_missing: return allow_missing, {} raise def __str__(self): return "%s.%s: location %s" % (self.__class__.__module__, self.__class__.__name__, self.base) def __repr__(self): return "<ebuild %s location=%r @%#8x>" % (self.__class__.__name__, self.base, id(self)) def _visibility_limiters(self): path = pjoin(self.base, 'profiles', 'package.mask') pos, neg = [], [] try: if self.config.profile_format not in ['pms', 'portage-2']: paths = sorted(x.location for x in iter_scan(path) if x.is_reg) else: paths = [path] for path in paths: for line in iter_read_bash(path): line = line.strip() if line in ('-', ''): raise profiles.ProfileError( pjoin(self.base, 'profiles'), 'package.mask', "encountered empty negation: -") if line.startswith('-'): neg.append(atom.atom(line[1:])) else: pos.append(atom.atom(line)) except IOError as i: if i.errno != errno.ENOENT: raise except ebuild_errors.MalformedAtom as ma: raise_from( profiles.ProfileError(pjoin(self.base, 'profiles'), 'package.mask', ma)) return [neg, pos] def _regen_operation_helper(self, **kwds): return _RegenOpHelper(self, force=bool(kwds.get('force', False)), eclass_caching=bool( kwds.get('eclass_caching', True)))
class _UnconfiguredTree(prototype.tree): """Raw implementation supporting standard ebuild tree. Return packages don't have USE configuration bound to them. """ false_packages = frozenset(["CVS", ".svn"]) false_categories = frozenset([ "eclass", "profiles", "packages", "distfiles", "metadata", "licenses", "scripts", "CVS", "local"]) configured = False configurables = ("domain", "settings") configure = None package_factory = staticmethod(ebuild_src.generate_new_factory) enable_gpg = False extension = '.ebuild' operations_kls = repo_operations pkgcore_config_type = ConfigHint({ 'location': 'str', 'eclass_cache': 'ref:eclass_cache', 'masters': 'refs:repo', 'cache': 'refs:cache', 'default_mirrors': 'list', 'ignore_paludis_versioning': 'bool', 'allow_missing_manifests': 'bool', 'repo_config': 'ref:repo_config', }, typename='repo') def __init__(self, location, eclass_cache=None, masters=(), cache=(), default_mirrors=None, ignore_paludis_versioning=False, allow_missing_manifests=False, repo_config=None): """ :param location: on disk location of the tree :param cache: sequence of :obj:`pkgcore.cache.template.database` instances to use for storing metadata :param masters: repo masters this repo inherits from :param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache` instance representing the eclasses available, if None, generates the eclass_cache itself :param default_mirrors: Either None, or sequence of mirrors to try fetching from first, then falling back to other uri :param ignore_paludis_versioning: If False, fail when -scm is encountered. if True, silently ignore -scm ebuilds. """ prototype.tree.__init__(self) self.base = self.location = location try: if not stat.S_ISDIR(os.stat(self.base).st_mode): raise errors.InitializationError( "base not a dir: %s" % self.base) except OSError: raise_from(errors.InitializationError( "lstat failed on base %s" % (self.base,))) if repo_config is None: repo_config = repo_objs.RepoConfig(location) self.config = repo_config if eclass_cache is None: eclass_cache = eclass_cache_module.cache(pjoin(self.location, 'eclass')) self.eclass_cache = eclass_cache self.masters = masters self.trees = tuple(masters) + (self,) self.licenses = repo_objs.Licenses(self.location) if masters: self.licenses = repo_objs.OverlayedLicenses(*self.trees) mirrors = {} fp = pjoin(self.location, metadata_offset, "thirdpartymirrors") try: for k, v in read_dict(fp, splitter=None).iteritems(): v = v.split() shuffle(v) mirrors[k] = v except EnvironmentError as ee: if ee.errno != errno.ENOENT: raise # use mirrors from masters if not defined in the repo for master in masters: for k, v in master.mirrors.iteritems(): if k not in mirrors: mirrors[k] = v if isinstance(cache, (tuple, list)): cache = tuple(cache) else: cache = (cache,) self.mirrors = mirrors self.default_mirrors = default_mirrors self.cache = cache self.ignore_paludis_versioning = ignore_paludis_versioning self._allow_missing_chksums = allow_missing_manifests self.package_class = self.package_factory( self, cache, self.eclass_cache, self.mirrors, self.default_mirrors) self._shared_pkg_cache = WeakValCache() repo_id = klass.alias_attr("config.repo_id") def path_restrict(self, path): """Return a restriction from a given path in a repo. :param path: full or partial path to an ebuild :return: a package restriction matching the given path if possible """ realpath = os.path.realpath(path) if realpath not in self: raise ValueError("'%s' repo doesn't contain: '%s'" % (self.repo_id, path)) relpath = realpath[len(os.path.realpath(self.location)):].strip('/') repo_path = relpath.split(os.path.sep) if relpath else [] restrictions = [] if os.path.isfile(realpath): if not path.endswith('.ebuild'): raise ValueError("file is not an ebuild: '%s'" % (path,)) elif len(repo_path) != 3: # ebuild isn't in a category/PN directory raise ValueError("ebuild not in the correct directory layout: '%s'" % (path,)) # add restrictions until path components run out try: restrictions.append(restricts.RepositoryDep(self.repo_id)) if repo_path[0] in self.categories: restrictions.append(restricts.CategoryDep(repo_path[0])) restrictions.append(restricts.PackageDep(repo_path[1])) base = cpv.versioned_CPV("%s/%s" % (repo_path[0], os.path.splitext(repo_path[2])[0])) restrictions.append(restricts.VersionMatch('=', base.version, rev=base.revision)) except IndexError: pass return packages.AndRestriction(*restrictions) def __getitem__(self, cpv): cpv_inst = self.package_class(*cpv) if cpv_inst.fullver not in self.versions[(cpv_inst.category, cpv_inst.package)]: if cpv_inst.revision is None: if '%s-r0' % cpv_inst.fullver in \ self.versions[(cpv_inst.category, cpv_inst.package)]: # ebuild on disk has an explicit -r0 in its name return cpv_inst raise KeyError(cpv) return cpv_inst def rebind(self, **kwds): """Generate a new tree instance with the same location using new keywords. :param kwds: see __init__ for valid values """ o = self.__class__(self.location, **kwds) o.categories = self.categories o.packages = self.packages o.versions = self.versions return o @klass.jit_attr def hardcoded_categories(self): # try reading $LOC/profiles/categories if it's available. categories = readlines( pjoin(self.base, 'profiles', 'categories'), True, True, True) if categories is not None: categories = tuple(imap(intern, categories)) return categories def _get_categories(self, *optional_category): # why the auto return? current porttrees don't allow/support # categories deeper then one dir. if optional_category: # raise KeyError return () categories = set() for repo in self.trees: if repo.hardcoded_categories is not None: categories.update(repo.hardcoded_categories) if categories: return tuple(categories) try: return tuple(imap(intern, ifilterfalse( self.false_categories.__contains__, (x for x in listdir_dirs(self.base) if x[0:1] != ".")))) except EnvironmentError as e: raise_from(KeyError("failed fetching categories: %s" % str(e))) def _get_packages(self, category): cpath = pjoin(self.base, category.lstrip(os.path.sep)) try: return tuple(ifilterfalse( self.false_packages.__contains__, listdir_dirs(cpath))) except EnvironmentError as e: if e.errno == errno.ENOENT: if category in self.categories: # ignore it, since it's PMS mandated that it be allowed. return () raise_from(KeyError( "failed fetching packages for category %s: %s" % (pjoin(self.base, category.lstrip(os.path.sep)), str(e)))) def _get_versions(self, catpkg): cppath = pjoin(self.base, catpkg[0], catpkg[1]) pkg = catpkg[-1] + "-" lp = len(pkg) extension = self.extension ext_len = -len(extension) try: ret = tuple(x[lp:ext_len] for x in listdir_files(cppath) if x[ext_len:] == extension and x[:lp] == pkg) if any(('scm' in x or '-try' in x) for x in ret): if not self.ignore_paludis_versioning: for x in ret: if 'scm' in x: raise ebuild_errors.InvalidCPV( "%s/%s-%s has nonstandard -scm " "version component" % (catpkg + (x,))) elif 'try' in x: raise ebuild_errors.InvalidCPV( "%s/%s-%s has nonstandard -try " "version component" % (catpkg + (x,))) raise AssertionError('unreachable codepoint was reached') return tuple(x for x in ret if ('scm' not in x and 'try' not in x)) return ret except EnvironmentError as e: raise_from(KeyError( "failed fetching versions for package %s: %s" % (pjoin(self.base, '/'.join(catpkg)), str(e)))) def _get_ebuild_path(self, pkg): if pkg.revision is None: if pkg.fullver not in self.versions[(pkg.category, pkg.package)]: # daft explicit -r0 on disk. return pjoin( self.base, pkg.category, pkg.package, "%s-%s-r0%s" % (pkg.package, pkg.fullver, self.extension)) return pjoin( self.base, pkg.category, pkg.package, "%s-%s%s" % (pkg.package, pkg.fullver, self.extension)) def _get_ebuild_src(self, pkg): return local_source(self._get_ebuild_path(pkg), encoding='utf8') def _get_shared_pkg_data(self, category, package): key = (category, package) o = self._shared_pkg_cache.get(key) if o is None: mxml = self._get_metadata_xml(category, package) manifest = self._get_manifest(category, package) o = repo_objs.SharedPkgData(mxml, manifest) self._shared_pkg_cache[key] = o return o def _get_metadata_xml(self, category, package): return repo_objs.LocalMetadataXml(pjoin( self.base, category, package, "metadata.xml")) def _get_manifest(self, category, package): return digest.Manifest(pjoin( self.base, category, package, "Manifest"), thin=self.config.manifests.thin, enforce_gpg=self.enable_gpg) def _get_digests(self, pkg, allow_missing=False): if self.config.manifests.disabled: return True, {} try: manifest = pkg._shared_pkg_data.manifest manifest.allow_missing = allow_missing return allow_missing, manifest.distfiles except pkg_errors.ParseChksumError as e: if e.missing and allow_missing: return allow_missing, {} raise def __repr__(self): return "<ebuild %s location=%r @%#8x>" % ( self.__class__.__name__, self.base, id(self)) def _visibility_limiters(self): path = pjoin(self.base, 'profiles', 'package.mask') pos, neg = [], [] try: if self.config.profile_formats.intersection(['portage-1', 'portage-2']): paths = sorted_scan(path) else: paths = [path] for path in paths: for line in iter_read_bash(path): line = line.strip() if line in ('-', ''): raise profiles.ProfileError( pjoin(self.base, 'profiles'), 'package.mask', "encountered empty negation: -") if line.startswith('-'): neg.append(atom.atom(line[1:])) else: pos.append(atom.atom(line)) except IOError as i: if i.errno != errno.ENOENT: raise except ebuild_errors.MalformedAtom as ma: raise_from(profiles.ProfileError( pjoin(self.base, 'profiles'), 'package.mask', ma)) return [neg, pos] def _regen_operation_helper(self, **kwds): return _RegenOpHelper( self, force=bool(kwds.get('force', False)), eclass_caching=bool(kwds.get('eclass_caching', True)))
def __init__(self, location=None, eclassdir=None): self._eclass_data_inst_cache = WeakValCache() # generate this. # self.eclasses = {} # {"Name": ("location", "_mtime_")} self.location = location self.eclassdir = eclassdir
def __init__(self, location, eclass_cache=None, masters=(), cache=(), default_mirrors=None, allow_missing_manifests=False, repo_config=None): """ :param location: on disk location of the tree :param cache: sequence of :obj:`pkgcore.cache.template.database` instances to use for storing metadata :param masters: repo masters this repo inherits from :param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache` instance representing the eclasses available, if None, generates the eclass_cache itself :param default_mirrors: Either None, or sequence of mirrors to try fetching from first, then falling back to other uri """ super().__init__() self.base = self.location = location try: if not stat.S_ISDIR(os.stat(self.base).st_mode): raise errors.InitializationError( f"base not a dir: {self.base}") except OSError as e: raise errors.InitializationError( f"lstat failed: {self.base}") from e if repo_config is None: repo_config = repo_objs.RepoConfig(location) self.config = repo_config # profiles dir is required by PMS if not os.path.isdir(self.config.profiles_base): raise errors.InvalidRepo( f'missing required profiles dir: {self.location!r}') # verify we support the repo's EAPI if not self.is_supported: raise errors.UnsupportedRepo(self) if eclass_cache is None: eclass_cache = eclass_cache_mod.cache(pjoin( self.location, 'eclass'), location=self.location) self.eclass_cache = eclass_cache self.masters = masters self.trees = tuple(masters) + (self, ) self.licenses = repo_objs.Licenses(self.location) if masters: self.licenses = repo_objs.OverlayedLicenses(*self.trees) mirrors = {} fp = pjoin(self.location, 'profiles', "thirdpartymirrors") try: for k, v in read_dict(fp, splitter=None).items(): v = v.split() shuffle(v) mirrors[k] = v except FileNotFoundError: pass # use mirrors from masters if not defined in the repo for master in masters: for k, v in master.mirrors.items(): if k not in mirrors: mirrors[k] = v if isinstance(cache, (tuple, list)): cache = tuple(cache) else: cache = (cache, ) self.mirrors = mirrors self.default_mirrors = default_mirrors self.cache = cache self._allow_missing_chksums = allow_missing_manifests self.package_class = self.package_factory(self, cache, self.eclass_cache, self.mirrors, self.default_mirrors) self._shared_pkg_cache = WeakValCache() self._masked = RestrictionRepo(repo_id='masked')
class UnconfiguredTree(prototype.tree): """Raw implementation supporting standard ebuild tree. Return packages don't have USE configuration bound to them. """ false_packages = frozenset(["CVS", ".svn"]) false_categories = frozenset([ "eclass", "profiles", "packages", "distfiles", "metadata", "licenses", "scripts", "CVS", "local" ]) configured = False configurables = ("domain", "settings") configure = None package_factory = staticmethod(ebuild_src.generate_new_factory) enable_gpg = False extension = '.ebuild' operations_kls = repo_operations pkgcore_config_type = ConfigHint( { 'location': 'str', 'eclass_cache': 'ref:eclass_cache', 'masters': 'refs:repo', 'cache': 'refs:cache', 'default_mirrors': 'list', 'allow_missing_manifests': 'bool', 'repo_config': 'ref:repo_config', }, typename='repo') def __init__(self, location, eclass_cache=None, masters=(), cache=(), default_mirrors=None, allow_missing_manifests=False, repo_config=None): """ :param location: on disk location of the tree :param cache: sequence of :obj:`pkgcore.cache.template.database` instances to use for storing metadata :param masters: repo masters this repo inherits from :param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache` instance representing the eclasses available, if None, generates the eclass_cache itself :param default_mirrors: Either None, or sequence of mirrors to try fetching from first, then falling back to other uri """ super().__init__() self.base = self.location = location try: if not stat.S_ISDIR(os.stat(self.base).st_mode): raise errors.InitializationError( f"base not a dir: {self.base}") except OSError as e: raise errors.InitializationError( f"lstat failed: {self.base}") from e if repo_config is None: repo_config = repo_objs.RepoConfig(location) self.config = repo_config # profiles dir is required by PMS if not os.path.isdir(self.config.profiles_base): raise errors.InvalidRepo( f'missing required profiles dir: {self.location!r}') # verify we support the repo's EAPI if not self.is_supported: raise errors.UnsupportedRepo(self) if eclass_cache is None: eclass_cache = eclass_cache_mod.cache(pjoin( self.location, 'eclass'), location=self.location) self.eclass_cache = eclass_cache self.masters = masters self.trees = tuple(masters) + (self, ) self.licenses = repo_objs.Licenses(self.location) if masters: self.licenses = repo_objs.OverlayedLicenses(*self.trees) mirrors = {} fp = pjoin(self.location, 'profiles', "thirdpartymirrors") try: for k, v in read_dict(fp, splitter=None).items(): v = v.split() shuffle(v) mirrors[k] = v except FileNotFoundError: pass # use mirrors from masters if not defined in the repo for master in masters: for k, v in master.mirrors.items(): if k not in mirrors: mirrors[k] = v if isinstance(cache, (tuple, list)): cache = tuple(cache) else: cache = (cache, ) self.mirrors = mirrors self.default_mirrors = default_mirrors self.cache = cache self._allow_missing_chksums = allow_missing_manifests self.package_class = self.package_factory(self, cache, self.eclass_cache, self.mirrors, self.default_mirrors) self._shared_pkg_cache = WeakValCache() self._masked = RestrictionRepo(repo_id='masked') repo_id = klass.alias_attr("config.repo_id") repo_name = klass.alias_attr("config.repo_name") eapi = klass.alias_attr('config.eapi') is_supported = klass.alias_attr('config.eapi.is_supported') @klass.jit_attr def known_arches(self): """Return all known arches for a repo (including masters).""" return frozenset( chain.from_iterable(r.config.known_arches for r in self.trees)) def path_restrict(self, path): """Return a restriction from a given path in a repo. :param path: full or partial path to an ebuild :return: a package restriction matching the given path if possible :raises ValueError: if the repo doesn't contain the given path, the path relates to a file that isn't an ebuild, or the ebuild isn't in the proper directory layout """ realpath = os.path.realpath(path) if realpath not in self: raise ValueError( f"{self.repo_id!r} repo doesn't contain: {path!r}") relpath = realpath[len(os.path.realpath(self.location)):].strip('/') repo_path = relpath.split(os.path.sep) if relpath else [] restrictions = [] if os.path.isfile(realpath): if not path.endswith('.ebuild'): raise ValueError(f"file is not an ebuild: {path!r}") elif len(repo_path) != 3: # ebuild isn't in a category/PN directory raise ValueError( f"ebuild not in the correct directory layout: {path!r}") # add restrictions until path components run out try: restrictions.append(restricts.RepositoryDep(self.repo_id)) if repo_path[0] in self.categories: restrictions.append(restricts.CategoryDep(repo_path[0])) restrictions.append(restricts.PackageDep(repo_path[1])) base = cpv.versioned_CPV( f"{repo_path[0]}/{os.path.splitext(repo_path[2])[0]}") restrictions.append( restricts.VersionMatch('=', base.version, rev=base.revision)) except IndexError: pass return packages.AndRestriction(*restrictions) def __getitem__(self, cpv): cpv_inst = self.package_class(*cpv) if cpv_inst.fullver not in self.versions[(cpv_inst.category, cpv_inst.package)]: if cpv_inst.revision is None: if f'{cpv_inst.fullver}-r0' in \ self.versions[(cpv_inst.category, cpv_inst.package)]: # ebuild on disk has an explicit -r0 in its name return cpv_inst raise KeyError(cpv) return cpv_inst def rebind(self, **kwds): """Generate a new tree instance with the same location using new keywords. :param kwds: see __init__ for valid values """ o = self.__class__(self.location, **kwds) o.categories = self.categories o.packages = self.packages o.versions = self.versions return o @klass.jit_attr def hardcoded_categories(self): # try reading $LOC/profiles/categories if it's available. categories = readlines(pjoin(self.base, 'profiles', 'categories'), True, True, True) if categories is not None: categories = tuple(map(intern, categories)) return categories def _get_categories(self, *optional_category): # why the auto return? current porttrees don't allow/support # categories deeper then one dir. if optional_category: # raise KeyError return () categories = set() for repo in self.trees: if repo.hardcoded_categories is not None: categories.update(repo.hardcoded_categories) if categories: return tuple(categories) try: return tuple( map( intern, filterfalse( self.false_categories.__contains__, (x for x in listdir_dirs(self.base) if x[0:1] != ".")))) except EnvironmentError as e: raise KeyError(f"failed fetching categories: {e}") from e def _get_packages(self, category): cpath = pjoin(self.base, category.lstrip(os.path.sep)) try: return tuple( filterfalse(self.false_packages.__contains__, listdir_dirs(cpath))) except FileNotFoundError: if category in self.categories: # ignore it, since it's PMS mandated that it be allowed. return () except EnvironmentError as e: category = pjoin(self.base, category.lstrip(os.path.sep)) raise KeyError( f'failed fetching packages for category {category}: {e}' ) from e def _get_versions(self, catpkg): cppath = pjoin(self.base, catpkg[0], catpkg[1]) pkg = f'{catpkg[-1]}-' lp = len(pkg) extension = self.extension ext_len = -len(extension) try: return tuple(x[lp:ext_len] for x in listdir_files(cppath) if x[ext_len:] == extension and x[:lp] == pkg) except EnvironmentError as e: raise KeyError("failed fetching versions for package %s: %s" % (pjoin(self.base, '/'.join(catpkg)), str(e))) from e def _pkg_filter(self, pkgs): """Filter packages with bad metadata.""" for pkg in pkgs: if pkg not in self._masked.itermatch(pkg.versioned_atom): # check pkgs for unsupported/invalid EAPIs and bad metadata try: if not pkg.is_supported: self._masked[pkg.versioned_atom] = MetadataException( pkg, 'eapi', f"EAPI '{pkg.eapi}' is not supported") continue # TODO: add a generic metadata validation method to avoid slow metadata checks? pkg.data pkg.required_use except MetadataException as e: self._masked[e.pkg.versioned_atom] = e continue yield pkg def itermatch(self, *args, **kwargs): kwargs.setdefault('pkg_filter', self._pkg_filter) return super().itermatch(*args, **kwargs) def _get_ebuild_path(self, pkg): if pkg.revision is None: try: if pkg.fullver not in self.versions[(pkg.category, pkg.package)]: # daft explicit -r0 on disk. return pjoin( self.base, pkg.category, pkg.package, f"{pkg.package}-{pkg.fullver}-r0{self.extension}") except KeyError as e: raise MetadataException(pkg, 'package', 'mismatched ebuild name') from e return pjoin(self.base, pkg.category, pkg.package, f"{pkg.package}-{pkg.fullver}{self.extension}") def _get_ebuild_src(self, pkg): return local_source(self._get_ebuild_path(pkg), encoding='utf8') def _get_shared_pkg_data(self, category, package): key = (category, package) o = self._shared_pkg_cache.get(key) if o is None: mxml = self._get_metadata_xml(category, package) manifest = self._get_manifest(category, package) o = repo_objs.SharedPkgData(mxml, manifest) self._shared_pkg_cache[key] = o return o def _get_metadata_xml(self, category, package): return repo_objs.LocalMetadataXml( pjoin(self.base, category, package, "metadata.xml")) def _get_manifest(self, category, package): return digest.Manifest(pjoin(self.base, category, package, "Manifest"), thin=self.config.manifests.thin, enforce_gpg=self.enable_gpg) def _get_digests(self, pkg, allow_missing=False): if self.config.manifests.disabled: return True, {} try: manifest = pkg._shared_pkg_data.manifest manifest.allow_missing = allow_missing return allow_missing, manifest.distfiles except pkg_errors.ParseChksumError as e: if e.missing and allow_missing: return allow_missing, {} raise def __repr__(self): return "<ebuild %s location=%r @%#8x>" % (self.__class__.__name__, self.base, id(self)) @klass.jit_attr def _visibility_limiters(self): path = pjoin(self.base, 'profiles', 'package.mask') pos, neg = [], [] try: if (self.config.eapi.options['has_profile_data_dirs'] or self.config.profile_formats.intersection( ['portage-1', 'portage-2'])): paths = sorted_scan(path) else: paths = [path] for path in paths: for line in iter_read_bash(path): line = line.strip() if line in ('-', ''): raise profiles.ProfileError( pjoin(self.base, 'profiles'), 'package.mask', "encountered empty negation: -") if line.startswith('-'): neg.append(atom.atom(line[1:])) else: pos.append(atom.atom(line)) except FileNotFoundError: pass except ebuild_errors.MalformedAtom as e: raise profiles.ProfileError(pjoin(self.base, 'profiles'), 'package.mask', e) from e return tuple(neg), tuple(pos) def _regen_operation_helper(self, **kwds): return _RegenOpHelper(self, force=bool(kwds.get('force', False)), eclass_caching=bool( kwds.get('eclass_caching', True)))
def __setstate__(self, state): self.__dict__ = state.copy() self.__dict__['_eclass_data_inst_cache'] = WeakValCache()
def setUp(self): self.o = RefObj() self.w = WeakValCache()
def __setstate__(self, state): self.__dict__ = state.copy() self.__dict__['_shared_pkg_cache'] = WeakValCache()
def __init__(self, location, eclass_cache=None, masters=(), cache=(), default_mirrors=None, allow_missing_manifests=False, repo_config=None): """ :param location: on disk location of the tree :param cache: sequence of :obj:`pkgcore.cache.template.database` instances to use for storing metadata :param masters: repo masters this repo inherits from :param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache` instance representing the eclasses available, if None, generates the eclass_cache itself :param default_mirrors: Either None, or sequence of mirrors to try fetching from first, then falling back to other uri """ super().__init__() self.base = self.location = location try: if not stat.S_ISDIR(os.stat(self.base).st_mode): raise errors.InitializationError(f"base not a dir: {self.base}") except OSError as e: raise errors.InitializationError(f"lstat failed: {self.base}") from e if repo_config is None: repo_config = repo_objs.RepoConfig(location) self.config = repo_config # profiles dir is required by PMS if not os.path.isdir(self.config.profiles_base): raise errors.InvalidRepo(f'missing required profiles dir: {self.location!r}') # verify we support the repo's EAPI if not self.is_supported: raise errors.UnsupportedRepo(self) if eclass_cache is None: eclass_cache = eclass_cache_mod.cache( pjoin(self.location, 'eclass'), location=self.location) self.eclass_cache = eclass_cache self.masters = masters self.trees = tuple(masters) + (self,) self.licenses = repo_objs.Licenses(self.location) if masters: self.licenses = repo_objs.OverlayedLicenses(*self.trees) mirrors = {} fp = pjoin(self.location, 'profiles', "thirdpartymirrors") try: for k, v in read_dict(fp, splitter=None).items(): v = v.split() shuffle(v) mirrors[k] = v except FileNotFoundError: pass # use mirrors from masters if not defined in the repo for master in masters: for k, v in master.mirrors.items(): if k not in mirrors: mirrors[k] = v if isinstance(cache, (tuple, list)): cache = tuple(cache) else: cache = (cache,) self.mirrors = mirrors self.default_mirrors = default_mirrors self.cache = cache self._allow_missing_chksums = allow_missing_manifests self.package_class = self.package_factory( self, cache, self.eclass_cache, self.mirrors, self.default_mirrors) self._shared_pkg_cache = WeakValCache() self._masked = RestrictionRepo(repo_id='masked')
class UnconfiguredTree(prototype.tree): """Raw implementation supporting standard ebuild tree. Return packages don't have USE configuration bound to them. """ false_packages = frozenset(["CVS", ".svn"]) false_categories = frozenset([ "eclass", "profiles", "packages", "distfiles", "metadata", "licenses", "scripts", "CVS", "local"]) configured = False configurables = ("domain", "settings") configure = None package_factory = staticmethod(ebuild_src.generate_new_factory) enable_gpg = False extension = '.ebuild' operations_kls = repo_operations pkgcore_config_type = ConfigHint({ 'location': 'str', 'eclass_cache': 'ref:eclass_cache', 'masters': 'refs:repo', 'cache': 'refs:cache', 'default_mirrors': 'list', 'allow_missing_manifests': 'bool', 'repo_config': 'ref:repo_config', }, typename='repo') def __init__(self, location, eclass_cache=None, masters=(), cache=(), default_mirrors=None, allow_missing_manifests=False, repo_config=None): """ :param location: on disk location of the tree :param cache: sequence of :obj:`pkgcore.cache.template.database` instances to use for storing metadata :param masters: repo masters this repo inherits from :param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache` instance representing the eclasses available, if None, generates the eclass_cache itself :param default_mirrors: Either None, or sequence of mirrors to try fetching from first, then falling back to other uri """ super().__init__() self.base = self.location = location try: if not stat.S_ISDIR(os.stat(self.base).st_mode): raise errors.InitializationError(f"base not a dir: {self.base}") except OSError as e: raise errors.InitializationError(f"lstat failed: {self.base}") from e if repo_config is None: repo_config = repo_objs.RepoConfig(location) self.config = repo_config # profiles dir is required by PMS if not os.path.isdir(self.config.profiles_base): raise errors.InvalidRepo(f'missing required profiles dir: {self.location!r}') # verify we support the repo's EAPI if not self.is_supported: raise errors.UnsupportedRepo(self) if eclass_cache is None: eclass_cache = eclass_cache_mod.cache( pjoin(self.location, 'eclass'), location=self.location) self.eclass_cache = eclass_cache self.masters = masters self.trees = tuple(masters) + (self,) self.licenses = repo_objs.Licenses(self.location) if masters: self.licenses = repo_objs.OverlayedLicenses(*self.trees) mirrors = {} fp = pjoin(self.location, 'profiles', "thirdpartymirrors") try: for k, v in read_dict(fp, splitter=None).items(): v = v.split() shuffle(v) mirrors[k] = v except FileNotFoundError: pass # use mirrors from masters if not defined in the repo for master in masters: for k, v in master.mirrors.items(): if k not in mirrors: mirrors[k] = v if isinstance(cache, (tuple, list)): cache = tuple(cache) else: cache = (cache,) self.mirrors = mirrors self.default_mirrors = default_mirrors self.cache = cache self._allow_missing_chksums = allow_missing_manifests self.package_class = self.package_factory( self, cache, self.eclass_cache, self.mirrors, self.default_mirrors) self._shared_pkg_cache = WeakValCache() self._masked = RestrictionRepo(repo_id='masked') repo_id = klass.alias_attr("config.repo_id") repo_name = klass.alias_attr("config.repo_name") eapi = klass.alias_attr('config.eapi') is_supported = klass.alias_attr('config.eapi.is_supported') @klass.jit_attr def known_arches(self): """Return all known arches for a repo (including masters).""" return frozenset(chain.from_iterable( r.config.known_arches for r in self.trees)) def path_restrict(self, path): """Return a restriction from a given path in a repo. :param path: full or partial path to an ebuild :return: a package restriction matching the given path if possible :raises ValueError: if the repo doesn't contain the given path, the path relates to a file that isn't an ebuild, or the ebuild isn't in the proper directory layout """ realpath = os.path.realpath(path) if realpath not in self: raise ValueError(f"{self.repo_id!r} repo doesn't contain: {path!r}") relpath = realpath[len(os.path.realpath(self.location)):].strip('/') repo_path = relpath.split(os.path.sep) if relpath else [] restrictions = [] if os.path.isfile(realpath): if not path.endswith('.ebuild'): raise ValueError(f"file is not an ebuild: {path!r}") elif len(repo_path) != 3: # ebuild isn't in a category/PN directory raise ValueError( f"ebuild not in the correct directory layout: {path!r}") # add restrictions until path components run out try: restrictions.append(restricts.RepositoryDep(self.repo_id)) if repo_path[0] in self.categories: restrictions.append(restricts.CategoryDep(repo_path[0])) restrictions.append(restricts.PackageDep(repo_path[1])) base = cpv.versioned_CPV(f"{repo_path[0]}/{os.path.splitext(repo_path[2])[0]}") restrictions.append(restricts.VersionMatch('=', base.version, rev=base.revision)) except IndexError: pass return packages.AndRestriction(*restrictions) def __getitem__(self, cpv): cpv_inst = self.package_class(*cpv) if cpv_inst.fullver not in self.versions[(cpv_inst.category, cpv_inst.package)]: if cpv_inst.revision is None: if f'{cpv_inst.fullver}-r0' in \ self.versions[(cpv_inst.category, cpv_inst.package)]: # ebuild on disk has an explicit -r0 in its name return cpv_inst raise KeyError(cpv) return cpv_inst def rebind(self, **kwds): """Generate a new tree instance with the same location using new keywords. :param kwds: see __init__ for valid values """ o = self.__class__(self.location, **kwds) o.categories = self.categories o.packages = self.packages o.versions = self.versions return o @klass.jit_attr def hardcoded_categories(self): # try reading $LOC/profiles/categories if it's available. categories = readlines( pjoin(self.base, 'profiles', 'categories'), True, True, True) if categories is not None: categories = tuple(map(intern, categories)) return categories def _get_categories(self, *optional_category): # why the auto return? current porttrees don't allow/support # categories deeper then one dir. if optional_category: # raise KeyError return () categories = set() for repo in self.trees: if repo.hardcoded_categories is not None: categories.update(repo.hardcoded_categories) if categories: return tuple(categories) try: return tuple(map(intern, filterfalse( self.false_categories.__contains__, (x for x in listdir_dirs(self.base) if x[0:1] != ".")))) except EnvironmentError as e: raise KeyError(f"failed fetching categories: {e}") from e def _get_packages(self, category): cpath = pjoin(self.base, category.lstrip(os.path.sep)) try: return tuple(filterfalse( self.false_packages.__contains__, listdir_dirs(cpath))) except FileNotFoundError: if category in self.categories: # ignore it, since it's PMS mandated that it be allowed. return () except EnvironmentError as e: category = pjoin(self.base, category.lstrip(os.path.sep)) raise KeyError( f'failed fetching packages for category {category}: {e}') from e def _get_versions(self, catpkg): cppath = pjoin(self.base, catpkg[0], catpkg[1]) pkg = f'{catpkg[-1]}-' lp = len(pkg) extension = self.extension ext_len = -len(extension) try: return tuple( x[lp:ext_len] for x in listdir_files(cppath) if x[ext_len:] == extension and x[:lp] == pkg) except EnvironmentError as e: raise KeyError( "failed fetching versions for package %s: %s" % (pjoin(self.base, '/'.join(catpkg)), str(e))) from e def _pkg_filter(self, pkgs): """Filter packages with bad metadata.""" for pkg in pkgs: if pkg not in self._masked.itermatch(pkg.versioned_atom): # check pkgs for unsupported/invalid EAPIs and bad metadata try: if not pkg.is_supported: self._masked[pkg.versioned_atom] = MetadataException( pkg, 'eapi', f"EAPI '{pkg.eapi}' is not supported") continue # TODO: add a generic metadata validation method to avoid slow metadata checks? pkg.data pkg.required_use except MetadataException as e: self._masked[e.pkg.versioned_atom] = e continue yield pkg def itermatch(self, *args, **kwargs): kwargs.setdefault('pkg_filter', self._pkg_filter) return super().itermatch(*args, **kwargs) def _get_ebuild_path(self, pkg): if pkg.revision is None: try: if pkg.fullver not in self.versions[(pkg.category, pkg.package)]: # daft explicit -r0 on disk. return pjoin( self.base, pkg.category, pkg.package, f"{pkg.package}-{pkg.fullver}-r0{self.extension}") except KeyError as e: raise MetadataException(pkg, 'package', 'mismatched ebuild name') from e return pjoin( self.base, pkg.category, pkg.package, f"{pkg.package}-{pkg.fullver}{self.extension}") def _get_ebuild_src(self, pkg): return local_source(self._get_ebuild_path(pkg), encoding='utf8') def _get_shared_pkg_data(self, category, package): key = (category, package) o = self._shared_pkg_cache.get(key) if o is None: mxml = self._get_metadata_xml(category, package) manifest = self._get_manifest(category, package) o = repo_objs.SharedPkgData(mxml, manifest) self._shared_pkg_cache[key] = o return o def _get_metadata_xml(self, category, package): return repo_objs.LocalMetadataXml(pjoin( self.base, category, package, "metadata.xml")) def _get_manifest(self, category, package): return digest.Manifest(pjoin( self.base, category, package, "Manifest"), thin=self.config.manifests.thin, enforce_gpg=self.enable_gpg) def _get_digests(self, pkg, allow_missing=False): if self.config.manifests.disabled: return True, {} try: manifest = pkg._shared_pkg_data.manifest manifest.allow_missing = allow_missing return allow_missing, manifest.distfiles except pkg_errors.ParseChksumError as e: if e.missing and allow_missing: return allow_missing, {} raise def __repr__(self): return "<ebuild %s location=%r @%#8x>" % ( self.__class__.__name__, self.base, id(self)) @klass.jit_attr def _visibility_limiters(self): path = pjoin(self.base, 'profiles', 'package.mask') pos, neg = [], [] try: if (self.config.eapi.options['has_profile_data_dirs'] or self.config.profile_formats.intersection(['portage-1', 'portage-2'])): paths = sorted_scan(path) else: paths = [path] for path in paths: for line in iter_read_bash(path): line = line.strip() if line in ('-', ''): raise profiles.ProfileError( pjoin(self.base, 'profiles'), 'package.mask', "encountered empty negation: -") if line.startswith('-'): neg.append(atom.atom(line[1:])) else: pos.append(atom.atom(line)) except FileNotFoundError: pass except ebuild_errors.MalformedAtom as e: raise profiles.ProfileError( pjoin(self.base, 'profiles'), 'package.mask', e) from e return tuple(neg), tuple(pos) def _regen_operation_helper(self, **kwds): return _RegenOpHelper( self, force=bool(kwds.get('force', False)), eclass_caching=bool(kwds.get('eclass_caching', True)))
def setup_method(self, method): self.o = RefObj() self.w = WeakValCache()
class TestWeakValCache(object): def setup_method(self, method): self.o = RefObj() self.w = WeakValCache() def test_setitem(self): s = "asdf" self.w[s] = self.o self.w["fds"] = self.o self.w[s] = self.o def test_getitem(self): s = "asdf" self.w[s] = self.o assert self.w[s] is self.o def test_expiring(self): s = "asdf" self.w[s] = self.o assert self.w[s] del self.o with pytest.raises(KeyError): self.w.__getitem__(s) def test_get(self): s = "asdf" with pytest.raises(KeyError): self.w.__getitem__(s) self.w[s] = self.o assert self.w.get(s) is self.o def test_keys(self): assert list(self.w.keys()) == [] self.w['a'] = self.o self.w['b'] = self.o self.w['c'] = self.o assert sorted(self.w.keys()) == ['a', 'b', 'c'] del self.o assert self.w.keys() == [] def test_values(self): assert list(self.w.values()) == [] self.w['a'] = self.o self.w['b'] = self.o self.w['c'] = self.o assert len(iter(self.w.values())) == 3 del self.o assert self.w.values() == [] def test_items(self): assert list(self.w.items()) == [] self.w['a'] = self.o self.w['b'] = self.o self.w['c'] = self.o assert len(iter(self.w.items())) == 3 del self.o assert self.w.items() == []
def __init__(self, portdir=None, eclassdir=None): self._eclass_data_inst_cache = WeakValCache() # generate this. # self.eclasses = {} # {"Name": ("location", "_mtime_")} self.portdir = portdir self.eclassdir = eclassdir
def __init__(self, location, eclass_cache, cache=(), default_mirrors=None, override_repo_id=None, ignore_paludis_versioning=False, allow_missing_manifests=False, repo_config=None): """ :param location: on disk location of the tree :param cache: sequence of :obj:`pkgcore.cache.template.database` instances to use for storing metadata :param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache` instance representing the eclasses available, if None, generates the eclass_cache itself :param default_mirrors: Either None, or sequence of mirrors to try fetching from first, then falling back to other uri :param override_repo_id: Either None, or string to force as the repository unique id :param ignore_paludis_versioning: If False, fail when -scm is encountred. if True, silently ignore -scm ebuilds. """ prototype.tree.__init__(self) if repo_config is None: repo_config = repo_objs.RepoConfig(location) self.config = repo_config self._repo_id = override_repo_id self.base = self.location = location try: if not stat.S_ISDIR(os.stat(self.base).st_mode): raise errors.InitializationError("base not a dir: %s" % self.base) except OSError: raise_from( errors.InitializationError("lstat failed on base %s" % (self.base, ))) self.eclass_cache = eclass_cache self.licenses = repo_objs.Licenses(location) fp = pjoin(self.base, metadata_offset, "thirdpartymirrors") mirrors = {} try: for k, v in read_dict(fp, splitter=None).iteritems(): v = v.split() shuffle(v) mirrors[k] = v except EnvironmentError as ee: if ee.errno != errno.ENOENT: raise if isinstance(cache, (tuple, list)): cache = tuple(cache) else: cache = (cache, ) self.mirrors = mirrors self.default_mirrors = default_mirrors self.cache = cache self.ignore_paludis_versioning = ignore_paludis_versioning self._allow_missing_chksums = allow_missing_manifests self.package_class = self.package_factory(self, cache, self.eclass_cache, self.mirrors, self.default_mirrors) self._shared_pkg_cache = WeakValCache()