class TestWeakValCache(TestCase):
if WeakValueDictionary is WeakValCache:
skip = "WeakValCache is weakref.WeakValueDictionary; indicates " \
"snakeoil._caching isn't compiled"
def setUp(self):
self.o = RefObj()
self.w = WeakValCache()
def test_setitem(self):
s = "asdf"
self.w[s] = self.o
self.w["fds"] = self.o
self.w[s] = self.o
def test_getitem(self):
s = "asdf"
self.w[s] = self.o
self.assertIdentical(self.w[s], self.o)
def test_expiring(self):
s = "asdf"
self.w[s] = self.o
self.assertTrue(self.w[s])
del self.o
self.assertRaises(KeyError, self.w.__getitem__, s)
def test_get(self):
s = "asdf"
self.assertRaises(KeyError, self.w.__getitem__, s)
self.w[s] = self.o
self.assertIdentical(self.w.get(s), self.o)
def test_keys(self):
self.assertEqual(list(self.w.keys()), [])
self.w['a'] = self.o
self.w['b'] = self.o
self.w['c'] = self.o
self.assertEqual(sorted(self.w.keys()), ['a', 'b', 'c'])
del self.o
self.assertEqual(sorted(self.w.keys()), [])
def test_values(self):
self.assertEqual(self.w.values(), [])
self.w['a'] = self.o
self.w['b'] = self.o
self.w['c'] = self.o
self.assertEqual(len(self.w.values()), 3)
del self.o
self.assertEqual(sorted(self.w.values()), [])
def test_items(self):
self.assertEqual(self.w.items(), [])
self.w['a'] = self.o
self.w['b'] = self.o
self.w['c'] = self.o
self.assertEqual(len(self.w.items()), 3)
del self.o
self.assertEqual(sorted(self.w.items()), [])
class TestWeakValCache(TestCase):
if WeakValueDictionary is WeakValCache:
skip = "WeakValCache is weakref.WeakValueDictionary; indicates " \
"snakeoil._caching isn't compiled"
def setUp(self):
self.o = RefObj()
self.w = WeakValCache()
def test_setitem(self):
s = "asdf"
self.w[s] = self.o
self.w["fds"] = self.o
self.w[s] = self.o
def test_getitem(self):
s = "asdf"
self.w[s] = self.o
self.assertIdentical(self.w[s], self.o)
def test_expiring(self):
s = "asdf"
self.w[s] = self.o
self.assertTrue(self.w[s])
del self.o
self.assertRaises(KeyError, self.w.__getitem__, s)
def test_get(self):
s = "asdf"
self.assertRaises(KeyError, self.w.__getitem__, s)
self.w[s] = self.o
self.assertIdentical(self.w.get(s), self.o)
def __init__(self, location, eclass_cache, cache=(),
default_mirrors=None, ignore_paludis_versioning=False,
allow_missing_manifests=False, repo_config=None):
"""
:param location: on disk location of the tree
:param cache: sequence of :obj:`pkgcore.cache.template.database` instances
to use for storing metadata
:param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache`
instance representing the eclasses available,
if None, generates the eclass_cache itself
:param default_mirrors: Either None, or sequence of mirrors to try
fetching from first, then falling back to other uri
:param ignore_paludis_versioning: If False, fail when -scm is encountred. if True,
silently ignore -scm ebuilds.
"""
prototype.tree.__init__(self)
if repo_config is None:
repo_config = repo_objs.RepoConfig(location)
self.config = repo_config
self.base = self.location = location
try:
if not stat.S_ISDIR(os.stat(self.base).st_mode):
raise errors.InitializationError(
"base not a dir: %s" % self.base)
except OSError:
raise_from(errors.InitializationError(
"lstat failed on base %s" % (self.base,)))
self.eclass_cache = eclass_cache
self.licenses = repo_objs.Licenses(location)
fp = pjoin(self.base, metadata_offset, "thirdpartymirrors")
mirrors = {}
try:
for k, v in read_dict(fp, splitter=None).iteritems():
v = v.split()
shuffle(v)
mirrors[k] = v
except EnvironmentError as ee:
if ee.errno != errno.ENOENT:
raise
if isinstance(cache, (tuple, list)):
cache = tuple(cache)
else:
cache = (cache,)
self.mirrors = mirrors
self.default_mirrors = default_mirrors
self.cache = cache
self.ignore_paludis_versioning = ignore_paludis_versioning
self._allow_missing_chksums = allow_missing_manifests
self.package_class = self.package_factory(
self, cache, self.eclass_cache, self.mirrors, self.default_mirrors)
self._shared_pkg_cache = WeakValCache()
class base(object):
"""
Maintains the cache information about eclasses available to an ebuild.
"""
def __init__(self, location=None, eclassdir=None):
self._eclass_data_inst_cache = WeakValCache()
# generate this.
# self.eclasses = {} # {"Name": ("location", "_mtime_")}
self.location = location
self.eclassdir = eclassdir
def get_eclass_data(self, inherits):
"""Return the cachable entries from a list of inherited eclasses.
Only make get_eclass_data calls for data you know came from
this eclass_cache, otherwise be ready to catch a KeyError
exception for any eclass that was requested, but not known to
this cache.
"""
keys = tuple(sorted(inherits))
o = self._eclass_data_inst_cache.get(keys)
if o is None:
o = ImmutableDict((k, self.eclasses[k]) for k in keys)
self._eclass_data_inst_cache[keys] = o
return o
def get_eclass(self, eclass):
o = self.eclasses.get(eclass)
if o is None:
return None
return local_source(o.path)
eclasses = jit_attr_ext_method("_load_eclasses", "_eclasses")
def rebuild_cache_entry(self, entry_eclasses):
"""Check if eclass data is still valid.
Given a dict as returned by get_eclass_data, walk it comparing
it to internal eclass view.
:return: a boolean representing whether that eclass data is still
up to date, or not
"""
ec = self.eclasses
d = {}
for eclass, chksums in entry_eclasses:
data = ec.get(eclass)
if any(val != getattr(data, chf, None) for chf, val in chksums):
return None
d[eclass] = data
return d
class base(object):
"""
Maintains the cache information about eclasses available to an ebuild.
"""
def __init__(self, portdir=None, eclassdir=None):
self._eclass_data_inst_cache = WeakValCache()
# generate this.
# self.eclasses = {} # {"Name": ("location", "_mtime_")}
self.portdir = portdir
self.eclassdir = eclassdir
def get_eclass_data(self, inherits):
"""Return the cachable entries from a list of inherited eclasses.
Only make get_eclass_data calls for data you know came from
this eclass_cache, otherwise be ready to catch a KeyError
exception for any eclass that was requested, but not known to
this cache.
"""
keys = tuple(sorted(inherits))
o = self._eclass_data_inst_cache.get(keys)
if o is None:
o = ImmutableDict((k, self.eclasses[k]) for k in keys)
self._eclass_data_inst_cache[keys] = o
return o
def get_eclass(self, eclass):
o = self.eclasses.get(eclass)
if o is None:
return None
return local_source(o.path)
eclasses = jit_attr_ext_method("_load_eclasses", "_eclasses")
def rebuild_cache_entry(self, entry_eclasses):
"""Check if eclass data is still valid.
Given a dict as returned by get_eclass_data, walk it comparing
it to internal eclass view.
:return: a boolean representing whether that eclass data is still
up to date, or not
"""
ec = self.eclasses
d = {}
for eclass, chksums in entry_eclasses:
data = ec.get(eclass)
if any(val != getattr(data, chf, None) for chf, val in chksums):
return None
d[eclass] = data
return d
class _UnconfiguredTree(prototype.tree):
"""
raw implementation supporting standard ebuild tree.
return packages don't have USE configuration bound to them.
"""
false_packages = frozenset(["CVS", ".svn"])
false_categories = frozenset([
"eclass", "profiles", "packages", "distfiles", "metadata", "licenses",
"scripts", "CVS", "local"
])
configured = False
configurables = ("domain", "settings")
configure = None
package_factory = staticmethod(ebuild_src.generate_new_factory)
# This attributes needs to be replaced/removed; it's a hack for pmerge.
repository_type = 'source'
enable_gpg = False
extension = '.ebuild'
operations_kls = repo_operations
pkgcore_config_type = ConfigHint(
{
'location': 'str',
'cache': 'refs:cache',
'eclass_cache': 'ref:eclass_cache',
'default_mirrors': 'list',
'override_repo_id': 'str',
'ignore_paludis_versioning': 'bool',
'allow_missing_manifests': 'bool',
'repo_config': 'ref:raw_repo',
},
typename='repo')
def __init__(self,
location,
eclass_cache,
cache=(),
default_mirrors=None,
override_repo_id=None,
ignore_paludis_versioning=False,
allow_missing_manifests=False,
repo_config=None):
"""
:param location: on disk location of the tree
:param cache: sequence of :obj:`pkgcore.cache.template.database` instances
to use for storing metadata
:param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache`
instance representing the eclasses available,
if None, generates the eclass_cache itself
:param default_mirrors: Either None, or sequence of mirrors to try
fetching from first, then falling back to other uri
:param override_repo_id: Either None, or string to force as the
repository unique id
:param ignore_paludis_versioning: If False, fail when -scm is encountred. if True,
silently ignore -scm ebuilds.
"""
prototype.tree.__init__(self)
if repo_config is None:
repo_config = repo_objs.RepoConfig(location)
self.config = repo_config
self._repo_id = override_repo_id
self.base = self.location = location
try:
if not stat.S_ISDIR(os.stat(self.base).st_mode):
raise errors.InitializationError("base not a dir: %s" %
self.base)
except OSError:
raise_from(
errors.InitializationError("lstat failed on base %s" %
(self.base, )))
self.eclass_cache = eclass_cache
self.licenses = repo_objs.Licenses(location)
fp = pjoin(self.base, metadata_offset, "thirdpartymirrors")
mirrors = {}
try:
for k, v in read_dict(fp, splitter=None).iteritems():
v = v.split()
shuffle(v)
mirrors[k] = v
except EnvironmentError as ee:
if ee.errno != errno.ENOENT:
raise
if isinstance(cache, (tuple, list)):
cache = tuple(cache)
else:
cache = (cache, )
self.mirrors = mirrors
self.default_mirrors = default_mirrors
self.cache = cache
self.ignore_paludis_versioning = ignore_paludis_versioning
self._allow_missing_chksums = allow_missing_manifests
self.package_class = self.package_factory(self, cache,
self.eclass_cache,
self.mirrors,
self.default_mirrors)
self._shared_pkg_cache = WeakValCache()
repo_id = klass.alias_attr("config.repo_id")
def __getitem__(self, cpv):
cpv_inst = self.package_class(*cpv)
if cpv_inst.fullver not in self.versions[(cpv_inst.category,
cpv_inst.package)]:
if cpv_inst.revision is None:
if '%s-r0' % cpv_inst.fullver in \
self.versions[(cpv_inst.category, cpv_inst.package)]:
# ebuild on disk has an explicit -r0 in it's name
return cpv_inst
raise KeyError(cpv)
return cpv_inst
def rebind(self, **kwds):
"""
generate a new tree instance with the same location using new keywords.
:param kwds: see __init__ for valid values
"""
o = self.__class__(self.location, **kwds)
o.categories = self.categories
o.packages = self.packages
o.versions = self.versions
return o
@klass.jit_attr
def hardcoded_categories(self):
# try reading $LOC/profiles/categories if it's available.
cats = readlines(pjoin(self.base, 'profiles', 'categories'), True,
True, True)
if cats is not None:
cats = tuple(imap(intern, cats))
return cats
def _get_categories(self, *optional_category):
# why the auto return? current porttrees don't allow/support
# categories deeper then one dir.
if optional_category:
#raise KeyError
return ()
cats = self.hardcoded_categories
if cats is not None:
return cats
try:
return tuple(
imap(
intern,
ifilterfalse(
self.false_categories.__contains__,
(x
for x in listdir_dirs(self.base) if x[0:1] != "."))))
except EnvironmentError as e:
raise_from(KeyError("failed fetching categories: %s" % str(e)))
def _get_packages(self, category):
cpath = pjoin(self.base, category.lstrip(os.path.sep))
try:
return tuple(
ifilterfalse(self.false_packages.__contains__,
listdir_dirs(cpath)))
except EnvironmentError as e:
if e.errno == errno.ENOENT:
if self.hardcoded_categories and category in self.hardcoded_categories or \
isinstance(self, _SlavedTree) and category in self.parent_repo.categories:
# ignore it, since it's PMS mandated that it be allowed.
return ()
raise_from(KeyError("failed fetching packages for category %s: %s" % \
(pjoin(self.base, category.lstrip(os.path.sep)), \
str(e))))
def _get_versions(self, catpkg):
cppath = pjoin(self.base, catpkg[0], catpkg[1])
pkg = catpkg[-1] + "-"
lp = len(pkg)
extension = self.extension
ext_len = -len(extension)
try:
ret = tuple(x[lp:ext_len] for x in listdir_files(cppath)
if x[ext_len:] == extension and x[:lp] == pkg)
if any(('scm' in x or '-try' in x) for x in ret):
if not self.ignore_paludis_versioning:
for x in ret:
if 'scm' in x:
raise ebuild_errors.InvalidCPV(
"%s/%s-%s has nonstandard -scm "
"version component" % (catpkg + (x, )))
elif 'try' in x:
raise ebuild_errors.InvalidCPV(
"%s/%s-%s has nonstandard -try "
"version component" % (catpkg + (x, )))
raise AssertionError('unreachable codepoint was reached')
return tuple(x for x in ret
if ('scm' not in x and 'try' not in x))
return ret
except EnvironmentError as e:
raise_from(KeyError("failed fetching versions for package %s: %s" % \
(pjoin(self.base, catpkg.lstrip(os.path.sep)), str(e))))
def _get_ebuild_path(self, pkg):
if pkg.revision is None:
if pkg.fullver not in self.versions[(pkg.category, pkg.package)]:
# daft explicit -r0 on disk.
return pjoin(
self.base, pkg.category, pkg.package,
"%s-%s-r0%s" % (pkg.package, pkg.fullver, self.extension))
return pjoin(self.base, pkg.category, pkg.package, \
"%s-%s%s" % (pkg.package, pkg.fullver, self.extension))
def _get_ebuild_src(self, pkg):
return local_source(self._get_ebuild_path(pkg), encoding='utf8')
def _get_shared_pkg_data(self, category, package):
key = (category, package)
o = self._shared_pkg_cache.get(key)
if o is None:
mxml = self._get_metadata_xml(category, package)
manifest = self._get_manifest(category, package)
o = repo_objs.SharedPkgData(mxml, manifest)
self._shared_pkg_cache[key] = o
return o
def _get_metadata_xml(self, category, package):
return repo_objs.LocalMetadataXml(
pjoin(self.base, category, package, "metadata.xml"))
def _get_manifest(self, category, package):
return digest.Manifest(pjoin(self.base, category, package, "Manifest"),
thin=self.config.manifests.thin,
enforce_gpg=self.enable_gpg)
def _get_digests(self, pkg, allow_missing=False):
if self.config.manifests.disabled:
return True, {}
try:
manifest = pkg._shared_pkg_data.manifest
return allow_missing, manifest.distfiles
except pkg_errors.ParseChksumError as e:
if e.missing and allow_missing:
return allow_missing, {}
raise
def __str__(self):
return "%s.%s: location %s" % (self.__class__.__module__,
self.__class__.__name__, self.base)
def __repr__(self):
return "<ebuild %s location=%r @%#8x>" % (self.__class__.__name__,
self.base, id(self))
def _visibility_limiters(self):
path = pjoin(self.base, 'profiles', 'package.mask')
pos, neg = [], []
try:
if self.config.profile_format not in ['pms', 'portage-2']:
paths = sorted(x.location for x in iter_scan(path) if x.is_reg)
else:
paths = [path]
for path in paths:
for line in iter_read_bash(path):
line = line.strip()
if line in ('-', ''):
raise profiles.ProfileError(
pjoin(self.base, 'profiles'), 'package.mask',
"encountered empty negation: -")
if line.startswith('-'):
neg.append(atom.atom(line[1:]))
else:
pos.append(atom.atom(line))
except IOError as i:
if i.errno != errno.ENOENT:
raise
except ebuild_errors.MalformedAtom as ma:
raise_from(
profiles.ProfileError(pjoin(self.base, 'profiles'),
'package.mask', ma))
return [neg, pos]
def _regen_operation_helper(self, **kwds):
return _RegenOpHelper(self,
force=bool(kwds.get('force', False)),
eclass_caching=bool(
kwds.get('eclass_caching', True)))
class _UnconfiguredTree(prototype.tree):
"""Raw implementation supporting standard ebuild tree.
Return packages don't have USE configuration bound to them.
"""
false_packages = frozenset(["CVS", ".svn"])
false_categories = frozenset([
"eclass", "profiles", "packages", "distfiles", "metadata",
"licenses", "scripts", "CVS", "local"])
configured = False
configurables = ("domain", "settings")
configure = None
package_factory = staticmethod(ebuild_src.generate_new_factory)
enable_gpg = False
extension = '.ebuild'
operations_kls = repo_operations
pkgcore_config_type = ConfigHint({
'location': 'str',
'eclass_cache': 'ref:eclass_cache',
'masters': 'refs:repo',
'cache': 'refs:cache',
'default_mirrors': 'list',
'ignore_paludis_versioning': 'bool',
'allow_missing_manifests': 'bool',
'repo_config': 'ref:repo_config',
},
typename='repo')
def __init__(self, location, eclass_cache=None, masters=(), cache=(),
default_mirrors=None, ignore_paludis_versioning=False,
allow_missing_manifests=False, repo_config=None):
"""
:param location: on disk location of the tree
:param cache: sequence of :obj:`pkgcore.cache.template.database` instances
to use for storing metadata
:param masters: repo masters this repo inherits from
:param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache`
instance representing the eclasses available,
if None, generates the eclass_cache itself
:param default_mirrors: Either None, or sequence of mirrors to try
fetching from first, then falling back to other uri
:param ignore_paludis_versioning: If False, fail when -scm is encountered. if True,
silently ignore -scm ebuilds.
"""
prototype.tree.__init__(self)
self.base = self.location = location
try:
if not stat.S_ISDIR(os.stat(self.base).st_mode):
raise errors.InitializationError(
"base not a dir: %s" % self.base)
except OSError:
raise_from(errors.InitializationError(
"lstat failed on base %s" % (self.base,)))
if repo_config is None:
repo_config = repo_objs.RepoConfig(location)
self.config = repo_config
if eclass_cache is None:
eclass_cache = eclass_cache_module.cache(pjoin(self.location, 'eclass'))
self.eclass_cache = eclass_cache
self.masters = masters
self.trees = tuple(masters) + (self,)
self.licenses = repo_objs.Licenses(self.location)
if masters:
self.licenses = repo_objs.OverlayedLicenses(*self.trees)
mirrors = {}
fp = pjoin(self.location, metadata_offset, "thirdpartymirrors")
try:
for k, v in read_dict(fp, splitter=None).iteritems():
v = v.split()
shuffle(v)
mirrors[k] = v
except EnvironmentError as ee:
if ee.errno != errno.ENOENT:
raise
# use mirrors from masters if not defined in the repo
for master in masters:
for k, v in master.mirrors.iteritems():
if k not in mirrors:
mirrors[k] = v
if isinstance(cache, (tuple, list)):
cache = tuple(cache)
else:
cache = (cache,)
self.mirrors = mirrors
self.default_mirrors = default_mirrors
self.cache = cache
self.ignore_paludis_versioning = ignore_paludis_versioning
self._allow_missing_chksums = allow_missing_manifests
self.package_class = self.package_factory(
self, cache, self.eclass_cache, self.mirrors, self.default_mirrors)
self._shared_pkg_cache = WeakValCache()
repo_id = klass.alias_attr("config.repo_id")
def path_restrict(self, path):
"""Return a restriction from a given path in a repo.
:param path: full or partial path to an ebuild
:return: a package restriction matching the given path if possible
"""
realpath = os.path.realpath(path)
if realpath not in self:
raise ValueError("'%s' repo doesn't contain: '%s'" % (self.repo_id, path))
relpath = realpath[len(os.path.realpath(self.location)):].strip('/')
repo_path = relpath.split(os.path.sep) if relpath else []
restrictions = []
if os.path.isfile(realpath):
if not path.endswith('.ebuild'):
raise ValueError("file is not an ebuild: '%s'" % (path,))
elif len(repo_path) != 3:
# ebuild isn't in a category/PN directory
raise ValueError("ebuild not in the correct directory layout: '%s'" % (path,))
# add restrictions until path components run out
try:
restrictions.append(restricts.RepositoryDep(self.repo_id))
if repo_path[0] in self.categories:
restrictions.append(restricts.CategoryDep(repo_path[0]))
restrictions.append(restricts.PackageDep(repo_path[1]))
base = cpv.versioned_CPV("%s/%s" % (repo_path[0], os.path.splitext(repo_path[2])[0]))
restrictions.append(restricts.VersionMatch('=', base.version, rev=base.revision))
except IndexError:
pass
return packages.AndRestriction(*restrictions)
def __getitem__(self, cpv):
cpv_inst = self.package_class(*cpv)
if cpv_inst.fullver not in self.versions[(cpv_inst.category, cpv_inst.package)]:
if cpv_inst.revision is None:
if '%s-r0' % cpv_inst.fullver in \
self.versions[(cpv_inst.category, cpv_inst.package)]:
# ebuild on disk has an explicit -r0 in its name
return cpv_inst
raise KeyError(cpv)
return cpv_inst
def rebind(self, **kwds):
"""Generate a new tree instance with the same location using new keywords.
:param kwds: see __init__ for valid values
"""
o = self.__class__(self.location, **kwds)
o.categories = self.categories
o.packages = self.packages
o.versions = self.versions
return o
@klass.jit_attr
def hardcoded_categories(self):
# try reading $LOC/profiles/categories if it's available.
categories = readlines(
pjoin(self.base, 'profiles', 'categories'),
True, True, True)
if categories is not None:
categories = tuple(imap(intern, categories))
return categories
def _get_categories(self, *optional_category):
# why the auto return? current porttrees don't allow/support
# categories deeper then one dir.
if optional_category:
# raise KeyError
return ()
categories = set()
for repo in self.trees:
if repo.hardcoded_categories is not None:
categories.update(repo.hardcoded_categories)
if categories:
return tuple(categories)
try:
return tuple(imap(intern, ifilterfalse(
self.false_categories.__contains__,
(x for x in listdir_dirs(self.base) if x[0:1] != "."))))
except EnvironmentError as e:
raise_from(KeyError("failed fetching categories: %s" % str(e)))
def _get_packages(self, category):
cpath = pjoin(self.base, category.lstrip(os.path.sep))
try:
return tuple(ifilterfalse(
self.false_packages.__contains__, listdir_dirs(cpath)))
except EnvironmentError as e:
if e.errno == errno.ENOENT:
if category in self.categories:
# ignore it, since it's PMS mandated that it be allowed.
return ()
raise_from(KeyError(
"failed fetching packages for category %s: %s" %
(pjoin(self.base, category.lstrip(os.path.sep)), str(e))))
def _get_versions(self, catpkg):
cppath = pjoin(self.base, catpkg[0], catpkg[1])
pkg = catpkg[-1] + "-"
lp = len(pkg)
extension = self.extension
ext_len = -len(extension)
try:
ret = tuple(x[lp:ext_len] for x in listdir_files(cppath)
if x[ext_len:] == extension and x[:lp] == pkg)
if any(('scm' in x or '-try' in x) for x in ret):
if not self.ignore_paludis_versioning:
for x in ret:
if 'scm' in x:
raise ebuild_errors.InvalidCPV(
"%s/%s-%s has nonstandard -scm "
"version component" % (catpkg + (x,)))
elif 'try' in x:
raise ebuild_errors.InvalidCPV(
"%s/%s-%s has nonstandard -try "
"version component" % (catpkg + (x,)))
raise AssertionError('unreachable codepoint was reached')
return tuple(x for x in ret
if ('scm' not in x and 'try' not in x))
return ret
except EnvironmentError as e:
raise_from(KeyError(
"failed fetching versions for package %s: %s" %
(pjoin(self.base, '/'.join(catpkg)), str(e))))
def _get_ebuild_path(self, pkg):
if pkg.revision is None:
if pkg.fullver not in self.versions[(pkg.category, pkg.package)]:
# daft explicit -r0 on disk.
return pjoin(
self.base, pkg.category, pkg.package,
"%s-%s-r0%s" % (pkg.package, pkg.fullver, self.extension))
return pjoin(
self.base, pkg.category, pkg.package,
"%s-%s%s" % (pkg.package, pkg.fullver, self.extension))
def _get_ebuild_src(self, pkg):
return local_source(self._get_ebuild_path(pkg), encoding='utf8')
def _get_shared_pkg_data(self, category, package):
key = (category, package)
o = self._shared_pkg_cache.get(key)
if o is None:
mxml = self._get_metadata_xml(category, package)
manifest = self._get_manifest(category, package)
o = repo_objs.SharedPkgData(mxml, manifest)
self._shared_pkg_cache[key] = o
return o
def _get_metadata_xml(self, category, package):
return repo_objs.LocalMetadataXml(pjoin(
self.base, category, package, "metadata.xml"))
def _get_manifest(self, category, package):
return digest.Manifest(pjoin(
self.base, category, package, "Manifest"),
thin=self.config.manifests.thin,
enforce_gpg=self.enable_gpg)
def _get_digests(self, pkg, allow_missing=False):
if self.config.manifests.disabled:
return True, {}
try:
manifest = pkg._shared_pkg_data.manifest
manifest.allow_missing = allow_missing
return allow_missing, manifest.distfiles
except pkg_errors.ParseChksumError as e:
if e.missing and allow_missing:
return allow_missing, {}
raise
def __repr__(self):
return "<ebuild %s location=%r @%#8x>" % (
self.__class__.__name__, self.base, id(self))
def _visibility_limiters(self):
path = pjoin(self.base, 'profiles', 'package.mask')
pos, neg = [], []
try:
if self.config.profile_formats.intersection(['portage-1', 'portage-2']):
paths = sorted_scan(path)
else:
paths = [path]
for path in paths:
for line in iter_read_bash(path):
line = line.strip()
if line in ('-', ''):
raise profiles.ProfileError(
pjoin(self.base, 'profiles'),
'package.mask', "encountered empty negation: -")
if line.startswith('-'):
neg.append(atom.atom(line[1:]))
else:
pos.append(atom.atom(line))
except IOError as i:
if i.errno != errno.ENOENT:
raise
except ebuild_errors.MalformedAtom as ma:
raise_from(profiles.ProfileError(
pjoin(self.base, 'profiles'),
'package.mask', ma))
return [neg, pos]
def _regen_operation_helper(self, **kwds):
return _RegenOpHelper(
self, force=bool(kwds.get('force', False)),
eclass_caching=bool(kwds.get('eclass_caching', True)))
def __init__(self, location=None, eclassdir=None):
self._eclass_data_inst_cache = WeakValCache()
# generate this.
# self.eclasses = {} # {"Name": ("location", "_mtime_")}
self.location = location
self.eclassdir = eclassdir
def __init__(self,
location,
eclass_cache=None,
masters=(),
cache=(),
default_mirrors=None,
allow_missing_manifests=False,
repo_config=None):
"""
:param location: on disk location of the tree
:param cache: sequence of :obj:`pkgcore.cache.template.database` instances
to use for storing metadata
:param masters: repo masters this repo inherits from
:param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache`
instance representing the eclasses available,
if None, generates the eclass_cache itself
:param default_mirrors: Either None, or sequence of mirrors to try
fetching from first, then falling back to other uri
"""
super().__init__()
self.base = self.location = location
try:
if not stat.S_ISDIR(os.stat(self.base).st_mode):
raise errors.InitializationError(
f"base not a dir: {self.base}")
except OSError as e:
raise errors.InitializationError(
f"lstat failed: {self.base}") from e
if repo_config is None:
repo_config = repo_objs.RepoConfig(location)
self.config = repo_config
# profiles dir is required by PMS
if not os.path.isdir(self.config.profiles_base):
raise errors.InvalidRepo(
f'missing required profiles dir: {self.location!r}')
# verify we support the repo's EAPI
if not self.is_supported:
raise errors.UnsupportedRepo(self)
if eclass_cache is None:
eclass_cache = eclass_cache_mod.cache(pjoin(
self.location, 'eclass'),
location=self.location)
self.eclass_cache = eclass_cache
self.masters = masters
self.trees = tuple(masters) + (self, )
self.licenses = repo_objs.Licenses(self.location)
if masters:
self.licenses = repo_objs.OverlayedLicenses(*self.trees)
mirrors = {}
fp = pjoin(self.location, 'profiles', "thirdpartymirrors")
try:
for k, v in read_dict(fp, splitter=None).items():
v = v.split()
shuffle(v)
mirrors[k] = v
except FileNotFoundError:
pass
# use mirrors from masters if not defined in the repo
for master in masters:
for k, v in master.mirrors.items():
if k not in mirrors:
mirrors[k] = v
if isinstance(cache, (tuple, list)):
cache = tuple(cache)
else:
cache = (cache, )
self.mirrors = mirrors
self.default_mirrors = default_mirrors
self.cache = cache
self._allow_missing_chksums = allow_missing_manifests
self.package_class = self.package_factory(self, cache,
self.eclass_cache,
self.mirrors,
self.default_mirrors)
self._shared_pkg_cache = WeakValCache()
self._masked = RestrictionRepo(repo_id='masked')
class UnconfiguredTree(prototype.tree):
"""Raw implementation supporting standard ebuild tree.
Return packages don't have USE configuration bound to them.
"""
false_packages = frozenset(["CVS", ".svn"])
false_categories = frozenset([
"eclass", "profiles", "packages", "distfiles", "metadata", "licenses",
"scripts", "CVS", "local"
])
configured = False
configurables = ("domain", "settings")
configure = None
package_factory = staticmethod(ebuild_src.generate_new_factory)
enable_gpg = False
extension = '.ebuild'
operations_kls = repo_operations
pkgcore_config_type = ConfigHint(
{
'location': 'str',
'eclass_cache': 'ref:eclass_cache',
'masters': 'refs:repo',
'cache': 'refs:cache',
'default_mirrors': 'list',
'allow_missing_manifests': 'bool',
'repo_config': 'ref:repo_config',
},
typename='repo')
def __init__(self,
location,
eclass_cache=None,
masters=(),
cache=(),
default_mirrors=None,
allow_missing_manifests=False,
repo_config=None):
"""
:param location: on disk location of the tree
:param cache: sequence of :obj:`pkgcore.cache.template.database` instances
to use for storing metadata
:param masters: repo masters this repo inherits from
:param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache`
instance representing the eclasses available,
if None, generates the eclass_cache itself
:param default_mirrors: Either None, or sequence of mirrors to try
fetching from first, then falling back to other uri
"""
super().__init__()
self.base = self.location = location
try:
if not stat.S_ISDIR(os.stat(self.base).st_mode):
raise errors.InitializationError(
f"base not a dir: {self.base}")
except OSError as e:
raise errors.InitializationError(
f"lstat failed: {self.base}") from e
if repo_config is None:
repo_config = repo_objs.RepoConfig(location)
self.config = repo_config
# profiles dir is required by PMS
if not os.path.isdir(self.config.profiles_base):
raise errors.InvalidRepo(
f'missing required profiles dir: {self.location!r}')
# verify we support the repo's EAPI
if not self.is_supported:
raise errors.UnsupportedRepo(self)
if eclass_cache is None:
eclass_cache = eclass_cache_mod.cache(pjoin(
self.location, 'eclass'),
location=self.location)
self.eclass_cache = eclass_cache
self.masters = masters
self.trees = tuple(masters) + (self, )
self.licenses = repo_objs.Licenses(self.location)
if masters:
self.licenses = repo_objs.OverlayedLicenses(*self.trees)
mirrors = {}
fp = pjoin(self.location, 'profiles', "thirdpartymirrors")
try:
for k, v in read_dict(fp, splitter=None).items():
v = v.split()
shuffle(v)
mirrors[k] = v
except FileNotFoundError:
pass
# use mirrors from masters if not defined in the repo
for master in masters:
for k, v in master.mirrors.items():
if k not in mirrors:
mirrors[k] = v
if isinstance(cache, (tuple, list)):
cache = tuple(cache)
else:
cache = (cache, )
self.mirrors = mirrors
self.default_mirrors = default_mirrors
self.cache = cache
self._allow_missing_chksums = allow_missing_manifests
self.package_class = self.package_factory(self, cache,
self.eclass_cache,
self.mirrors,
self.default_mirrors)
self._shared_pkg_cache = WeakValCache()
self._masked = RestrictionRepo(repo_id='masked')
repo_id = klass.alias_attr("config.repo_id")
repo_name = klass.alias_attr("config.repo_name")
eapi = klass.alias_attr('config.eapi')
is_supported = klass.alias_attr('config.eapi.is_supported')
@klass.jit_attr
def known_arches(self):
"""Return all known arches for a repo (including masters)."""
return frozenset(
chain.from_iterable(r.config.known_arches for r in self.trees))
def path_restrict(self, path):
"""Return a restriction from a given path in a repo.
:param path: full or partial path to an ebuild
:return: a package restriction matching the given path if possible
:raises ValueError: if the repo doesn't contain the given path, the
path relates to a file that isn't an ebuild, or the ebuild isn't in the
proper directory layout
"""
realpath = os.path.realpath(path)
if realpath not in self:
raise ValueError(
f"{self.repo_id!r} repo doesn't contain: {path!r}")
relpath = realpath[len(os.path.realpath(self.location)):].strip('/')
repo_path = relpath.split(os.path.sep) if relpath else []
restrictions = []
if os.path.isfile(realpath):
if not path.endswith('.ebuild'):
raise ValueError(f"file is not an ebuild: {path!r}")
elif len(repo_path) != 3:
# ebuild isn't in a category/PN directory
raise ValueError(
f"ebuild not in the correct directory layout: {path!r}")
# add restrictions until path components run out
try:
restrictions.append(restricts.RepositoryDep(self.repo_id))
if repo_path[0] in self.categories:
restrictions.append(restricts.CategoryDep(repo_path[0]))
restrictions.append(restricts.PackageDep(repo_path[1]))
base = cpv.versioned_CPV(
f"{repo_path[0]}/{os.path.splitext(repo_path[2])[0]}")
restrictions.append(
restricts.VersionMatch('=',
base.version,
rev=base.revision))
except IndexError:
pass
return packages.AndRestriction(*restrictions)
def __getitem__(self, cpv):
cpv_inst = self.package_class(*cpv)
if cpv_inst.fullver not in self.versions[(cpv_inst.category,
cpv_inst.package)]:
if cpv_inst.revision is None:
if f'{cpv_inst.fullver}-r0' in \
self.versions[(cpv_inst.category, cpv_inst.package)]:
# ebuild on disk has an explicit -r0 in its name
return cpv_inst
raise KeyError(cpv)
return cpv_inst
def rebind(self, **kwds):
"""Generate a new tree instance with the same location using new keywords.
:param kwds: see __init__ for valid values
"""
o = self.__class__(self.location, **kwds)
o.categories = self.categories
o.packages = self.packages
o.versions = self.versions
return o
@klass.jit_attr
def hardcoded_categories(self):
# try reading $LOC/profiles/categories if it's available.
categories = readlines(pjoin(self.base, 'profiles', 'categories'),
True, True, True)
if categories is not None:
categories = tuple(map(intern, categories))
return categories
def _get_categories(self, *optional_category):
# why the auto return? current porttrees don't allow/support
# categories deeper then one dir.
if optional_category:
# raise KeyError
return ()
categories = set()
for repo in self.trees:
if repo.hardcoded_categories is not None:
categories.update(repo.hardcoded_categories)
if categories:
return tuple(categories)
try:
return tuple(
map(
intern,
filterfalse(
self.false_categories.__contains__,
(x
for x in listdir_dirs(self.base) if x[0:1] != "."))))
except EnvironmentError as e:
raise KeyError(f"failed fetching categories: {e}") from e
def _get_packages(self, category):
cpath = pjoin(self.base, category.lstrip(os.path.sep))
try:
return tuple(
filterfalse(self.false_packages.__contains__,
listdir_dirs(cpath)))
except FileNotFoundError:
if category in self.categories:
# ignore it, since it's PMS mandated that it be allowed.
return ()
except EnvironmentError as e:
category = pjoin(self.base, category.lstrip(os.path.sep))
raise KeyError(
f'failed fetching packages for category {category}: {e}'
) from e
def _get_versions(self, catpkg):
cppath = pjoin(self.base, catpkg[0], catpkg[1])
pkg = f'{catpkg[-1]}-'
lp = len(pkg)
extension = self.extension
ext_len = -len(extension)
try:
return tuple(x[lp:ext_len] for x in listdir_files(cppath)
if x[ext_len:] == extension and x[:lp] == pkg)
except EnvironmentError as e:
raise KeyError("failed fetching versions for package %s: %s" %
(pjoin(self.base, '/'.join(catpkg)), str(e))) from e
def _pkg_filter(self, pkgs):
"""Filter packages with bad metadata."""
for pkg in pkgs:
if pkg not in self._masked.itermatch(pkg.versioned_atom):
# check pkgs for unsupported/invalid EAPIs and bad metadata
try:
if not pkg.is_supported:
self._masked[pkg.versioned_atom] = MetadataException(
pkg, 'eapi', f"EAPI '{pkg.eapi}' is not supported")
continue
# TODO: add a generic metadata validation method to avoid slow metadata checks?
pkg.data
pkg.required_use
except MetadataException as e:
self._masked[e.pkg.versioned_atom] = e
continue
yield pkg
def itermatch(self, *args, **kwargs):
kwargs.setdefault('pkg_filter', self._pkg_filter)
return super().itermatch(*args, **kwargs)
def _get_ebuild_path(self, pkg):
if pkg.revision is None:
try:
if pkg.fullver not in self.versions[(pkg.category,
pkg.package)]:
# daft explicit -r0 on disk.
return pjoin(
self.base, pkg.category, pkg.package,
f"{pkg.package}-{pkg.fullver}-r0{self.extension}")
except KeyError as e:
raise MetadataException(pkg, 'package',
'mismatched ebuild name') from e
return pjoin(self.base, pkg.category, pkg.package,
f"{pkg.package}-{pkg.fullver}{self.extension}")
def _get_ebuild_src(self, pkg):
return local_source(self._get_ebuild_path(pkg), encoding='utf8')
def _get_shared_pkg_data(self, category, package):
key = (category, package)
o = self._shared_pkg_cache.get(key)
if o is None:
mxml = self._get_metadata_xml(category, package)
manifest = self._get_manifest(category, package)
o = repo_objs.SharedPkgData(mxml, manifest)
self._shared_pkg_cache[key] = o
return o
def _get_metadata_xml(self, category, package):
return repo_objs.LocalMetadataXml(
pjoin(self.base, category, package, "metadata.xml"))
def _get_manifest(self, category, package):
return digest.Manifest(pjoin(self.base, category, package, "Manifest"),
thin=self.config.manifests.thin,
enforce_gpg=self.enable_gpg)
def _get_digests(self, pkg, allow_missing=False):
if self.config.manifests.disabled:
return True, {}
try:
manifest = pkg._shared_pkg_data.manifest
manifest.allow_missing = allow_missing
return allow_missing, manifest.distfiles
except pkg_errors.ParseChksumError as e:
if e.missing and allow_missing:
return allow_missing, {}
raise
def __repr__(self):
return "<ebuild %s location=%r @%#8x>" % (self.__class__.__name__,
self.base, id(self))
@klass.jit_attr
def _visibility_limiters(self):
path = pjoin(self.base, 'profiles', 'package.mask')
pos, neg = [], []
try:
if (self.config.eapi.options['has_profile_data_dirs']
or self.config.profile_formats.intersection(
['portage-1', 'portage-2'])):
paths = sorted_scan(path)
else:
paths = [path]
for path in paths:
for line in iter_read_bash(path):
line = line.strip()
if line in ('-', ''):
raise profiles.ProfileError(
pjoin(self.base, 'profiles'), 'package.mask',
"encountered empty negation: -")
if line.startswith('-'):
neg.append(atom.atom(line[1:]))
else:
pos.append(atom.atom(line))
except FileNotFoundError:
pass
except ebuild_errors.MalformedAtom as e:
raise profiles.ProfileError(pjoin(self.base, 'profiles'),
'package.mask', e) from e
return tuple(neg), tuple(pos)
def _regen_operation_helper(self, **kwds):
return _RegenOpHelper(self,
force=bool(kwds.get('force', False)),
eclass_caching=bool(
kwds.get('eclass_caching', True)))
def __setstate__(self, state):
self.__dict__ = state.copy()
self.__dict__['_eclass_data_inst_cache'] = WeakValCache()
def setUp(self):
self.o = RefObj()
self.w = WeakValCache()
def __setstate__(self, state):
self.__dict__ = state.copy()
self.__dict__['_shared_pkg_cache'] = WeakValCache()
def __init__(self, location, eclass_cache=None, masters=(), cache=(),
default_mirrors=None, allow_missing_manifests=False, repo_config=None):
"""
:param location: on disk location of the tree
:param cache: sequence of :obj:`pkgcore.cache.template.database` instances
to use for storing metadata
:param masters: repo masters this repo inherits from
:param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache`
instance representing the eclasses available,
if None, generates the eclass_cache itself
:param default_mirrors: Either None, or sequence of mirrors to try
fetching from first, then falling back to other uri
"""
super().__init__()
self.base = self.location = location
try:
if not stat.S_ISDIR(os.stat(self.base).st_mode):
raise errors.InitializationError(f"base not a dir: {self.base}")
except OSError as e:
raise errors.InitializationError(f"lstat failed: {self.base}") from e
if repo_config is None:
repo_config = repo_objs.RepoConfig(location)
self.config = repo_config
# profiles dir is required by PMS
if not os.path.isdir(self.config.profiles_base):
raise errors.InvalidRepo(f'missing required profiles dir: {self.location!r}')
# verify we support the repo's EAPI
if not self.is_supported:
raise errors.UnsupportedRepo(self)
if eclass_cache is None:
eclass_cache = eclass_cache_mod.cache(
pjoin(self.location, 'eclass'), location=self.location)
self.eclass_cache = eclass_cache
self.masters = masters
self.trees = tuple(masters) + (self,)
self.licenses = repo_objs.Licenses(self.location)
if masters:
self.licenses = repo_objs.OverlayedLicenses(*self.trees)
mirrors = {}
fp = pjoin(self.location, 'profiles', "thirdpartymirrors")
try:
for k, v in read_dict(fp, splitter=None).items():
v = v.split()
shuffle(v)
mirrors[k] = v
except FileNotFoundError:
pass
# use mirrors from masters if not defined in the repo
for master in masters:
for k, v in master.mirrors.items():
if k not in mirrors:
mirrors[k] = v
if isinstance(cache, (tuple, list)):
cache = tuple(cache)
else:
cache = (cache,)
self.mirrors = mirrors
self.default_mirrors = default_mirrors
self.cache = cache
self._allow_missing_chksums = allow_missing_manifests
self.package_class = self.package_factory(
self, cache, self.eclass_cache, self.mirrors, self.default_mirrors)
self._shared_pkg_cache = WeakValCache()
self._masked = RestrictionRepo(repo_id='masked')
class UnconfiguredTree(prototype.tree):
"""Raw implementation supporting standard ebuild tree.
Return packages don't have USE configuration bound to them.
"""
false_packages = frozenset(["CVS", ".svn"])
false_categories = frozenset([
"eclass", "profiles", "packages", "distfiles", "metadata",
"licenses", "scripts", "CVS", "local"])
configured = False
configurables = ("domain", "settings")
configure = None
package_factory = staticmethod(ebuild_src.generate_new_factory)
enable_gpg = False
extension = '.ebuild'
operations_kls = repo_operations
pkgcore_config_type = ConfigHint({
'location': 'str',
'eclass_cache': 'ref:eclass_cache',
'masters': 'refs:repo',
'cache': 'refs:cache',
'default_mirrors': 'list',
'allow_missing_manifests': 'bool',
'repo_config': 'ref:repo_config',
},
typename='repo')
def __init__(self, location, eclass_cache=None, masters=(), cache=(),
default_mirrors=None, allow_missing_manifests=False, repo_config=None):
"""
:param location: on disk location of the tree
:param cache: sequence of :obj:`pkgcore.cache.template.database` instances
to use for storing metadata
:param masters: repo masters this repo inherits from
:param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache`
instance representing the eclasses available,
if None, generates the eclass_cache itself
:param default_mirrors: Either None, or sequence of mirrors to try
fetching from first, then falling back to other uri
"""
super().__init__()
self.base = self.location = location
try:
if not stat.S_ISDIR(os.stat(self.base).st_mode):
raise errors.InitializationError(f"base not a dir: {self.base}")
except OSError as e:
raise errors.InitializationError(f"lstat failed: {self.base}") from e
if repo_config is None:
repo_config = repo_objs.RepoConfig(location)
self.config = repo_config
# profiles dir is required by PMS
if not os.path.isdir(self.config.profiles_base):
raise errors.InvalidRepo(f'missing required profiles dir: {self.location!r}')
# verify we support the repo's EAPI
if not self.is_supported:
raise errors.UnsupportedRepo(self)
if eclass_cache is None:
eclass_cache = eclass_cache_mod.cache(
pjoin(self.location, 'eclass'), location=self.location)
self.eclass_cache = eclass_cache
self.masters = masters
self.trees = tuple(masters) + (self,)
self.licenses = repo_objs.Licenses(self.location)
if masters:
self.licenses = repo_objs.OverlayedLicenses(*self.trees)
mirrors = {}
fp = pjoin(self.location, 'profiles', "thirdpartymirrors")
try:
for k, v in read_dict(fp, splitter=None).items():
v = v.split()
shuffle(v)
mirrors[k] = v
except FileNotFoundError:
pass
# use mirrors from masters if not defined in the repo
for master in masters:
for k, v in master.mirrors.items():
if k not in mirrors:
mirrors[k] = v
if isinstance(cache, (tuple, list)):
cache = tuple(cache)
else:
cache = (cache,)
self.mirrors = mirrors
self.default_mirrors = default_mirrors
self.cache = cache
self._allow_missing_chksums = allow_missing_manifests
self.package_class = self.package_factory(
self, cache, self.eclass_cache, self.mirrors, self.default_mirrors)
self._shared_pkg_cache = WeakValCache()
self._masked = RestrictionRepo(repo_id='masked')
repo_id = klass.alias_attr("config.repo_id")
repo_name = klass.alias_attr("config.repo_name")
eapi = klass.alias_attr('config.eapi')
is_supported = klass.alias_attr('config.eapi.is_supported')
@klass.jit_attr
def known_arches(self):
"""Return all known arches for a repo (including masters)."""
return frozenset(chain.from_iterable(
r.config.known_arches for r in self.trees))
def path_restrict(self, path):
"""Return a restriction from a given path in a repo.
:param path: full or partial path to an ebuild
:return: a package restriction matching the given path if possible
:raises ValueError: if the repo doesn't contain the given path, the
path relates to a file that isn't an ebuild, or the ebuild isn't in the
proper directory layout
"""
realpath = os.path.realpath(path)
if realpath not in self:
raise ValueError(f"{self.repo_id!r} repo doesn't contain: {path!r}")
relpath = realpath[len(os.path.realpath(self.location)):].strip('/')
repo_path = relpath.split(os.path.sep) if relpath else []
restrictions = []
if os.path.isfile(realpath):
if not path.endswith('.ebuild'):
raise ValueError(f"file is not an ebuild: {path!r}")
elif len(repo_path) != 3:
# ebuild isn't in a category/PN directory
raise ValueError(
f"ebuild not in the correct directory layout: {path!r}")
# add restrictions until path components run out
try:
restrictions.append(restricts.RepositoryDep(self.repo_id))
if repo_path[0] in self.categories:
restrictions.append(restricts.CategoryDep(repo_path[0]))
restrictions.append(restricts.PackageDep(repo_path[1]))
base = cpv.versioned_CPV(f"{repo_path[0]}/{os.path.splitext(repo_path[2])[0]}")
restrictions.append(restricts.VersionMatch('=', base.version, rev=base.revision))
except IndexError:
pass
return packages.AndRestriction(*restrictions)
def __getitem__(self, cpv):
cpv_inst = self.package_class(*cpv)
if cpv_inst.fullver not in self.versions[(cpv_inst.category, cpv_inst.package)]:
if cpv_inst.revision is None:
if f'{cpv_inst.fullver}-r0' in \
self.versions[(cpv_inst.category, cpv_inst.package)]:
# ebuild on disk has an explicit -r0 in its name
return cpv_inst
raise KeyError(cpv)
return cpv_inst
def rebind(self, **kwds):
"""Generate a new tree instance with the same location using new keywords.
:param kwds: see __init__ for valid values
"""
o = self.__class__(self.location, **kwds)
o.categories = self.categories
o.packages = self.packages
o.versions = self.versions
return o
@klass.jit_attr
def hardcoded_categories(self):
# try reading $LOC/profiles/categories if it's available.
categories = readlines(
pjoin(self.base, 'profiles', 'categories'),
True, True, True)
if categories is not None:
categories = tuple(map(intern, categories))
return categories
def _get_categories(self, *optional_category):
# why the auto return? current porttrees don't allow/support
# categories deeper then one dir.
if optional_category:
# raise KeyError
return ()
categories = set()
for repo in self.trees:
if repo.hardcoded_categories is not None:
categories.update(repo.hardcoded_categories)
if categories:
return tuple(categories)
try:
return tuple(map(intern, filterfalse(
self.false_categories.__contains__,
(x for x in listdir_dirs(self.base) if x[0:1] != "."))))
except EnvironmentError as e:
raise KeyError(f"failed fetching categories: {e}") from e
def _get_packages(self, category):
cpath = pjoin(self.base, category.lstrip(os.path.sep))
try:
return tuple(filterfalse(
self.false_packages.__contains__, listdir_dirs(cpath)))
except FileNotFoundError:
if category in self.categories:
# ignore it, since it's PMS mandated that it be allowed.
return ()
except EnvironmentError as e:
category = pjoin(self.base, category.lstrip(os.path.sep))
raise KeyError(
f'failed fetching packages for category {category}: {e}') from e
def _get_versions(self, catpkg):
cppath = pjoin(self.base, catpkg[0], catpkg[1])
pkg = f'{catpkg[-1]}-'
lp = len(pkg)
extension = self.extension
ext_len = -len(extension)
try:
return tuple(
x[lp:ext_len] for x in listdir_files(cppath)
if x[ext_len:] == extension and x[:lp] == pkg)
except EnvironmentError as e:
raise KeyError(
"failed fetching versions for package %s: %s" %
(pjoin(self.base, '/'.join(catpkg)), str(e))) from e
def _pkg_filter(self, pkgs):
"""Filter packages with bad metadata."""
for pkg in pkgs:
if pkg not in self._masked.itermatch(pkg.versioned_atom):
# check pkgs for unsupported/invalid EAPIs and bad metadata
try:
if not pkg.is_supported:
self._masked[pkg.versioned_atom] = MetadataException(
pkg, 'eapi', f"EAPI '{pkg.eapi}' is not supported")
continue
# TODO: add a generic metadata validation method to avoid slow metadata checks?
pkg.data
pkg.required_use
except MetadataException as e:
self._masked[e.pkg.versioned_atom] = e
continue
yield pkg
def itermatch(self, *args, **kwargs):
kwargs.setdefault('pkg_filter', self._pkg_filter)
return super().itermatch(*args, **kwargs)
def _get_ebuild_path(self, pkg):
if pkg.revision is None:
try:
if pkg.fullver not in self.versions[(pkg.category, pkg.package)]:
# daft explicit -r0 on disk.
return pjoin(
self.base, pkg.category, pkg.package,
f"{pkg.package}-{pkg.fullver}-r0{self.extension}")
except KeyError as e:
raise MetadataException(pkg, 'package', 'mismatched ebuild name') from e
return pjoin(
self.base, pkg.category, pkg.package,
f"{pkg.package}-{pkg.fullver}{self.extension}")
def _get_ebuild_src(self, pkg):
return local_source(self._get_ebuild_path(pkg), encoding='utf8')
def _get_shared_pkg_data(self, category, package):
key = (category, package)
o = self._shared_pkg_cache.get(key)
if o is None:
mxml = self._get_metadata_xml(category, package)
manifest = self._get_manifest(category, package)
o = repo_objs.SharedPkgData(mxml, manifest)
self._shared_pkg_cache[key] = o
return o
def _get_metadata_xml(self, category, package):
return repo_objs.LocalMetadataXml(pjoin(
self.base, category, package, "metadata.xml"))
def _get_manifest(self, category, package):
return digest.Manifest(pjoin(
self.base, category, package, "Manifest"),
thin=self.config.manifests.thin,
enforce_gpg=self.enable_gpg)
def _get_digests(self, pkg, allow_missing=False):
if self.config.manifests.disabled:
return True, {}
try:
manifest = pkg._shared_pkg_data.manifest
manifest.allow_missing = allow_missing
return allow_missing, manifest.distfiles
except pkg_errors.ParseChksumError as e:
if e.missing and allow_missing:
return allow_missing, {}
raise
def __repr__(self):
return "<ebuild %s location=%r @%#8x>" % (
self.__class__.__name__, self.base, id(self))
@klass.jit_attr
def _visibility_limiters(self):
path = pjoin(self.base, 'profiles', 'package.mask')
pos, neg = [], []
try:
if (self.config.eapi.options['has_profile_data_dirs'] or
self.config.profile_formats.intersection(['portage-1', 'portage-2'])):
paths = sorted_scan(path)
else:
paths = [path]
for path in paths:
for line in iter_read_bash(path):
line = line.strip()
if line in ('-', ''):
raise profiles.ProfileError(
pjoin(self.base, 'profiles'),
'package.mask', "encountered empty negation: -")
if line.startswith('-'):
neg.append(atom.atom(line[1:]))
else:
pos.append(atom.atom(line))
except FileNotFoundError:
pass
except ebuild_errors.MalformedAtom as e:
raise profiles.ProfileError(
pjoin(self.base, 'profiles'), 'package.mask', e) from e
return tuple(neg), tuple(pos)
def _regen_operation_helper(self, **kwds):
return _RegenOpHelper(
self, force=bool(kwds.get('force', False)),
eclass_caching=bool(kwds.get('eclass_caching', True)))
def setup_method(self, method):
self.o = RefObj()
self.w = WeakValCache()
class TestWeakValCache(object):
def setup_method(self, method):
self.o = RefObj()
self.w = WeakValCache()
def test_setitem(self):
s = "asdf"
self.w[s] = self.o
self.w["fds"] = self.o
self.w[s] = self.o
def test_getitem(self):
s = "asdf"
self.w[s] = self.o
assert self.w[s] is self.o
def test_expiring(self):
s = "asdf"
self.w[s] = self.o
assert self.w[s]
del self.o
with pytest.raises(KeyError):
self.w.__getitem__(s)
def test_get(self):
s = "asdf"
with pytest.raises(KeyError):
self.w.__getitem__(s)
self.w[s] = self.o
assert self.w.get(s) is self.o
def test_keys(self):
assert list(self.w.keys()) == []
self.w['a'] = self.o
self.w['b'] = self.o
self.w['c'] = self.o
assert sorted(self.w.keys()) == ['a', 'b', 'c']
del self.o
assert self.w.keys() == []
def test_values(self):
assert list(self.w.values()) == []
self.w['a'] = self.o
self.w['b'] = self.o
self.w['c'] = self.o
assert len(iter(self.w.values())) == 3
del self.o
assert self.w.values() == []
def test_items(self):
assert list(self.w.items()) == []
self.w['a'] = self.o
self.w['b'] = self.o
self.w['c'] = self.o
assert len(iter(self.w.items())) == 3
del self.o
assert self.w.items() == []
def __init__(self, portdir=None, eclassdir=None):
self._eclass_data_inst_cache = WeakValCache()
# generate this.
# self.eclasses = {} # {"Name": ("location", "_mtime_")}
self.portdir = portdir
self.eclassdir = eclassdir
def __init__(self,
location,
eclass_cache,
cache=(),
default_mirrors=None,
override_repo_id=None,
ignore_paludis_versioning=False,
allow_missing_manifests=False,
repo_config=None):
"""
:param location: on disk location of the tree
:param cache: sequence of :obj:`pkgcore.cache.template.database` instances
to use for storing metadata
:param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache`
instance representing the eclasses available,
if None, generates the eclass_cache itself
:param default_mirrors: Either None, or sequence of mirrors to try
fetching from first, then falling back to other uri
:param override_repo_id: Either None, or string to force as the
repository unique id
:param ignore_paludis_versioning: If False, fail when -scm is encountred. if True,
silently ignore -scm ebuilds.
"""
prototype.tree.__init__(self)
if repo_config is None:
repo_config = repo_objs.RepoConfig(location)
self.config = repo_config
self._repo_id = override_repo_id
self.base = self.location = location
try:
if not stat.S_ISDIR(os.stat(self.base).st_mode):
raise errors.InitializationError("base not a dir: %s" %
self.base)
except OSError:
raise_from(
errors.InitializationError("lstat failed on base %s" %
(self.base, )))
self.eclass_cache = eclass_cache
self.licenses = repo_objs.Licenses(location)
fp = pjoin(self.base, metadata_offset, "thirdpartymirrors")
mirrors = {}
try:
for k, v in read_dict(fp, splitter=None).iteritems():
v = v.split()
shuffle(v)
mirrors[k] = v
except EnvironmentError as ee:
if ee.errno != errno.ENOENT:
raise
if isinstance(cache, (tuple, list)):
cache = tuple(cache)
else:
cache = (cache, )
self.mirrors = mirrors
self.default_mirrors = default_mirrors
self.cache = cache
self.ignore_paludis_versioning = ignore_paludis_versioning
self._allow_missing_chksums = allow_missing_manifests
self.package_class = self.package_factory(self, cache,
self.eclass_cache,
self.mirrors,
self.default_mirrors)
self._shared_pkg_cache = WeakValCache()
