def test_builtin_full_override(self):
# check our assumptions...
# and yes, the signatures below are for file rather than
# dict; we need a cpy class for the test, the ConfigHint doesn't
# have to be accurate however
class cls(dict):
__slots__ = ()
self.assertRaises(TypeError, basics.ConfigType, cls)
raw_hint = ConfigHint(types={
"filename": "str",
"mode": "r",
"buffering": "int"
},
typename='file',
required=['filename'],
positional=['filename'])
# make sure it still tries to introspect, and throws typeerror.
# introspection is generally wanted- if it must be skipped, the
# ConfigHint must make it explicit
cls.pkgcore_config_type = raw_hint
self.assertRaises(TypeError, basics.ConfigType, cls)
cls.pkgcore_config_type = raw_hint.clone(authorative=True)
conf = basics.ConfigType(cls)
self.assertEqual(conf.name, 'file')
self.assertEqual(list(conf.required), ['filename'])
self.assertEqual(list(conf.positional), ['filename'])
self.assertEqual(sorted(conf.types), ['buffering', 'filename', 'mode'])
def test_clone(self):
c = ConfigHint(types={'foo':'list', 'one':'str'},
positional=['one'], required=['one'],
typename='barn', doc='orig doc')
c2 = c.clone(types={'foo':'list', 'one':'str', 'two':'str'},
required=['one', 'two'])
self.assertEqual(c2.types, {'foo':'list', 'one':'str', 'two':'str'})
self.assertEqual(c2.positional, c.positional)
self.assertEqual(c2.required, ['one', 'two'])
self.assertEqual(c2.typename, c.typename)
self.assertEqual(c2.allow_unknowns, c.allow_unknowns)
self.assertEqual(c2.doc, c.doc)
class EclassConsumerSet(VersionedInstalled):
pkgcore_config_type = ConfigHint(
typename='pkgset',
types={
'vdb': 'refs:repo',
'repos': 'refs:repo',
'eclasses': 'list'
},
)
def __init__(self, vdb, repos, eclasses):
VersionedInstalled.__init__(self, vdb)
self.repos = RepositoryGroup(repos)
self.eclasses = frozenset(eclasses)
def __iter__(self):
for atom in VersionedInstalled.__iter__(self):
pkgs = self.repos.match(atom)
if not pkgs:
# pkg is installed but no longer in any repo, just ignore it.
continue
assert len(
pkgs) == 1, 'I do not know what I am doing: %r' % (pkgs, )
pkg = pkgs[0]
if self.eclasses.isdisjoint(pkg.data.get('_eclasses_', ())):
yield atom
class FakeRepo(FakeEbuildRepo):
pkgcore_config_type = ConfigHint({}, typename='repo')
def __init__(self,
repo_id='faker',
arches=('amd64', 'x86', 'arm', 'arm64')):
config = RepoConfig('nonexistent')
object.__setattr__(config, 'raw_known_arches', frozenset(arches))
pkgs = [
FakePkg('app-arch/bzip2-1.0.1-r1',
repo=self,
data={'SLOT': '0'},
keywords=('x86', )),
FakePkg('app-arch/bzip2-1.0.5-r2',
repo=self,
data={'SLOT': '0'},
keywords=('x86', )),
FakePkg('sys-apps/coreutils-8.25', repo=self, data={'SLOT': '0'}),
FakePkg('x11-libs/gtk+-2.24',
repo=self,
data={'SLOT': '2'},
keywords=('amd64', )),
FakePkg('x11-libs/gtk+-3.22',
repo=self,
data={'SLOT': '3'},
keywords=('amd64', 'x86')),
]
super().__init__(repo_id=repo_id, pkgs=pkgs, config=config)
class cache(base):
pkgcore_config_type = ConfigHint({"path":"str", "location":"str"},
typename='eclass_cache')
def __init__(self, path, location=None):
"""
:param location: ondisk location of the tree we're working with
"""
base.__init__(self, location=location, eclassdir=normpath(path))
def _load_eclasses(self):
"""Force an update of the internal view of on disk/remote eclasses."""
ec = {}
eclass_len = len(".eclass")
try:
files = listdir_files(self.eclassdir)
except (FileNotFoundError, NotADirectoryError):
return ImmutableDict()
for y in files:
if not y.endswith(".eclass"):
continue
ys = y[:-eclass_len]
ec[intern(ys)] = LazilyHashedPath(
pjoin(self.eclassdir, y), eclassdir=self.eclassdir)
return ImmutableDict(ec)
class EclassConsumerSet(VersionedInstalled):
pkgcore_config_type = ConfigHint(
{
'vdb': 'refs:repo',
'portdir': 'ref:repo',
'eclasses': 'list'
},
typename='pkgset')
def __init__(self, vdb, portdir, eclasses):
VersionedInstalled.__init__(self, vdb)
self.portdir = portdir
self.eclasses = frozenset(eclasses)
def __iter__(self):
for atom in VersionedInstalled.__iter__(self):
pkgs = self.portdir.match(atom)
if not pkgs:
# This thing is in the vdb but no longer in portdir
# (or someone misconfigured us to use a bogus
# portdir). Just ignore it.
continue
assert len(
pkgs) == 1, 'I do not know what I am doing: %r' % (pkgs, )
pkg = pkgs[0]
if self.eclasses.isdisjoint(pkg.data.get('_eclasses_', ())):
yield atom
class WorldFile(FileList):
"""Set of packages contained in the world file."""
pkgcore_config_type = ConfigHint(typename='pkgset')
error_on_subsets = False
def __init__(self,
location=const.WORLD_FILE,
gid=os_data.portage_gid,
mode=0644):
FileList.__init__(self, location, gid=gid, mode=mode)
def add(self, atom_inst):
self._modify(atom_inst, FileList.add)
def remove(self, atom_inst):
self._modify(atom_inst, FileList.remove)
def _modify(self, atom_inst, func):
if atom_inst.slot:
for slot in atom_inst.slot:
if slot == '0':
new_atom_inst = atom(atom_inst.key)
else:
new_atom_inst = atom(atom_inst.key + ":" + slot)
func(self, new_atom_inst)
else:
atom_inst = atom(atom_inst.key)
func(self, atom_inst)
class SecurityUpgrades(object):
"""
pkgset that can be used directly from pkgcore configuration.
generates set of restrictions of required upgrades.
"""
pkgcore_config_type = ConfigHint(
{
'ebuild_repo': 'ref:repo',
'vdb': 'ref:vdb'
}, typename='pkgset')
__metaclass__ = generic_equality
__attr_comparison__ = ('arch', 'glsa_src', 'vdb')
def __init__(self, ebuild_repo, vdb, arch):
self.glsa_src = GlsaDirSet(ebuild_repo)
self.vdb = vdb
self.arch = arch
def __iter__(self):
for glsa, matches in find_vulnerable_repo_pkgs(self.glsa_src,
self.vdb,
grouped=True,
arch=self.arch):
yield packages.KeyedAndRestriction(glsa[0],
restriction.Negate(glsa[1]))
class paludis_flat_list(database):
"""(Hopefully) write a paludis specific form of flat_list format cache.
Not very well tested.
Difference from a normal flat_list cache is that mtime is set to ebuild
for normal, for paludis it's max mtime of eclasses/ebuild involved.
"""
pkgcore_config_type = ConfigHint(
{
'readonly': 'bool',
'location': 'str',
'label': 'str'
},
required=['location'],
positional=['location'],
typename='cache')
def __init__(self, *args, **config):
config['auxdbkeys'] = self.auxdbkeys_order
database.__init__(self, *args, **config)
def _set_mtime(self, fp, values, eclasses):
mtime = values.get("_mtime_", 0)
if eclasses:
self._ensure_access(
fp,
mtime=max(max(mtime for path, mtime in eclasses.itervalues()),
mtime))
else:
self._ensure_access(fp, mtime)
class FakeDomain(object):
pkgcore_config_type = ConfigHint({'repo': 'ref:repo'}, typename='domain')
def __init__(self, repo):
object.__init__(self)
self.ebuild_repos_unfiltered = repo
class StackedCaches(base):
"""
collapse multiple eclass caches into one.
Does L->R searching for eclass matches.
"""
pkgcore_config_type = ConfigHint(
{
'caches': 'refs:eclass_cache',
'portdir': 'str',
'eclassdir': 'str'
},
typename='eclass_cache')
def __init__(self, caches, **kwds):
"""
:param caches: :obj:`cache` instances to stack;
ordering should be desired lookup order
:keyword eclassdir: override for the master eclass dir, required for
eapi0 and idiot eclass usage. defaults to pulling from the first
cache.
"""
if len(caches) < 2:
raise TypeError("%s requires at least two eclass_caches" %
self.__class__)
kwds.setdefault("eclassdir", caches[0].eclassdir)
kwds.setdefault("portdir",
os.path.dirname(kwds["eclassdir"].rstrip(os.path.sep)))
self._caches = caches
base.__init__(self, **kwds)
def _load_eclasses(self):
return StackedDict(*[ec.eclasses for ec in self._caches])
class cache(base):
pkgcore_config_type = ConfigHint({
"path": "str",
"portdir": "str"
},
typename='eclass_cache')
def __init__(self, path, portdir=None):
"""
:param portdir: ondisk location of the tree we're working with
"""
base.__init__(self, portdir=portdir, eclassdir=normpath(path))
def _load_eclasses(self):
"""Force an update of the internal view of on disk/remote eclasses."""
ec = {}
eclass_len = len(".eclass")
try:
files = listdir_files(self.eclassdir)
except EnvironmentError as e:
if e.errno not in (errno.ENOENT, errno.ENOTDIR):
raise
return ImmutableDict()
for y in files:
if not y.endswith(".eclass"):
continue
ys = y[:-eclass_len]
ec[intern(ys)] = LazilyHashedPath(pjoin(self.eclassdir, y),
eclassdir=self.eclassdir)
return ImmutableDict(ec)
class FileList(object):
pkgcore_config_type = ConfigHint({'location': 'str'}, typename='pkgset')
error_on_subsets = True
def __init__(self, location, gid=os_data.portage_gid, mode=0644):
self.path = location
self.gid = gid
self.mode = mode
# note that _atoms is generated on the fly.
@klass.jit_attr
def _atoms(self):
try:
s = set()
for x in readlines_ascii(self.path, True):
if not x or x.startswith("#"):
continue
elif x.startswith("@"):
if self.error_on_subsets:
raise ValueError(
"set %s isn't a valid atom in pkgset %r" %
(x, self.path))
logger.warning(
"set item %r found in pkgset %r: it will be "
"wiped on update since portage/pkgcore store set items "
"in a separate way", x[1:], self.path)
continue
s.add(atom(x))
except InvalidDependency as e:
compatibility.raise_from(
errors.ParsingError("parsing %r" % self.path, exception=e))
return s
def __iter__(self):
return iter(self._atoms)
def __len__(self):
return len(self._atoms)
def __contains__(self, key):
return key in self._atoms
def add(self, atom_inst):
self._atoms.add(atom_inst)
def remove(self, atom_inst):
self._atoms.remove(atom_inst)
def flush(self):
f = None
try:
f = AtomicWriteFile(self.path, gid=self.gid, perms=self.mode)
f.write("\n".join(str(x) for x in sorted(self._atoms)))
f.close()
except:
if f is not None:
f.discard()
raise
class SyncableRepo(syncable.tree, util.SimpleTree):
pkgcore_config_type = ConfigHint(typename='raw_repo')
def __init__(self, succeed=True):
util.SimpleTree.__init__(self, {})
syncer = FakeSyncer('/fake', 'fake', succeed=succeed)
syncable.tree.__init__(self, syncer)
class Blacklist(_CheckSet):
"""Only run checks not matching any of the provided patterns."""
pkgcore_config_type = ConfigHint({'patterns': 'list'},
typename='pkgcheck_checkset')
def filter(self, checks):
return list(c for c in checks if not any(
p(f'{c.__module__}.{c.__name__}') for p in self.patterns))
class Whitelist(_CheckSet):
"""Only run checks matching one of the provided patterns."""
pkgcore_config_type = ConfigHint({'patterns': 'list'},
typename='pkgcheck_checkset')
def filter(self, checks):
return list(c for c in checks if any(
f('%s.%s' % (c.__module__, c.__name__)) for f in self.patterns))
class OnDiskProfile(ProfileStack):
pkgcore_config_type = ConfigHint(
{
'basepath': 'str',
'profile': 'str'
},
required=('basepath', 'profile'),
typename='profile',
)
def __init__(self, basepath, profile, load_profile_base=True):
super().__init__(pjoin(basepath, profile))
self.basepath = basepath
self.load_profile_base = load_profile_base
@staticmethod
def split_abspath(path):
path = abspath(path)
# filter's heavy, but it handles '/' while also
# suppressing the leading '/'
chunks = [x for x in path.split("/") if x]
try:
# poor mans rindex.
pbase = max(x for x in enumerate(chunks) if x[1] == 'profiles')[0]
except ValueError:
# no base found.
return None
return pjoin("/", *chunks[:pbase + 1]), '/'.join(chunks[pbase + 1:])
@classmethod
def from_abspath(cls, path):
vals = cls.split_abspath(path)
if vals is not None:
vals = cls(load_profile_base=True, *vals)
return vals
@klass.jit_attr
def stack(self):
l = ProfileStack.stack.function(self)
if self.load_profile_base:
l = (EmptyRootNode._autodetect_and_create(self.basepath), ) + l
return l
@klass.jit_attr
def _incremental_masks(self):
stack = self.stack
if self.load_profile_base:
stack = stack[1:]
return ProfileStack._incremental_masks(self, stack_override=stack)
@klass.jit_attr
def _incremental_unmasks(self):
stack = self.stack
if self.load_profile_base:
stack = stack[1:]
return ProfileStack._incremental_unmasks(self, stack_override=stack)
def make_repo_config(repo_data, livefs=False, frozen=False, repo_id=None):
def repo():
return fake_repo(repo_data,
livefs=livefs,
frozen=frozen,
repo_id=repo_id)
repo.pkgcore_config_type = ConfigHint(typename='repo')
return basics.HardCodedConfigSection({'class': repo})
class FakeDomain(object):
pkgcore_config_type = ConfigHint({'repos': 'refs:repo',
'vdb': 'refs:repo'},
typename='domain')
def __init__(self, repos, vdb):
object.__init__(self)
self.repos = repos
self.vdb = vdb
class FakeDomain(object):
pkgcore_config_type = ConfigHint({'repos': 'refs:repo'}, typename='domain')
def __init__(self, repos):
self.all_ebuild_repos_raw = repos
self.root = None
def add_repo(self, *args, **kwargs):
"""stubbed"""
class Scope(object):
"""Only run checks matching any of the given scopes."""
pkgcore_config_type = ConfigHint({'scopes': 'list'},
typename='pkgcheck_checkset')
def __init__(self, scopes):
self.scopes = tuple(int(x) for x in scopes)
def filter(self, checks):
return list(c for c in checks if c.scope in self.scopes)
class SystemSet(object):
"""Set of packages defined by the selected profile."""
pkgcore_config_type = ConfigHint({'profile': 'ref:profile'},
typename='pkgset')
def __init__(self, profile):
self.system = frozenset(profile.system)
def __iter__(self):
for pkg in self.system:
yield pkg
class Suite(object):
pkgcore_config_type = ConfigHint(
{
'target_repo': 'ref:repo',
'checkset': 'ref:pkgcheck_checkset'
},
typename='pkgcheck_suite')
def __init__(self, target_repo, checkset=None):
self.target_repo = target_repo
self.checkset = checkset
def test_clone(self):
c = ConfigHint(types={
'foo': 'list',
'one': 'str'
},
positional=['one'],
required=['one'],
typename='barn',
doc='orig doc')
c2 = c.clone(types={
'foo': 'list',
'one': 'str',
'two': 'str'
},
required=['one', 'two'])
self.assertEqual(c2.types, {'foo': 'list', 'one': 'str', 'two': 'str'})
self.assertEqual(c2.positional, c.positional)
self.assertEqual(c2.required, ['one', 'two'])
self.assertEqual(c2.typename, c.typename)
self.assertEqual(c2.allow_unknowns, c.allow_unknowns)
self.assertEqual(c2.doc, c.doc)
def test_builtin_full_override(self):
# check our assumptions...
# and yes, the signatures below are for file rather than
# dict; we need a cpy class for the test, the ConfigHint doesn't
# have to be accurate however
class cls(dict):
__slots__ = ()
self.assertRaises(TypeError, basics.ConfigType, cls)
raw_hint = ConfigHint(types={"filename":"str", "mode":"r",
"buffering":"int"}, typename='file',
required=['filename'], positional=['filename'])
# make sure it still tries to introspect, and throws typeerror.
# introspection is generally wanted- if it must be skipped, the
# ConfigHint must make it explicit
cls.pkgcore_config_type = raw_hint
self.assertRaises(TypeError, basics.ConfigType, cls)
cls.pkgcore_config_type = raw_hint.clone(authorative=True)
conf = basics.ConfigType(cls)
self.assertEqual(conf.name, 'file')
self.assertEqual(list(conf.required), ['filename'])
self.assertEqual(list(conf.positional), ['filename'])
self.assertEqual(sorted(conf.types), ['buffering', 'filename', 'mode'])
class FakeDomain(object):
pkgcore_config_type = ConfigHint(
{
'repos': 'refs:repo',
'binpkg': 'refs:repo',
'vdb': 'refs:repo'
},
typename='domain')
def __init__(self, repos, binpkg, vdb):
object.__init__(self)
self.repos = repos
self.source_repos = util.RepositoryGroup(repos)
self.installed_repos = util.RepositoryGroup(vdb)
self.binary_repos_raw = util.RepositoryGroup(binpkg)
self.vdb = vdb
class UserProfile(OnDiskProfile):
pkgcore_config_type = ConfigHint(
{
'user_path': 'str',
'parent_path': 'str',
'parent_profile': 'str'
},
required=('user_path', 'parent_path', 'parent_profile'),
typename='profile',
)
def __init__(self,
user_path,
parent_path,
parent_profile,
load_profile_base=True):
super().__init__(parent_path, parent_profile, load_profile_base)
self.node = UserProfileNode(user_path,
pjoin(parent_path, parent_profile))
class tree(prototype.tree):
livefs = True
configured = False
configurables = ("domain", "settings")
configure = None
package_factory = staticmethod(ebuild_built.generate_new_factory)
operations_kls = repo_ops.operations
pkgcore_config_type = ConfigHint(
{
'location': 'str',
'cache_location': 'str',
'repo_id': 'str',
'disable_cache': 'bool'
},
typename='repo')
def __init__(self,
location,
cache_location=None,
repo_id='vdb',
disable_cache=False):
prototype.tree.__init__(self, frozen=False)
self.repo_id = repo_id
self.location = location
if disable_cache:
cache_location = None
elif cache_location is None:
cache_location = pjoin("/var/cache/edb/dep", location.lstrip("/"))
self.cache_location = cache_location
self._versions_tmp_cache = {}
try:
st = os.stat(self.location)
if not stat.S_ISDIR(st.st_mode):
raise errors.InitializationError("base not a dir: %r" %
self.location)
elif not st.st_mode & (os.X_OK | os.R_OK):
raise errors.InitializationError(
"base lacks read/executable: %r" % self.location)
except OSError as e:
if e.errno != errno.ENOENT:
compatibility.raise_from(
errors.InitializationError("lstat failed on base %r" %
self.location))
self.package_class = self.package_factory(self)
def _get_categories(self, *optional_category):
# return if optional_category is passed... cause it's not yet supported
if optional_category:
return {}
try:
try:
return tuple(x for x in listdir_dirs(self.location)
if not x.startswith('.'))
except EnvironmentError as e:
compatibility.raise_from(
KeyError("failed fetching categories: %s" % str(e)))
finally:
pass
def _get_packages(self, category):
cpath = pjoin(self.location, category.lstrip(os.path.sep))
l = set()
d = {}
bad = False
try:
for x in listdir_dirs(cpath):
if x.startswith(".tmp.") or x.endswith(".lockfile") \
or x.startswith("-MERGING-"):
continue
try:
pkg = versioned_CPV(category + "/" + x)
except InvalidCPV:
bad = True
if bad or not pkg.fullver:
if '-scm' in x:
bad = 'scm'
elif '-try' in x:
bad = 'try'
else:
raise InvalidCPV("%s/%s: no version component" %
(category, x))
logger.error(
"merged -%s pkg detected: %s/%s. "
"throwing exception due to -%s not being a valid"
" version component. Silently ignoring that "
"specific version is not viable either since it "
"would result in pkgcore stomping whatever it was "
"that -%s version merged. "
"This is why embrace and extend is bad, mm'kay. "
"Use the offending pkg manager that merged it to "
"unmerge it.", bad, category, x, bad, bad)
raise InvalidCPV("%s/%s: -%s version component is "
"not standard." % (category, x, bad))
l.add(pkg.package)
d.setdefault((category, pkg.package), []).append(pkg.fullver)
except EnvironmentError as e:
compatibility.raise_from(
KeyError(
"failed fetching packages for category %s: %s" % (pjoin(
self.location, category.lstrip(os.path.sep)), str(e))))
self._versions_tmp_cache.update(d)
return tuple(l)
def _get_versions(self, catpkg):
return tuple(self._versions_tmp_cache.pop(catpkg))
def _get_ebuild_path(self, pkg):
s = "%s-%s" % (pkg.package, pkg.fullver)
return pjoin(self.location, pkg.category, s, s + ".ebuild")
def _get_path(self, pkg):
s = "%s-%s" % (pkg.package, pkg.fullver)
return pjoin(self.location, pkg.category, s)
_metadata_rewrites = {
"depends": "DEPEND",
"rdepends": "RDEPEND",
"post_rdepends": "PDEPEND",
"use": "USE",
"eapi": "EAPI",
"CONTENTS": "contents",
"source_repository": "repository",
"fullslot": "SLOT"
}
def _get_metadata(self, pkg):
return IndeterminantDict(
partial(
self._internal_load_key,
pjoin(self.location, pkg.category,
"%s-%s" % (pkg.package, pkg.fullver))))
def _internal_load_key(self, path, key):
key = self._metadata_rewrites.get(key, key)
if key == "contents":
data = ContentsFile(pjoin(path, "CONTENTS"), mutable=True)
elif key == "environment":
fp = pjoin(path, key)
if not os.path.exists(fp + ".bz2"):
if not os.path.exists(fp):
# icky.
raise KeyError("environment: no environment file found")
data = data_source.local_source(fp)
else:
data = data_source.bz2_source(fp + ".bz2")
elif key == "ebuild":
fp = pjoin(path,
os.path.basename(path.rstrip(os.path.sep)) + ".ebuild")
data = data_source.local_source(fp)
elif key == 'repo':
# try both, for portage/paludis compatibility.
data = readfile(pjoin(path, 'repository'), True)
if data is None:
data = readfile(pjoin(path, 'REPOSITORY'), True)
if data is None:
raise KeyError(key)
else:
data = readfile(pjoin(path, key), True)
if data is None:
raise KeyError((path, key))
return data
def notify_remove_package(self, pkg):
remove_it = len(self.packages[pkg.category]) == 1
prototype.tree.notify_remove_package(self, pkg)
if remove_it:
try:
os.rmdir(pjoin(self.location, pkg.category))
except OSError as oe:
# POSIX specifies either ENOTEMPTY or EEXIST for non-empty dir
# in particular, Solaris uses EEXIST in that case.
# https://github.com/pkgcore/pkgcore/pull/181
if oe.errno not in (errno.ENOTEMPTY, errno.EEXIST):
raise
# silently swallow it;
del oe
def __str__(self):
return '%s: location %s' % (self.repo_id, self.location)
class RepoConfig(syncable.tree):
layout_offset = "metadata/layout.conf"
default_hashes = ('size', 'sha256', 'sha512', 'whirlpool')
supported_profile_formats = ('pms', 'portage-1', 'portage-2')
supported_cache_formats = ('pms', 'md5-dict')
klass.inject_immutable_instance(locals())
__metaclass__ = WeakInstMeta
__inst_caching__ = True
pkgcore_config_type = ConfigHint(typename='repo_config',
types={
'config_name': 'str',
'syncer': 'lazy_ref:syncer',
})
def __init__(self,
location,
config_name=None,
syncer=None,
profiles_base='profiles'):
object.__setattr__(self, 'config_name', config_name)
object.__setattr__(self, 'location', location)
object.__setattr__(self, 'profiles_base',
pjoin(self.location, profiles_base))
syncable.tree.__init__(self, syncer)
self._parse_config()
def _parse_config(self):
"""Load data from the repo's metadata/layout.conf file."""
path = pjoin(self.location, self.layout_offset)
data = read_dict(iter_read_bash(readlines_ascii(path, True, True)),
source_isiter=True,
strip=True,
filename=path)
sf = object.__setattr__
hashes = data.get('manifest-hashes', '').lower().split()
if hashes:
hashes = ['size'] + hashes
hashes = tuple(iter_stable_unique(hashes))
else:
hashes = self.default_hashes
manifest_policy = data.get('use-manifests', 'strict').lower()
d = {
'disabled': (manifest_policy == 'false'),
'strict': (manifest_policy == 'strict'),
'thin': (data.get('thin-manifests', '').lower() == 'true'),
'signed': (data.get('sign-manifests', 'true').lower() == 'true'),
'hashes': hashes,
}
# complain if profiles/repo_name is missing
repo_name = readfile(pjoin(self.profiles_base, 'repo_name'), True)
if repo_name is None:
if not self.is_empty:
logger.warning("repo lacks a defined name: %r", self.location)
repo_name = '<unlabeled repo %s>' % self.location
# repo-name setting from metadata/layout.conf overrides profiles/repo_name if it exists
sf(self, 'repo_name', data.get('repo-name', repo_name.strip()))
sf(self, 'manifests', _immutable_attr_dict(d))
masters = data.get('masters')
if masters is None:
if not self.is_empty:
logger.warning(
"repo at %r, named %r, doesn't specify masters in metadata/layout.conf. "
"Please explicitly set masters (use \"masters =\" if the repo "
"is standalone).", self.location, self.repo_id)
masters = ()
else:
masters = tuple(iter_stable_unique(masters.split()))
sf(self, 'masters', masters)
aliases = data.get('aliases',
'').split() + [self.repo_id, self.location]
sf(self, 'aliases', tuple(iter_stable_unique(aliases)))
sf(self, 'eapis_deprecated',
tuple(iter_stable_unique(data.get('eapis-deprecated', '').split())))
v = set(data.get('cache-formats', 'pms').lower().split())
if not v:
v = [None]
elif not v.intersection(self.supported_cache_formats):
v = ['pms']
sf(self, 'cache_format', list(v)[0])
profile_formats = set(
data.get('profile-formats', 'pms').lower().split())
if not profile_formats:
logger.warning(
"repo at %r has unset profile-formats, defaulting to pms")
profile_formats = set(['pms'])
unknown = profile_formats.difference(self.supported_profile_formats)
if unknown:
logger.warning("repo at %r has unsupported profile format%s: %s",
self.location, pluralism(unknown),
', '.join(sorted(unknown)))
profile_formats.difference_update(unknown)
profile_formats.add('pms')
sf(self, 'profile_formats', profile_formats)
@klass.jit_attr
def raw_known_arches(self):
"""All valid KEYWORDS for the repo."""
try:
return frozenset(
iter_read_bash(pjoin(self.profiles_base, 'arch.list')))
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
return frozenset()
@klass.jit_attr
def raw_use_desc(self):
"""Global USE flags for the repo."""
# todo: convert this to using a common exception base, with
# conversion of ValueErrors...
def converter(key):
return (packages.AlwaysTrue, key)
return tuple(self._split_use_desc_file('use.desc', converter))
@klass.jit_attr
def raw_use_local_desc(self):
"""Local USE flags for the repo."""
def converter(key):
# todo: convert this to using a common exception base, with
# conversion of ValueErrors/atom exceptions...
chunks = key.split(':', 1)
return (atom.atom(chunks[0]), chunks[1])
return tuple(self._split_use_desc_file('use.local.desc', converter))
@klass.jit_attr
def raw_use_expand_desc(self):
"""USE_EXPAND settings for the repo."""
base = pjoin(self.profiles_base, 'desc')
try:
targets = sorted(listdir_files(base))
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
return ()
def f():
for use_group in targets:
group = use_group.split('.', 1)[0] + "_"
def converter(key):
return (packages.AlwaysTrue, group + key)
for x in self._split_use_desc_file('desc/%s' % use_group,
converter):
yield x
return tuple(f())
def _split_use_desc_file(self, name, converter):
line = None
fp = pjoin(self.profiles_base, name)
try:
for line in iter_read_bash(fp):
key, val = line.split(None, 1)
key = converter(key)
yield key[0], (key[1], val.split('-', 1)[1].strip())
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
except ValueError:
if line is None:
raise
compatibility.raise_from(
ValueError("Failed parsing %r: line was %r" % (fp, line)))
known_arches = klass.alias_attr('raw_known_arches')
use_desc = klass.alias_attr('raw_use_desc')
use_local_desc = klass.alias_attr('raw_use_local_desc')
use_expand_desc = klass.alias_attr('raw_use_expand_desc')
@klass.jit_attr
def is_empty(self):
"""Return boolean related to if the repo has files in it."""
result = True
try:
# any files existing means it's not empty
result = not listdir(self.location)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
if result:
logger.debug("repo is empty: %r", self.location)
return result
@klass.jit_attr
def repo_id(self):
"""Main identifier for the repo.
The name set in repos.conf for a repo overrides any repo-name settings
in the repo.
"""
if self.config_name is not None:
return self.config_name
return self.repo_name
arch_profiles = klass.alias_attr('profiles.arch_profiles')
@klass.jit_attr
def profiles(self):
return BundledProfiles(self.profiles_base)
class fetcher(base.fetcher):
pkgcore_config_type = ConfigHint(
{
'userpriv': 'bool',
'required_chksums': 'list',
'distdir': 'str',
'command': 'str',
'resume_command': 'str'
},
allow_unknowns=True)
def __init__(self,
distdir,
command,
resume_command=None,
required_chksums=None,
userpriv=True,
attempts=10,
readonly=False,
**extra_env):
"""
:param distdir: directory to download files to
:type distdir: string
:param command: shell command to execute to fetch a file
:type command: string
:param resume_command: if not None, command to use for resuming-
if None, command is reused
:param required_chksums: if None, all chksums must be verified,
else only chksums listed
:type required_chksums: None or sequence
:param userpriv: depriv for fetching?
:param attempts: max number of attempts before failing the fetch
:param readonly: controls whether fetching is allowed
"""
base.fetcher.__init__(self)
self.distdir = distdir
if required_chksums is not None:
required_chksums = [x.lower() for x in required_chksums]
else:
required_chksums = []
if len(required_chksums) == 1 and required_chksums[0] == "all":
self.required_chksums = None
else:
self.required_chksums = required_chksums
def rewrite_command(string):
new_command = string.replace("${DISTDIR}", self.distdir)
new_command = new_command.replace("$DISTDIR", self.distdir)
new_command = new_command.replace("${URI}", "%(URI)s")
new_command = new_command.replace("$URI", "%(URI)s")
new_command = new_command.replace("${FILE}", "%(FILE)s")
new_command = new_command.replace("$FILE", "%(FILE)s")
if new_command == string:
raise MalformedCommand(string)
try:
new_command % {"URI": "blah", "FILE": "blah"}
except KeyError as k:
raise MalformedCommand("%s: unexpected key %s" %
(command, k.args[0]))
return new_command
self.command = rewrite_command(command)
if resume_command is None:
self.resume_command = self.command
else:
self.resume_command = rewrite_command(resume_command)
self.attempts = attempts
self.userpriv = userpriv
self.readonly = readonly
self.extra_env = extra_env
def fetch(self, target):
"""
fetch a file
:type target: :obj:`pkgcore.fetch.fetchable` instance
:return: None if fetching failed,
else on disk location of the copied file
"""
if not isinstance(target, fetchable):
raise TypeError(
"target must be fetchable instance/derivative: %s" % target)
kw = {"mode": 0775}
if self.readonly:
kw["mode"] = 0555
if self.userpriv:
kw["gid"] = portage_gid
kw["minimal"] = True
if not ensure_dirs(self.distdir, **kw):
raise errors.distdirPerms(
self.distdir, "if userpriv, uid must be %i, gid must be %i. "
"if not readonly, directory must be 0775, else 0555" %
(portage_uid, portage_gid))
fp = pjoin(self.distdir, target.filename)
filename = os.path.basename(fp)
uri = iter(target.uri)
if self.userpriv and is_userpriv_capable():
extra = {"uid": portage_uid, "gid": portage_gid}
else:
extra = {}
extra["umask"] = 0002
extra["env"] = self.extra_env
attempts = self.attempts
last_exc = None
try:
while attempts >= 0:
try:
c = self._verify(fp, target)
return fp
except errors.MissingDistfile:
command = self.command
last_exc = sys.exc_info()
except errors.FetchFailed as e:
last_exc = sys.exc_info()
if not e.resumable:
try:
os.unlink(fp)
command = self.command
except OSError as oe:
raise_from(errors.UnmodifiableFile(fp, oe))
else:
command = self.resume_command
# yeah, it's funky, but it works.
if attempts > 0:
u = uri.next()
# note we're not even checking the results. the
# verify portion of the loop handles this. iow,
# don't trust their exit code. trust our chksums
# instead.
spawn_bash(command % {"URI": u, "FILE": filename}, **extra)
attempts -= 1
assert last_exc is not None
raise last_exc[0], last_exc[1], last_exc[2]
except StopIteration:
# ran out of uris
raise errors.FetchFailed(fp, "Ran out of urls to fetch from")
def get_path(self, fetchable):
fp = pjoin(self.distdir, fetchable.filename)
if self._verify(fp, fetchable) is None:
return fp
return None
def get_storage_path(self):
return self.distdir
class Cache(object):
pkgcore_config_type = ConfigHint(typename='cache')
def __init__(self, readonly=True):
self.readonly = self.frozen = readonly
class RepoConfig(syncable.tree):
layout_offset = "metadata/layout.conf"
default_hashes = ('size', 'sha256', 'sha512', 'whirlpool')
klass.inject_immutable_instance(locals())
__metaclass__ = WeakInstMeta
__inst_caching__ = True
pkgcore_config_type = ConfigHint(typename='raw_repo',
types={'syncer': 'lazy_ref:syncer'})
def __init__(self, location, syncer=None, profiles_base='profiles'):
object.__setattr__(self, 'location', location)
object.__setattr__(self, 'profiles_base',
pjoin(self.location, profiles_base))
syncable.tree.__init__(self, syncer)
self.parse_config()
def load_config(self):
path = pjoin(self.location, self.layout_offset)
return read_dict(iter_read_bash(readlines_ascii(path, True, True)),
source_isiter=True,
strip=True,
filename=path)
def parse_config(self):
data = self.load_config()
sf = object.__setattr__
hashes = data.get('manifest-hashes', '').lower().split()
if hashes:
hashes = ['size'] + hashes
hashes = tuple(iter_stable_unique(hashes))
else:
hashes = self.default_hashes
manifest_policy = data.get('use-manifests', 'strict').lower()
d = {
'disabled': (manifest_policy == 'false'),
'strict': (manifest_policy == 'strict'),
'thin': (data.get('thin-manifests', '').lower() == 'true'),
'signed': (data.get('sign-manifests', 'true').lower() == 'true'),
'hashes': hashes,
}
sf(self, 'manifests', _immutable_attr_dict(d))
masters = data.get('masters')
if masters is None:
if self.repo_id != 'gentoo' and not self.is_empty:
logger.warning(
"repository at %r, named %r, doesn't specify masters in metadata/layout.conf. "
"Defaulting to whatever repository is defined as 'default' (gentoo usually). "
"Please explicitly set the masters, or set masters = '' if the repository "
"is standalone.", self.location, self.repo_id)
else:
masters = tuple(iter_stable_unique(masters.split()))
sf(self, 'masters', masters)
sf(self, 'aliases',
tuple(iter_stable_unique(data.get('aliases', '').split())))
sf(self, 'eapis_deprecated',
tuple(iter_stable_unique(data.get('eapis-deprecated', '').split())))
v = set(data.get('cache-formats', 'pms').lower().split())
if not v.intersection(['pms', 'md5-dict']):
v = 'pms'
sf(self, 'cache_format', list(v)[0])
v = set(data.get('profile-formats', 'pms').lower().split())
if not v:
# dumb ass overlay devs, treat it as missing.
v = set(['pms'])
unknown = v.difference(['pms', 'portage-1', 'portage-2'])
if unknown:
logger.warning(
"repository at %r has an unsupported profile format: %s" %
(self.location, ', '.join(repr(x) for x in sorted(v))))
v = 'pms'
sf(self, 'profile_format', list(v)[0])
@klass.jit_attr
def raw_known_arches(self):
try:
return frozenset(
iter_read_bash(pjoin(self.profiles_base, 'arch.list')))
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
return frozenset()
@klass.jit_attr
def raw_use_desc(self):
# todo: convert this to using a common exception base, with
# conversion of ValueErrors...
def converter(key):
return (packages.AlwaysTrue, key)
return tuple(self._split_use_desc_file('use.desc', converter))
@klass.jit_attr
def raw_use_local_desc(self):
def converter(key):
# todo: convert this to using a common exception base, with
# conversion of ValueErrors/atom exceptoins...
chunks = key.split(':', 1)
return (atom.atom(chunks[0]), chunks[1])
return tuple(self._split_use_desc_file('use.local.desc', converter))
@klass.jit_attr
def raw_use_expand_desc(self):
base = pjoin(self.profiles_base, 'desc')
try:
targets = sorted(listdir_files(base))
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
return ()
def f():
for use_group in targets:
group = use_group.split('.', 1)[0] + "_"
def converter(key):
return (packages.AlwaysTrue, group + key)
for blah in self._split_use_desc_file('desc/%s' % use_group,
converter):
yield blah
return tuple(f())
def _split_use_desc_file(self, name, converter):
line = None
fp = pjoin(self.profiles_base, name)
try:
for line in iter_read_bash(fp):
key, val = line.split(None, 1)
key = converter(key)
yield key[0], (key[1], val.split('-', 1)[1].strip())
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
except ValueError as v:
if line is None:
raise
compatibility.raise_from(
ValueError("Failed parsing %r: line was %r" % (fp, line)))
known_arches = klass.alias_attr('raw_known_arches')
use_desc = klass.alias_attr('raw_use_desc')
use_local_desc = klass.alias_attr('raw_use_local_desc')
use_expand_desc = klass.alias_attr('raw_use_expand_desc')
@klass.jit_attr
def is_empty(self):
result = True
try:
# any files existing means it's not empty
result = not listdir(self.location)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
if result:
logger.debug("repository at %r is empty" % (self.location, ))
return result
@klass.jit_attr
def repo_id(self):
val = readfile(pjoin(self.profiles_base, 'repo_name'), True)
if val is None:
if not self.is_empty:
logger.warning(
"repository at location %r lacks a defined repo_name",
self.location)
val = '<unlabeled repository %s>' % self.location
return val.strip()
arch_profiles = klass.alias_attr('profiles.arch_profiles')
@klass.jit_attr
def profiles(self):
return BundledProfiles(self.profiles_base)
