def test_misc(self):
section = basics.FakeIncrementalDictConfigSection(
self._convert, {'list': [1, 2]})
self.assertFalse('foo' in section)
self.assertTrue('list' in section)
self.assertEqual(['list'], section.keys())
self.assertRaises(
errors.ConfigurationError,
basics.FakeIncrementalDictConfigSection(self._fail, {
'a': 'b'
}).render_value, None, 'a', 'str')
def test_repr(self):
def asis(central, value, arg_type):
assert arg_type == 'repr', arg_type
return value
section = basics.FakeIncrementalDictConfigSection(
asis, {
'seq.append': ('list', [1, 2]),
'simple': ('bool', True),
'multistr': ('str', 'body'),
'multistr.prepend': ('str', 'head'),
'refs': ('str', 'lost'),
'refs.append': ('ref', 'main'),
'refs.prepend': ('refs', ['a', 'b']),
'strlist': ('callable', asis),
'strlist.prepend': ('str', 'whatever'),
'wrong.prepend': ('wrong', 'wrong'),
})
manager = object()
self.assertRaises(KeyError, section.render_value, manager, 'spoon',
'repr')
self.assertEqual(('list', [None, None, [1, 2]]),
section.render_value(manager, 'seq', 'repr'))
self.assertEqual(('bool', True),
section.render_value(manager, 'simple', 'repr'))
self.assertEqual(('str', ['head', 'body', None]),
section.render_value(manager, 'multistr', 'repr'))
self.assertEqual(('refs', [['a', 'b'], ['lost'], ['main']]),
section.render_value(manager, 'refs', 'repr'))
self.assertEqual(
('list', [['whatever'], ['pkgcore.test.config.test_basics.asis'],
None]), section.render_value(manager, 'strlist', 'repr'))
self.assertRaises(errors.ConfigurationError, section.render_value,
manager, 'wrong', 'repr')
def test_fake_incrementals(self):
section = basics.FakeIncrementalDictConfigSection(
self._convert, {'seq.append': [1, 2]})
manager = object()
self.assertEqual([None, None, (manager, [1, 2], 'list')],
section.render_value(manager, 'seq', 'list'))
def _repr(central, value, arg_type):
return 'list', ['thing']
section = basics.FakeIncrementalDictConfigSection(_repr, {'foo': None})
self.assertEqual(('list', (None, ['thing'], None)),
section.render_value(manager, 'foo', 'repr'))
self.assertRaises(
errors.ConfigurationError,
basics.FakeIncrementalDictConfigSection(self._fail, {
'a.prepend': 'b'
}).render_value, None, 'a', 'list')
def config_from_make_conf(location="/etc/", profile_override=None, **kwargs):
"""
generate a config from a file location
:param location: location the portage configuration is based in,
defaults to /etc
:param profile_override: profile to use instead of the current system
profile, i.e. the target of the /etc/portage/make.profile
(or deprecated /etc/make.profile) symlink
"""
# this actually differs from portage parsing- we allow
# make.globals to provide vars used in make.conf, portage keeps
# them separate (kind of annoying)
config_root = os.environ.get("PORTAGE_CONFIGROOT", "/")
base_path = pjoin(config_root, location.strip("/"))
portage_base = pjoin(base_path, "portage")
# this isn't preserving incremental behaviour for features/use
# unfortunately
conf_dict = {}
try:
load_make_config(conf_dict, pjoin(base_path, 'make.globals'))
except errors.ParsingError as e:
if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT:
raise
try:
load_make_config(conf_dict, const.MAKE_GLOBALS)
except IGNORED_EXCEPTIONS:
raise
except:
raise_from(
errors.ParsingError("failed to find a usable make.globals"))
load_make_config(conf_dict,
pjoin(base_path, 'make.conf'),
required=False,
allow_sourcing=True,
incrementals=True)
load_make_config(conf_dict,
pjoin(portage_base, 'make.conf'),
required=False,
allow_sourcing=True,
incrementals=True)
root = os.environ.get("ROOT", conf_dict.get("ROOT", "/"))
gentoo_mirrors = [
x.rstrip("/") + "/distfiles"
for x in conf_dict.pop("GENTOO_MIRRORS", "").split()
]
# this is flawed... it'll pick up -some-feature
features = conf_dict.get("FEATURES", "").split()
new_config = {}
triggers = []
def add_trigger(name, kls_path, **extra_args):
d = extra_args.copy()
d['class'] = kls_path
new_config[name] = basics.ConfigSectionFromStringDict(d)
triggers.append(name)
# sets...
add_sets(new_config, root, portage_base)
user_profile_path = pjoin(base_path, "portage", "profile")
add_profile(new_config, base_path, user_profile_path, profile_override)
kwds = {
"class":
"pkgcore.vdb.ondisk.tree",
"location":
pjoin(root, 'var', 'db', 'pkg'),
"cache_location":
pjoin(config_root, 'var', 'cache', 'edb', 'dep', 'var', 'db', 'pkg'),
}
new_config["vdb"] = basics.AutoConfigSection(kwds)
# options used by rsync-based syncers
rsync_opts = isolate_rsync_opts(conf_dict)
repo_opts = {}
overlay_syncers = {}
try:
default_repo_opts, repo_opts = load_repos_conf(
pjoin(portage_base, 'repos.conf'))
except errors.ParsingError as e:
if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT:
raise
if repo_opts:
main_repo_id = default_repo_opts['main-repo']
main_repo = repo_opts[main_repo_id]['location']
overlay_repos = [
opts['location'] for repo, opts in repo_opts.iteritems()
if opts['location'] != main_repo
]
main_syncer = repo_opts[main_repo_id].get('sync-uri', None)
else:
# fallback to PORTDIR and PORTDIR_OVERLAY settings
main_repo = normpath(
os.environ.get("PORTDIR", conf_dict.pop("PORTDIR",
"/usr/portage")).strip())
overlay_repos = os.environ.get("PORTDIR_OVERLAY",
conf_dict.pop("PORTDIR_OVERLAY",
"")).split()
overlay_repos = [normpath(x) for x in overlay_repos]
main_syncer = conf_dict.pop("SYNC", None)
if overlay_repos and '-layman-sync' not in features:
overlay_syncers = add_layman_syncers(new_config,
rsync_opts,
overlay_repos,
config_root=config_root)
if main_syncer is not None:
make_syncer(new_config, main_repo, main_syncer, rsync_opts)
if overlay_repos and '-autodetect-sync' not in features:
for path in overlay_repos:
if path not in overlay_syncers:
overlay_syncers[path] = make_autodetect_syncer(
new_config, path)
repos = [main_repo] + overlay_repos
default_repos = list(reversed(repos))
new_config['ebuild-repo-common'] = basics.AutoConfigSection({
'class':
'pkgcore.ebuild.repository.slavedtree',
'default_mirrors':
gentoo_mirrors,
'inherit-only':
True,
'ignore_paludis_versioning': ('ignore-paludis-versioning' in features),
})
rsync_portdir_cache = 'metadata-transfer' not in features
# if a metadata cache exists, use it.
if rsync_portdir_cache:
for cache_type, frag in (('flat_hash.md5_cache', 'md5-cache'),
('metadata.database', 'cache')):
if not os.path.exists(pjoin(main_repo, 'metadata', frag)):
continue
new_config["cache:%s/metadata/cache" %
(main_repo, )] = basics.AutoConfigSection({
'class':
'pkgcore.cache.' + cache_type,
'readonly':
True,
'location':
main_repo,
})
break
else:
rsync_portdir_cache = False
repo_map = {}
for tree_loc in repos:
# XXX: Hack for portage-2 profile format support.
repo_config = RepoConfig(tree_loc)
repo_map[repo_config.repo_id] = repo_config
# repo configs
conf = {
'class': 'pkgcore.ebuild.repo_objs.RepoConfig',
'location': tree_loc,
}
if 'sync:%s' % (tree_loc, ) in new_config:
conf['syncer'] = 'sync:%s' % (tree_loc, )
if tree_loc == main_repo:
conf['default'] = True
new_config['raw:' + tree_loc] = basics.AutoConfigSection(conf)
# repo trees
kwds = {
'inherit': ('ebuild-repo-common', ),
'raw_repo': ('raw:' + tree_loc),
}
cache_name = 'cache:%s' % (tree_loc, )
new_config[cache_name] = mk_simple_cache(config_root, tree_loc)
kwds['cache'] = cache_name
if tree_loc == main_repo:
kwds['class'] = 'pkgcore.ebuild.repository.tree'
if rsync_portdir_cache:
kwds['cache'] = 'cache:%s/metadata/cache %s' % (main_repo,
cache_name)
else:
kwds['parent_repo'] = main_repo
new_config[tree_loc] = basics.AutoConfigSection(kwds)
new_config['portdir'] = basics.section_alias(main_repo, 'repo')
# XXX: Hack for portage-2 profile format support. We need to figure out how
# to dynamically create this from the config at runtime on attr access.
profiles.ProfileNode._repo_map = ImmutableDict(repo_map)
if overlay_repos:
new_config['repo-stack'] = basics.FakeIncrementalDictConfigSection(
my_convert_hybrid, {
'class': 'pkgcore.repository.multiplex.config_tree',
'repositories': tuple(default_repos)
})
else:
new_config['repo-stack'] = basics.section_alias(main_repo, 'repo')
new_config['vuln'] = basics.AutoConfigSection({
'class': SecurityUpgradesViaProfile,
'ebuild_repo': 'repo-stack',
'vdb': 'vdb',
'profile': 'profile',
})
new_config['glsa'] = basics.section_alias(
'vuln', SecurityUpgradesViaProfile.pkgcore_config_type.typename)
# binpkg.
buildpkg = 'buildpkg' in features or kwargs.get('buildpkg', False)
pkgdir = os.environ.get("PKGDIR", conf_dict.pop('PKGDIR', None))
if pkgdir is not None:
try:
pkgdir = abspath(pkgdir)
except OSError as oe:
if oe.errno != errno.ENOENT:
raise
if buildpkg or set(features).intersection(
('pristine-buildpkg', 'buildsyspkg', 'unmerge-backup')):
logger.warning(
"disabling buildpkg related features since PKGDIR doesn't exist"
)
pkgdir = None
else:
if not ensure_dirs(pkgdir, mode=0755, minimal=True):
logger.warning(
"disabling buildpkg related features since PKGDIR either doesn't "
"exist, or lacks 0755 minimal permissions")
pkgdir = None
# finally... domain.
conf_dict.update({
'class': 'pkgcore.ebuild.domain.domain',
'repositories': tuple(default_repos),
'fetcher': 'fetcher',
'default': True,
'vdb': ('vdb', ),
'profile': 'profile',
'name': 'livefs domain',
'root': root,
})
for f in ("package.mask", "package.unmask", "package.accept_keywords",
"package.keywords", "package.license", "package.use",
"package.env", "env:ebuild_hook_dir", "bashrc"):
fp = pjoin(portage_base, f.split(":")[0])
try:
os.stat(fp)
except OSError as oe:
if oe.errno != errno.ENOENT:
raise
else:
conf_dict[f.split(":")[-1]] = fp
if triggers:
conf_dict['triggers'] = tuple(triggers)
new_config['livefs domain'] = basics.FakeIncrementalDictConfigSection(
my_convert_hybrid, conf_dict)
return new_config
def __init__(self, location=None, profile_override=None, **kwargs):
"""
Args:
location (optional[str]): path to the portage config directory,
(defaults to /etc/portage)
profile_override (optional[str]): profile to use instead of the current system
profile, i.e. the target of the /etc/portage/make.profile symlink
configroot (optional[str]): location for various portage config files (defaults to /)
root (optional[str]): target root filesystem (defaults to /)
buildpkg (optional[bool]): forcibly disable/enable building binpkgs, otherwise
FEATURES=buildpkg from make.conf is used
Returns:
dict: config settings
"""
self._config = {}
location = location if location is not None else '/etc/portage'
self.dir = pjoin(
os.environ.get('PORTAGE_CONFIGROOT', kwargs.pop('configroot',
'/')),
location.lstrip('/'))
# this actually differs from portage parsing- we allow
# make.globals to provide vars used in make.conf, portage keeps
# them separate (kind of annoying)
#
# this isn't preserving incremental behaviour for features/use unfortunately
make_conf = {}
try:
self.load_make_conf(make_conf,
pjoin(const.CONFIG_PATH, 'make.globals'))
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
raise config_errors.ParsingError(
"failed to load make.globals") from e
self.load_make_conf(make_conf,
pjoin(self.dir, 'make.conf'),
required=False,
allow_sourcing=True,
incrementals=True)
self.root = os.environ.get(
"ROOT", kwargs.pop('root', make_conf.get("ROOT", "/")))
gentoo_mirrors = [
x.rstrip("/") + "/distfiles"
for x in make_conf.pop("GENTOO_MIRRORS", "").split()
]
self.features = frozenset(
optimize_incrementals(make_conf.get('FEATURES', '').split()))
self._add_sets()
self._add_profile(profile_override)
self['vdb'] = basics.AutoConfigSection({
'class':
'pkgcore.vdb.ondisk.tree',
'location':
pjoin(self.root, 'var', 'db', 'pkg'),
'cache_location':
'/var/cache/edb/dep/var/db/pkg',
})
try:
repos_conf_defaults, repos_conf = self.load_repos_conf(
pjoin(self.dir, 'repos.conf'))
except config_errors.ParsingError as e:
if not getattr(getattr(e, 'exc', None), 'errno',
None) == errno.ENOENT:
raise
try:
# fallback to defaults provided by pkgcore
repos_conf_defaults, repos_conf = self.load_repos_conf(
pjoin(const.CONFIG_PATH, 'repos.conf'))
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
raise config_errors.ParsingError(
'failed to find a usable repos.conf') from e
self['ebuild-repo-common'] = basics.AutoConfigSection({
'class':
'pkgcore.ebuild.repository.tree',
'default_mirrors':
gentoo_mirrors,
'inherit-only':
True,
})
repo_map = {}
for repo_name, repo_opts in list(repos_conf.items()):
repo_cls = repo_opts.pop('repo-type')
try:
repo = repo_cls(self,
repo_name=repo_name,
repo_opts=repo_opts,
repo_map=repo_map,
defaults=repos_conf_defaults)
except repo_errors.UnsupportedRepo as e:
logger.warning(
f'skipping {repo_name!r} repo: unsupported EAPI {str(e.repo.eapi)!r}'
)
del repos_conf[repo_name]
continue
self[repo_name] = basics.AutoConfigSection(repo)
# XXX: Hack for portage-2 profile format support. We need to figure out how
# to dynamically create this from the config at runtime on attr access.
profiles.ProfileNode._repo_map = ImmutableDict(repo_map)
self._make_repo_syncers(repos_conf, make_conf)
repos = [name for name in repos_conf.keys()]
if repos:
self['repo-stack'] = basics.FakeIncrementalDictConfigSection(
my_convert_hybrid, {
'class': 'pkgcore.repository.multiplex.config_tree',
'repos': tuple(repos)
})
self['vuln'] = basics.AutoConfigSection({
'class': SecurityUpgradesViaProfile,
'ebuild_repo': 'repo-stack',
'vdb': 'vdb',
'profile': 'profile',
})
# check if package building was forced on by the user
forced_buildpkg = kwargs.pop('buildpkg', False)
if forced_buildpkg:
make_conf['FEATURES'] += ' buildpkg'
# now add the fetcher- we delay it till here to clean out the environ
# it passes to the command.
# *everything* in make_conf must be str values also.
self._add_fetcher(make_conf)
# finally... domain.
make_conf.update({
'class': 'pkgcore.ebuild.domain.domain',
'repos': tuple(repos),
'fetcher': 'fetcher',
'default': True,
'vdb': ('vdb', ),
'profile': 'profile',
'name': 'livefs',
'root': self.root,
'config_dir': self.dir,
})
self['livefs'] = basics.FakeIncrementalDictConfigSection(
my_convert_hybrid, make_conf)
def config_from_make_conf(location=None, profile_override=None, **kwargs):
"""generate a config using portage's config files
Args:
location (optional[str]): path to the portage config directory,
(defaults to /etc/portage)
profile_override (optional[str]): profile to use instead of the current system
profile, i.e. the target of the /etc/portage/make.profile symlink
configroot (optional[str]): location for various portage config files (defaults to /)
root (optional[str]): target root filesystem (defaults to /)
buildpkg (optional[bool]): forcibly disable/enable building binpkgs, otherwise
FEATURES=buildpkg from make.conf is used
Returns:
dict: config settings
"""
# this actually differs from portage parsing- we allow
# make.globals to provide vars used in make.conf, portage keeps
# them separate (kind of annoying)
config_dir = location if location is not None else '/etc/portage'
config_dir = pjoin(
os.environ.get('PORTAGE_CONFIGROOT', kwargs.pop('configroot', '/')),
config_dir.lstrip('/'))
# this isn't preserving incremental behaviour for features/use unfortunately
make_conf = {}
try:
load_make_conf(make_conf, pjoin(const.CONFIG_PATH, 'make.globals'))
except IGNORED_EXCEPTIONS:
raise
except:
raise_from(errors.ParsingError("failed to load make.globals"))
load_make_conf(make_conf,
pjoin(config_dir, 'make.conf'),
required=False,
allow_sourcing=True,
incrementals=True)
root = os.environ.get("ROOT", kwargs.pop('root',
make_conf.get("ROOT", "/")))
gentoo_mirrors = [
x.rstrip("/") + "/distfiles"
for x in make_conf.pop("GENTOO_MIRRORS", "").split()
]
# this is flawed... it'll pick up -some-feature
features = make_conf.get("FEATURES", "").split()
config = {}
triggers = []
def add_trigger(name, kls_path, **extra_args):
d = extra_args.copy()
d['class'] = kls_path
config[name] = basics.ConfigSectionFromStringDict(d)
triggers.append(name)
# sets...
add_sets(config, root, config_dir)
add_profile(config, config_dir, profile_override)
kwds = {
"class": "pkgcore.vdb.ondisk.tree",
"location": pjoin(root, 'var', 'db', 'pkg'),
"cache_location": '/var/cache/edb/dep/var/db/pkg',
}
config["vdb"] = basics.AutoConfigSection(kwds)
try:
repos_conf_defaults, repos_conf = load_repos_conf(
pjoin(config_dir, 'repos.conf'))
except errors.ParsingError as e:
if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT:
raise
try:
# fallback to defaults provided by pkgcore
repos_conf_defaults, repos_conf = load_repos_conf(
pjoin(const.CONFIG_PATH, 'repos.conf'))
except IGNORED_EXCEPTIONS:
raise
except:
raise_from(
errors.ParsingError("failed to find a usable repos.conf"))
make_repo_syncers(config, repos_conf, make_conf)
config['ebuild-repo-common'] = basics.AutoConfigSection({
'class':
'pkgcore.ebuild.repository.tree',
'default_mirrors':
gentoo_mirrors,
'inherit-only':
True,
'ignore_paludis_versioning': ('ignore-paludis-versioning' in features),
})
default_repo_path = repos_conf[
repos_conf_defaults['main-repo']]['location']
repo_map = {}
for repo_name, repo_opts in repos_conf.iteritems():
repo_path = repo_opts['location']
# XXX: Hack for portage-2 profile format support.
repo_config = RepoConfig(repo_path, repo_name)
repo_map[repo_config.repo_id] = repo_config
# repo configs
repo_conf = {
'class': 'pkgcore.ebuild.repo_objs.RepoConfig',
'config_name': repo_name,
'location': repo_path,
'syncer': 'sync:' + repo_name,
}
# repo trees
repo = {
'inherit': ('ebuild-repo-common', ),
'repo_config': 'conf:' + repo_name,
}
# metadata cache
if repo_config.cache_format is not None:
cache_name = 'cache:' + repo_name
config[cache_name] = make_cache(repo_config.cache_format,
repo_path)
repo['cache'] = cache_name
if repo_path == default_repo_path:
repo_conf['default'] = True
config['conf:' + repo_name] = basics.AutoConfigSection(repo_conf)
config[repo_name] = basics.AutoConfigSection(repo)
# XXX: Hack for portage-2 profile format support. We need to figure out how
# to dynamically create this from the config at runtime on attr access.
profiles.ProfileNode._repo_map = ImmutableDict(repo_map)
repos = [name for name in repos_conf.iterkeys()]
if len(repos) > 1:
config['repo-stack'] = basics.FakeIncrementalDictConfigSection(
my_convert_hybrid, {
'class': 'pkgcore.repository.multiplex.config_tree',
'repositories': tuple(repos)
})
else:
config['repo-stack'] = basics.section_alias(repos[0], 'repo')
config['vuln'] = basics.AutoConfigSection({
'class': SecurityUpgradesViaProfile,
'ebuild_repo': 'repo-stack',
'vdb': 'vdb',
'profile': 'profile',
})
config['glsa'] = basics.section_alias(
'vuln', SecurityUpgradesViaProfile.pkgcore_config_type.typename)
# binpkg.
buildpkg = 'buildpkg' in features or kwargs.pop('buildpkg', False)
pkgdir = os.environ.get("PKGDIR", make_conf.pop('PKGDIR', None))
if pkgdir is not None:
try:
pkgdir = abspath(pkgdir)
except OSError as oe:
if oe.errno != errno.ENOENT:
raise
if buildpkg or set(features).intersection(
('pristine-buildpkg', 'buildsyspkg', 'unmerge-backup')):
logger.warning(
"disabling buildpkg related features since PKGDIR doesn't exist"
)
pkgdir = None
else:
if not ensure_dirs(pkgdir, mode=0755, minimal=True):
logger.warning(
"disabling buildpkg related features since PKGDIR either doesn't "
"exist, or lacks 0755 minimal permissions")
pkgdir = None
# finally... domain.
make_conf.update({
'class': 'pkgcore.ebuild.domain.domain',
'repositories': tuple(repos),
'fetcher': 'fetcher',
'default': True,
'vdb': ('vdb', ),
'profile': 'profile',
'name': 'livefs',
'root': root,
})
for f in ("package.mask", "package.unmask", "package.accept_keywords",
"package.keywords", "package.license", "package.use",
"package.env", "env:ebuild_hook_dir", "bashrc"):
fp = pjoin(config_dir, f.split(":")[0])
try:
os.stat(fp)
except OSError as oe:
if oe.errno != errno.ENOENT:
raise
else:
make_conf[f.split(":")[-1]] = fp
if triggers:
make_conf['triggers'] = tuple(triggers)
config['livefs'] = basics.FakeIncrementalDictConfigSection(
my_convert_hybrid, make_conf)
return config
