def test_misc(self): section = basics.FakeIncrementalDictConfigSection( self._convert, {'list': [1, 2]}) self.assertFalse('foo' in section) self.assertTrue('list' in section) self.assertEqual(['list'], section.keys()) self.assertRaises( errors.ConfigurationError, basics.FakeIncrementalDictConfigSection(self._fail, { 'a': 'b' }).render_value, None, 'a', 'str')
def test_repr(self): def asis(central, value, arg_type): assert arg_type == 'repr', arg_type return value section = basics.FakeIncrementalDictConfigSection( asis, { 'seq.append': ('list', [1, 2]), 'simple': ('bool', True), 'multistr': ('str', 'body'), 'multistr.prepend': ('str', 'head'), 'refs': ('str', 'lost'), 'refs.append': ('ref', 'main'), 'refs.prepend': ('refs', ['a', 'b']), 'strlist': ('callable', asis), 'strlist.prepend': ('str', 'whatever'), 'wrong.prepend': ('wrong', 'wrong'), }) manager = object() self.assertRaises(KeyError, section.render_value, manager, 'spoon', 'repr') self.assertEqual(('list', [None, None, [1, 2]]), section.render_value(manager, 'seq', 'repr')) self.assertEqual(('bool', True), section.render_value(manager, 'simple', 'repr')) self.assertEqual(('str', ['head', 'body', None]), section.render_value(manager, 'multistr', 'repr')) self.assertEqual(('refs', [['a', 'b'], ['lost'], ['main']]), section.render_value(manager, 'refs', 'repr')) self.assertEqual( ('list', [['whatever'], ['pkgcore.test.config.test_basics.asis'], None]), section.render_value(manager, 'strlist', 'repr')) self.assertRaises(errors.ConfigurationError, section.render_value, manager, 'wrong', 'repr')
def test_fake_incrementals(self): section = basics.FakeIncrementalDictConfigSection( self._convert, {'seq.append': [1, 2]}) manager = object() self.assertEqual([None, None, (manager, [1, 2], 'list')], section.render_value(manager, 'seq', 'list')) def _repr(central, value, arg_type): return 'list', ['thing'] section = basics.FakeIncrementalDictConfigSection(_repr, {'foo': None}) self.assertEqual(('list', (None, ['thing'], None)), section.render_value(manager, 'foo', 'repr')) self.assertRaises( errors.ConfigurationError, basics.FakeIncrementalDictConfigSection(self._fail, { 'a.prepend': 'b' }).render_value, None, 'a', 'list')
def config_from_make_conf(location="/etc/", profile_override=None, **kwargs): """ generate a config from a file location :param location: location the portage configuration is based in, defaults to /etc :param profile_override: profile to use instead of the current system profile, i.e. the target of the /etc/portage/make.profile (or deprecated /etc/make.profile) symlink """ # this actually differs from portage parsing- we allow # make.globals to provide vars used in make.conf, portage keeps # them separate (kind of annoying) config_root = os.environ.get("PORTAGE_CONFIGROOT", "/") base_path = pjoin(config_root, location.strip("/")) portage_base = pjoin(base_path, "portage") # this isn't preserving incremental behaviour for features/use # unfortunately conf_dict = {} try: load_make_config(conf_dict, pjoin(base_path, 'make.globals')) except errors.ParsingError as e: if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT: raise try: load_make_config(conf_dict, const.MAKE_GLOBALS) except IGNORED_EXCEPTIONS: raise except: raise_from( errors.ParsingError("failed to find a usable make.globals")) load_make_config(conf_dict, pjoin(base_path, 'make.conf'), required=False, allow_sourcing=True, incrementals=True) load_make_config(conf_dict, pjoin(portage_base, 'make.conf'), required=False, allow_sourcing=True, incrementals=True) root = os.environ.get("ROOT", conf_dict.get("ROOT", "/")) gentoo_mirrors = [ x.rstrip("/") + "/distfiles" for x in conf_dict.pop("GENTOO_MIRRORS", "").split() ] # this is flawed... it'll pick up -some-feature features = conf_dict.get("FEATURES", "").split() new_config = {} triggers = [] def add_trigger(name, kls_path, **extra_args): d = extra_args.copy() d['class'] = kls_path new_config[name] = basics.ConfigSectionFromStringDict(d) triggers.append(name) # sets... add_sets(new_config, root, portage_base) user_profile_path = pjoin(base_path, "portage", "profile") add_profile(new_config, base_path, user_profile_path, profile_override) kwds = { "class": "pkgcore.vdb.ondisk.tree", "location": pjoin(root, 'var', 'db', 'pkg'), "cache_location": pjoin(config_root, 'var', 'cache', 'edb', 'dep', 'var', 'db', 'pkg'), } new_config["vdb"] = basics.AutoConfigSection(kwds) # options used by rsync-based syncers rsync_opts = isolate_rsync_opts(conf_dict) repo_opts = {} overlay_syncers = {} try: default_repo_opts, repo_opts = load_repos_conf( pjoin(portage_base, 'repos.conf')) except errors.ParsingError as e: if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT: raise if repo_opts: main_repo_id = default_repo_opts['main-repo'] main_repo = repo_opts[main_repo_id]['location'] overlay_repos = [ opts['location'] for repo, opts in repo_opts.iteritems() if opts['location'] != main_repo ] main_syncer = repo_opts[main_repo_id].get('sync-uri', None) else: # fallback to PORTDIR and PORTDIR_OVERLAY settings main_repo = normpath( os.environ.get("PORTDIR", conf_dict.pop("PORTDIR", "/usr/portage")).strip()) overlay_repos = os.environ.get("PORTDIR_OVERLAY", conf_dict.pop("PORTDIR_OVERLAY", "")).split() overlay_repos = [normpath(x) for x in overlay_repos] main_syncer = conf_dict.pop("SYNC", None) if overlay_repos and '-layman-sync' not in features: overlay_syncers = add_layman_syncers(new_config, rsync_opts, overlay_repos, config_root=config_root) if main_syncer is not None: make_syncer(new_config, main_repo, main_syncer, rsync_opts) if overlay_repos and '-autodetect-sync' not in features: for path in overlay_repos: if path not in overlay_syncers: overlay_syncers[path] = make_autodetect_syncer( new_config, path) repos = [main_repo] + overlay_repos default_repos = list(reversed(repos)) new_config['ebuild-repo-common'] = basics.AutoConfigSection({ 'class': 'pkgcore.ebuild.repository.slavedtree', 'default_mirrors': gentoo_mirrors, 'inherit-only': True, 'ignore_paludis_versioning': ('ignore-paludis-versioning' in features), }) rsync_portdir_cache = 'metadata-transfer' not in features # if a metadata cache exists, use it. if rsync_portdir_cache: for cache_type, frag in (('flat_hash.md5_cache', 'md5-cache'), ('metadata.database', 'cache')): if not os.path.exists(pjoin(main_repo, 'metadata', frag)): continue new_config["cache:%s/metadata/cache" % (main_repo, )] = basics.AutoConfigSection({ 'class': 'pkgcore.cache.' + cache_type, 'readonly': True, 'location': main_repo, }) break else: rsync_portdir_cache = False repo_map = {} for tree_loc in repos: # XXX: Hack for portage-2 profile format support. repo_config = RepoConfig(tree_loc) repo_map[repo_config.repo_id] = repo_config # repo configs conf = { 'class': 'pkgcore.ebuild.repo_objs.RepoConfig', 'location': tree_loc, } if 'sync:%s' % (tree_loc, ) in new_config: conf['syncer'] = 'sync:%s' % (tree_loc, ) if tree_loc == main_repo: conf['default'] = True new_config['raw:' + tree_loc] = basics.AutoConfigSection(conf) # repo trees kwds = { 'inherit': ('ebuild-repo-common', ), 'raw_repo': ('raw:' + tree_loc), } cache_name = 'cache:%s' % (tree_loc, ) new_config[cache_name] = mk_simple_cache(config_root, tree_loc) kwds['cache'] = cache_name if tree_loc == main_repo: kwds['class'] = 'pkgcore.ebuild.repository.tree' if rsync_portdir_cache: kwds['cache'] = 'cache:%s/metadata/cache %s' % (main_repo, cache_name) else: kwds['parent_repo'] = main_repo new_config[tree_loc] = basics.AutoConfigSection(kwds) new_config['portdir'] = basics.section_alias(main_repo, 'repo') # XXX: Hack for portage-2 profile format support. We need to figure out how # to dynamically create this from the config at runtime on attr access. profiles.ProfileNode._repo_map = ImmutableDict(repo_map) if overlay_repos: new_config['repo-stack'] = basics.FakeIncrementalDictConfigSection( my_convert_hybrid, { 'class': 'pkgcore.repository.multiplex.config_tree', 'repositories': tuple(default_repos) }) else: new_config['repo-stack'] = basics.section_alias(main_repo, 'repo') new_config['vuln'] = basics.AutoConfigSection({ 'class': SecurityUpgradesViaProfile, 'ebuild_repo': 'repo-stack', 'vdb': 'vdb', 'profile': 'profile', }) new_config['glsa'] = basics.section_alias( 'vuln', SecurityUpgradesViaProfile.pkgcore_config_type.typename) # binpkg. buildpkg = 'buildpkg' in features or kwargs.get('buildpkg', False) pkgdir = os.environ.get("PKGDIR", conf_dict.pop('PKGDIR', None)) if pkgdir is not None: try: pkgdir = abspath(pkgdir) except OSError as oe: if oe.errno != errno.ENOENT: raise if buildpkg or set(features).intersection( ('pristine-buildpkg', 'buildsyspkg', 'unmerge-backup')): logger.warning( "disabling buildpkg related features since PKGDIR doesn't exist" ) pkgdir = None else: if not ensure_dirs(pkgdir, mode=0755, minimal=True): logger.warning( "disabling buildpkg related features since PKGDIR either doesn't " "exist, or lacks 0755 minimal permissions") pkgdir = None
# finally... domain. conf_dict.update({ 'class': 'pkgcore.ebuild.domain.domain', 'repositories': tuple(default_repos), 'fetcher': 'fetcher', 'default': True, 'vdb': ('vdb', ), 'profile': 'profile', 'name': 'livefs domain', 'root': root, }) for f in ("package.mask", "package.unmask", "package.accept_keywords", "package.keywords", "package.license", "package.use", "package.env", "env:ebuild_hook_dir", "bashrc"): fp = pjoin(portage_base, f.split(":")[0]) try: os.stat(fp) except OSError as oe: if oe.errno != errno.ENOENT: raise else: conf_dict[f.split(":")[-1]] = fp if triggers: conf_dict['triggers'] = tuple(triggers) new_config['livefs domain'] = basics.FakeIncrementalDictConfigSection( my_convert_hybrid, conf_dict) return new_config
def __init__(self, location=None, profile_override=None, **kwargs): """ Args: location (optional[str]): path to the portage config directory, (defaults to /etc/portage) profile_override (optional[str]): profile to use instead of the current system profile, i.e. the target of the /etc/portage/make.profile symlink configroot (optional[str]): location for various portage config files (defaults to /) root (optional[str]): target root filesystem (defaults to /) buildpkg (optional[bool]): forcibly disable/enable building binpkgs, otherwise FEATURES=buildpkg from make.conf is used Returns: dict: config settings """ self._config = {} location = location if location is not None else '/etc/portage' self.dir = pjoin( os.environ.get('PORTAGE_CONFIGROOT', kwargs.pop('configroot', '/')), location.lstrip('/')) # this actually differs from portage parsing- we allow # make.globals to provide vars used in make.conf, portage keeps # them separate (kind of annoying) # # this isn't preserving incremental behaviour for features/use unfortunately make_conf = {} try: self.load_make_conf(make_conf, pjoin(const.CONFIG_PATH, 'make.globals')) except IGNORED_EXCEPTIONS: raise except Exception as e: raise config_errors.ParsingError( "failed to load make.globals") from e self.load_make_conf(make_conf, pjoin(self.dir, 'make.conf'), required=False, allow_sourcing=True, incrementals=True) self.root = os.environ.get( "ROOT", kwargs.pop('root', make_conf.get("ROOT", "/"))) gentoo_mirrors = [ x.rstrip("/") + "/distfiles" for x in make_conf.pop("GENTOO_MIRRORS", "").split() ] self.features = frozenset( optimize_incrementals(make_conf.get('FEATURES', '').split())) self._add_sets() self._add_profile(profile_override) self['vdb'] = basics.AutoConfigSection({ 'class': 'pkgcore.vdb.ondisk.tree', 'location': pjoin(self.root, 'var', 'db', 'pkg'), 'cache_location': '/var/cache/edb/dep/var/db/pkg', }) try: repos_conf_defaults, repos_conf = self.load_repos_conf( pjoin(self.dir, 'repos.conf')) except config_errors.ParsingError as e: if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT: raise try: # fallback to defaults provided by pkgcore repos_conf_defaults, repos_conf = self.load_repos_conf( pjoin(const.CONFIG_PATH, 'repos.conf')) except IGNORED_EXCEPTIONS: raise except Exception as e: raise config_errors.ParsingError( 'failed to find a usable repos.conf') from e self['ebuild-repo-common'] = basics.AutoConfigSection({ 'class': 'pkgcore.ebuild.repository.tree', 'default_mirrors': gentoo_mirrors, 'inherit-only': True, }) repo_map = {} for repo_name, repo_opts in list(repos_conf.items()): repo_cls = repo_opts.pop('repo-type') try: repo = repo_cls(self, repo_name=repo_name, repo_opts=repo_opts, repo_map=repo_map, defaults=repos_conf_defaults) except repo_errors.UnsupportedRepo as e: logger.warning( f'skipping {repo_name!r} repo: unsupported EAPI {str(e.repo.eapi)!r}' ) del repos_conf[repo_name] continue self[repo_name] = basics.AutoConfigSection(repo) # XXX: Hack for portage-2 profile format support. We need to figure out how # to dynamically create this from the config at runtime on attr access. profiles.ProfileNode._repo_map = ImmutableDict(repo_map) self._make_repo_syncers(repos_conf, make_conf) repos = [name for name in repos_conf.keys()] if repos: self['repo-stack'] = basics.FakeIncrementalDictConfigSection( my_convert_hybrid, { 'class': 'pkgcore.repository.multiplex.config_tree', 'repos': tuple(repos) }) self['vuln'] = basics.AutoConfigSection({ 'class': SecurityUpgradesViaProfile, 'ebuild_repo': 'repo-stack', 'vdb': 'vdb', 'profile': 'profile', }) # check if package building was forced on by the user forced_buildpkg = kwargs.pop('buildpkg', False) if forced_buildpkg: make_conf['FEATURES'] += ' buildpkg' # now add the fetcher- we delay it till here to clean out the environ # it passes to the command. # *everything* in make_conf must be str values also. self._add_fetcher(make_conf) # finally... domain. make_conf.update({ 'class': 'pkgcore.ebuild.domain.domain', 'repos': tuple(repos), 'fetcher': 'fetcher', 'default': True, 'vdb': ('vdb', ), 'profile': 'profile', 'name': 'livefs', 'root': self.root, 'config_dir': self.dir, }) self['livefs'] = basics.FakeIncrementalDictConfigSection( my_convert_hybrid, make_conf)
def config_from_make_conf(location=None, profile_override=None, **kwargs): """generate a config using portage's config files Args: location (optional[str]): path to the portage config directory, (defaults to /etc/portage) profile_override (optional[str]): profile to use instead of the current system profile, i.e. the target of the /etc/portage/make.profile symlink configroot (optional[str]): location for various portage config files (defaults to /) root (optional[str]): target root filesystem (defaults to /) buildpkg (optional[bool]): forcibly disable/enable building binpkgs, otherwise FEATURES=buildpkg from make.conf is used Returns: dict: config settings """ # this actually differs from portage parsing- we allow # make.globals to provide vars used in make.conf, portage keeps # them separate (kind of annoying) config_dir = location if location is not None else '/etc/portage' config_dir = pjoin( os.environ.get('PORTAGE_CONFIGROOT', kwargs.pop('configroot', '/')), config_dir.lstrip('/')) # this isn't preserving incremental behaviour for features/use unfortunately make_conf = {} try: load_make_conf(make_conf, pjoin(const.CONFIG_PATH, 'make.globals')) except IGNORED_EXCEPTIONS: raise except: raise_from(errors.ParsingError("failed to load make.globals")) load_make_conf(make_conf, pjoin(config_dir, 'make.conf'), required=False, allow_sourcing=True, incrementals=True) root = os.environ.get("ROOT", kwargs.pop('root', make_conf.get("ROOT", "/"))) gentoo_mirrors = [ x.rstrip("/") + "/distfiles" for x in make_conf.pop("GENTOO_MIRRORS", "").split() ] # this is flawed... it'll pick up -some-feature features = make_conf.get("FEATURES", "").split() config = {} triggers = [] def add_trigger(name, kls_path, **extra_args): d = extra_args.copy() d['class'] = kls_path config[name] = basics.ConfigSectionFromStringDict(d) triggers.append(name) # sets... add_sets(config, root, config_dir) add_profile(config, config_dir, profile_override) kwds = { "class": "pkgcore.vdb.ondisk.tree", "location": pjoin(root, 'var', 'db', 'pkg'), "cache_location": '/var/cache/edb/dep/var/db/pkg', } config["vdb"] = basics.AutoConfigSection(kwds) try: repos_conf_defaults, repos_conf = load_repos_conf( pjoin(config_dir, 'repos.conf')) except errors.ParsingError as e: if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT: raise try: # fallback to defaults provided by pkgcore repos_conf_defaults, repos_conf = load_repos_conf( pjoin(const.CONFIG_PATH, 'repos.conf')) except IGNORED_EXCEPTIONS: raise except: raise_from( errors.ParsingError("failed to find a usable repos.conf")) make_repo_syncers(config, repos_conf, make_conf) config['ebuild-repo-common'] = basics.AutoConfigSection({ 'class': 'pkgcore.ebuild.repository.tree', 'default_mirrors': gentoo_mirrors, 'inherit-only': True, 'ignore_paludis_versioning': ('ignore-paludis-versioning' in features), }) default_repo_path = repos_conf[ repos_conf_defaults['main-repo']]['location'] repo_map = {} for repo_name, repo_opts in repos_conf.iteritems(): repo_path = repo_opts['location'] # XXX: Hack for portage-2 profile format support. repo_config = RepoConfig(repo_path, repo_name) repo_map[repo_config.repo_id] = repo_config # repo configs repo_conf = { 'class': 'pkgcore.ebuild.repo_objs.RepoConfig', 'config_name': repo_name, 'location': repo_path, 'syncer': 'sync:' + repo_name, } # repo trees repo = { 'inherit': ('ebuild-repo-common', ), 'repo_config': 'conf:' + repo_name, } # metadata cache if repo_config.cache_format is not None: cache_name = 'cache:' + repo_name config[cache_name] = make_cache(repo_config.cache_format, repo_path) repo['cache'] = cache_name if repo_path == default_repo_path: repo_conf['default'] = True config['conf:' + repo_name] = basics.AutoConfigSection(repo_conf) config[repo_name] = basics.AutoConfigSection(repo) # XXX: Hack for portage-2 profile format support. We need to figure out how # to dynamically create this from the config at runtime on attr access. profiles.ProfileNode._repo_map = ImmutableDict(repo_map) repos = [name for name in repos_conf.iterkeys()] if len(repos) > 1: config['repo-stack'] = basics.FakeIncrementalDictConfigSection( my_convert_hybrid, { 'class': 'pkgcore.repository.multiplex.config_tree', 'repositories': tuple(repos) }) else: config['repo-stack'] = basics.section_alias(repos[0], 'repo') config['vuln'] = basics.AutoConfigSection({ 'class': SecurityUpgradesViaProfile, 'ebuild_repo': 'repo-stack', 'vdb': 'vdb', 'profile': 'profile', }) config['glsa'] = basics.section_alias( 'vuln', SecurityUpgradesViaProfile.pkgcore_config_type.typename) # binpkg. buildpkg = 'buildpkg' in features or kwargs.pop('buildpkg', False) pkgdir = os.environ.get("PKGDIR", make_conf.pop('PKGDIR', None)) if pkgdir is not None: try: pkgdir = abspath(pkgdir) except OSError as oe: if oe.errno != errno.ENOENT: raise if buildpkg or set(features).intersection( ('pristine-buildpkg', 'buildsyspkg', 'unmerge-backup')): logger.warning( "disabling buildpkg related features since PKGDIR doesn't exist" ) pkgdir = None else: if not ensure_dirs(pkgdir, mode=0755, minimal=True): logger.warning( "disabling buildpkg related features since PKGDIR either doesn't " "exist, or lacks 0755 minimal permissions") pkgdir = None
# finally... domain. make_conf.update({ 'class': 'pkgcore.ebuild.domain.domain', 'repositories': tuple(repos), 'fetcher': 'fetcher', 'default': True, 'vdb': ('vdb', ), 'profile': 'profile', 'name': 'livefs', 'root': root, }) for f in ("package.mask", "package.unmask", "package.accept_keywords", "package.keywords", "package.license", "package.use", "package.env", "env:ebuild_hook_dir", "bashrc"): fp = pjoin(config_dir, f.split(":")[0]) try: os.stat(fp) except OSError as oe: if oe.errno != errno.ENOENT: raise else: make_conf[f.split(":")[-1]] = fp if triggers: make_conf['triggers'] = tuple(triggers) config['livefs'] = basics.FakeIncrementalDictConfigSection( my_convert_hybrid, make_conf) return config