Example #1
0
 def test_alias(self):
     def myspork():
         return object
     manager = central.ConfigManager(
         [{'spork': basics.HardCodedConfigSection({'class': myspork}),
           'foon': basics.section_alias('spork', 'myspork'),
           }])
     # This tests both the detected typename of foon and the caching.
     self.assertIdentical(manager.objects.myspork['spork'], manager.objects.myspork['foon'])
Example #2
0
 def test_alias(self):
     def myspork():
         return object
     manager = central.ConfigManager(
         [{'spork': basics.HardCodedConfigSection({'class': myspork}),
           'foon': basics.section_alias('spork', 'myspork'),
           }])
     # This tests both the detected typename of foon and the caching.
     self.assertIdentical(manager.objects.myspork['spork'], manager.objects.myspork['foon'])
Example #3
0
 def test_alias(self):
     def spoon():
         """Noop."""
     foon = central.CollapsedConfig(basics.ConfigType(spoon), {}, None)
     class MockManager(object):
         def collapse_named_section(self, name):
             if name == 'foon':
                 return foon
             return object()
     manager = MockManager()
     alias = basics.section_alias('foon', 'spoon')
     type_obj = basics.ConfigType(alias.render_value(manager, 'class',
                                                  'callable'))
     self.assertEqual('spoon', type_obj.name)
     self.assertIdentical(
         foon,
         alias.render_value(manager, 'target', 'ref:spoon').collapse())
Example #4
0
 def test_alias(self):
     def spoon():
         """Noop."""
     foon = central.CollapsedConfig(basics.ConfigType(spoon), {}, None)
     class MockManager:
         def collapse_named_section(self, name):
             if name == 'foon':
                 return foon
             return object()
     manager = MockManager()
     alias = basics.section_alias('foon', 'spoon')
     type_obj = basics.ConfigType(alias.render_value(manager, 'class',
                                                  'callable'))
     self.assertEqual('spoon', type_obj.name)
     self.assertIdentical(
         foon,
         alias.render_value(manager, 'target', 'ref:spoon').collapse())
Example #5
0
def config_from_make_conf(location="/etc/", profile_override=None, **kwargs):
    """
    generate a config from a file location

    :param location: location the portage configuration is based in,
        defaults to /etc
    :param profile_override: profile to use instead of the current system
        profile, i.e. the target of the /etc/portage/make.profile
        (or deprecated /etc/make.profile) symlink
    """

    # this actually differs from portage parsing- we allow
    # make.globals to provide vars used in make.conf, portage keeps
    # them separate (kind of annoying)

    config_root = os.environ.get("PORTAGE_CONFIGROOT", "/")
    base_path = pjoin(config_root, location.strip("/"))
    portage_base = pjoin(base_path, "portage")

    # this isn't preserving incremental behaviour for features/use
    # unfortunately

    conf_dict = {}
    try:
        load_make_config(conf_dict, pjoin(base_path, 'make.globals'))
    except errors.ParsingError as e:
        if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT:
            raise
        try:
            load_make_config(conf_dict, const.MAKE_GLOBALS)
        except IGNORED_EXCEPTIONS:
            raise
        except:
            raise_from(
                errors.ParsingError("failed to find a usable make.globals"))
    load_make_config(conf_dict,
                     pjoin(base_path, 'make.conf'),
                     required=False,
                     allow_sourcing=True,
                     incrementals=True)
    load_make_config(conf_dict,
                     pjoin(portage_base, 'make.conf'),
                     required=False,
                     allow_sourcing=True,
                     incrementals=True)

    root = os.environ.get("ROOT", conf_dict.get("ROOT", "/"))
    gentoo_mirrors = [
        x.rstrip("/") + "/distfiles"
        for x in conf_dict.pop("GENTOO_MIRRORS", "").split()
    ]

    # this is flawed... it'll pick up -some-feature
    features = conf_dict.get("FEATURES", "").split()

    new_config = {}
    triggers = []

    def add_trigger(name, kls_path, **extra_args):
        d = extra_args.copy()
        d['class'] = kls_path
        new_config[name] = basics.ConfigSectionFromStringDict(d)
        triggers.append(name)

    # sets...
    add_sets(new_config, root, portage_base)

    user_profile_path = pjoin(base_path, "portage", "profile")
    add_profile(new_config, base_path, user_profile_path, profile_override)

    kwds = {
        "class":
        "pkgcore.vdb.ondisk.tree",
        "location":
        pjoin(root, 'var', 'db', 'pkg'),
        "cache_location":
        pjoin(config_root, 'var', 'cache', 'edb', 'dep', 'var', 'db', 'pkg'),
    }
    new_config["vdb"] = basics.AutoConfigSection(kwds)

    # options used by rsync-based syncers
    rsync_opts = isolate_rsync_opts(conf_dict)

    repo_opts = {}
    overlay_syncers = {}
    try:
        default_repo_opts, repo_opts = load_repos_conf(
            pjoin(portage_base, 'repos.conf'))
    except errors.ParsingError as e:
        if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT:
            raise

    if repo_opts:
        main_repo_id = default_repo_opts['main-repo']
        main_repo = repo_opts[main_repo_id]['location']
        overlay_repos = [
            opts['location'] for repo, opts in repo_opts.iteritems()
            if opts['location'] != main_repo
        ]
        main_syncer = repo_opts[main_repo_id].get('sync-uri', None)
    else:
        # fallback to PORTDIR and PORTDIR_OVERLAY settings
        main_repo = normpath(
            os.environ.get("PORTDIR", conf_dict.pop("PORTDIR",
                                                    "/usr/portage")).strip())
        overlay_repos = os.environ.get("PORTDIR_OVERLAY",
                                       conf_dict.pop("PORTDIR_OVERLAY",
                                                     "")).split()
        overlay_repos = [normpath(x) for x in overlay_repos]
        main_syncer = conf_dict.pop("SYNC", None)

        if overlay_repos and '-layman-sync' not in features:
            overlay_syncers = add_layman_syncers(new_config,
                                                 rsync_opts,
                                                 overlay_repos,
                                                 config_root=config_root)

    if main_syncer is not None:
        make_syncer(new_config, main_repo, main_syncer, rsync_opts)

    if overlay_repos and '-autodetect-sync' not in features:
        for path in overlay_repos:
            if path not in overlay_syncers:
                overlay_syncers[path] = make_autodetect_syncer(
                    new_config, path)

    repos = [main_repo] + overlay_repos
    default_repos = list(reversed(repos))

    new_config['ebuild-repo-common'] = basics.AutoConfigSection({
        'class':
        'pkgcore.ebuild.repository.slavedtree',
        'default_mirrors':
        gentoo_mirrors,
        'inherit-only':
        True,
        'ignore_paludis_versioning': ('ignore-paludis-versioning' in features),
    })

    rsync_portdir_cache = 'metadata-transfer' not in features
    # if a metadata cache exists, use it.
    if rsync_portdir_cache:
        for cache_type, frag in (('flat_hash.md5_cache', 'md5-cache'),
                                 ('metadata.database', 'cache')):
            if not os.path.exists(pjoin(main_repo, 'metadata', frag)):
                continue
            new_config["cache:%s/metadata/cache" %
                       (main_repo, )] = basics.AutoConfigSection({
                           'class':
                           'pkgcore.cache.' + cache_type,
                           'readonly':
                           True,
                           'location':
                           main_repo,
                       })
            break
        else:
            rsync_portdir_cache = False

    repo_map = {}

    for tree_loc in repos:
        # XXX: Hack for portage-2 profile format support.
        repo_config = RepoConfig(tree_loc)
        repo_map[repo_config.repo_id] = repo_config

        # repo configs
        conf = {
            'class': 'pkgcore.ebuild.repo_objs.RepoConfig',
            'location': tree_loc,
        }
        if 'sync:%s' % (tree_loc, ) in new_config:
            conf['syncer'] = 'sync:%s' % (tree_loc, )
        if tree_loc == main_repo:
            conf['default'] = True
        new_config['raw:' + tree_loc] = basics.AutoConfigSection(conf)

        # repo trees
        kwds = {
            'inherit': ('ebuild-repo-common', ),
            'raw_repo': ('raw:' + tree_loc),
        }
        cache_name = 'cache:%s' % (tree_loc, )
        new_config[cache_name] = mk_simple_cache(config_root, tree_loc)
        kwds['cache'] = cache_name
        if tree_loc == main_repo:
            kwds['class'] = 'pkgcore.ebuild.repository.tree'
            if rsync_portdir_cache:
                kwds['cache'] = 'cache:%s/metadata/cache %s' % (main_repo,
                                                                cache_name)
        else:
            kwds['parent_repo'] = main_repo
        new_config[tree_loc] = basics.AutoConfigSection(kwds)

    new_config['portdir'] = basics.section_alias(main_repo, 'repo')

    # XXX: Hack for portage-2 profile format support. We need to figure out how
    # to dynamically create this from the config at runtime on attr access.
    profiles.ProfileNode._repo_map = ImmutableDict(repo_map)

    if overlay_repos:
        new_config['repo-stack'] = basics.FakeIncrementalDictConfigSection(
            my_convert_hybrid, {
                'class': 'pkgcore.repository.multiplex.config_tree',
                'repositories': tuple(default_repos)
            })
    else:
        new_config['repo-stack'] = basics.section_alias(main_repo, 'repo')

    new_config['vuln'] = basics.AutoConfigSection({
        'class': SecurityUpgradesViaProfile,
        'ebuild_repo': 'repo-stack',
        'vdb': 'vdb',
        'profile': 'profile',
    })
    new_config['glsa'] = basics.section_alias(
        'vuln', SecurityUpgradesViaProfile.pkgcore_config_type.typename)

    # binpkg.
    buildpkg = 'buildpkg' in features or kwargs.get('buildpkg', False)
    pkgdir = os.environ.get("PKGDIR", conf_dict.pop('PKGDIR', None))
    if pkgdir is not None:
        try:
            pkgdir = abspath(pkgdir)
        except OSError as oe:
            if oe.errno != errno.ENOENT:
                raise
            if buildpkg or set(features).intersection(
                ('pristine-buildpkg', 'buildsyspkg', 'unmerge-backup')):
                logger.warning(
                    "disabling buildpkg related features since PKGDIR doesn't exist"
                )
            pkgdir = None
        else:
            if not ensure_dirs(pkgdir, mode=0755, minimal=True):
                logger.warning(
                    "disabling buildpkg related features since PKGDIR either doesn't "
                    "exist, or lacks 0755 minimal permissions")
                pkgdir = None
Example #6
0
def config_from_make_conf(location="/etc/", profile_override=None, **kwargs):
    """
    generate a config from a file location

    :param location: location the portage configuration is based in,
        defaults to /etc
    :param profile_override: profile to use instead of the current system
        profile, i.e. the target of the /etc/portage/make.profile
        (or deprecated /etc/make.profile) symlink
    """

    # this actually differs from portage parsing- we allow
    # make.globals to provide vars used in make.conf, portage keeps
    # them separate (kind of annoying)

    config_root = os.environ.get("PORTAGE_CONFIGROOT", "/")
    base_path = pjoin(config_root, location.strip("/"))
    portage_base = pjoin(base_path, "portage")

    # this isn't preserving incremental behaviour for features/use
    # unfortunately

    conf_dict = {}
    try:
        load_make_config(conf_dict, pjoin(base_path, 'make.globals'))
    except errors.ParsingError as e:
        if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT:
            raise
        try:
            if 'PKGCORE_REPO_PATH' in os.environ:
                config_path = pjoin(os.environ['PKGCORE_REPO_PATH'], 'config')
            else:
                config_path = pjoin(
                    config_root, sys.prefix.lstrip('/'), 'share/pkgcore/config')
            load_make_config(conf_dict, pjoin(config_path, 'make.globals'))
        except IGNORED_EXCEPTIONS:
            raise
        except:
            raise_from(errors.ParsingError(
                "failed to find a usable make.globals"))
    load_make_config(
        conf_dict, pjoin(base_path, 'make.conf'), required=False,
        allow_sourcing=True, incrementals=True)
    load_make_config(
        conf_dict, pjoin(portage_base, 'make.conf'), required=False,
        allow_sourcing=True, incrementals=True)

    root = os.environ.get("ROOT", conf_dict.get("ROOT", "/"))
    gentoo_mirrors = [
        x.rstrip("/") + "/distfiles" for x in conf_dict.pop("GENTOO_MIRRORS", "").split()]

    # this is flawed... it'll pick up -some-feature
    features = conf_dict.get("FEATURES", "").split()

    new_config = {}
    triggers = []

    def add_trigger(name, kls_path, **extra_args):
        d = extra_args.copy()
        d['class'] = kls_path
        new_config[name] = basics.ConfigSectionFromStringDict(d)
        triggers.append(name)

    # sets...
    add_sets(new_config, root, portage_base)

    user_profile_path = pjoin(base_path, "portage", "profile")
    add_profile(new_config, base_path, user_profile_path, profile_override)

    kwds = {
        "class": "pkgcore.vdb.ondisk.tree",
        "location": pjoin(root, 'var', 'db', 'pkg'),
        "cache_location": pjoin(
            config_root, 'var', 'cache', 'edb', 'dep', 'var', 'db', 'pkg'),
    }
    new_config["vdb"] = basics.AutoConfigSection(kwds)

    # options used by rsync-based syncers
    rsync_opts = isolate_rsync_opts(conf_dict)

    repo_opts = {}
    overlay_syncers = {}
    try:
        default_repo_opts, repo_opts = load_repos_conf(
            pjoin(portage_base, 'repos.conf'))
    except errors.ParsingError as e:
        if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT:
            raise

    if repo_opts:
        main_repo_id = default_repo_opts['main-repo']
        main_repo = repo_opts[main_repo_id]['location']
        overlay_repos = [opts['location'] for repo, opts in repo_opts.iteritems()
                         if opts['location'] != main_repo]
        main_syncer = repo_opts[main_repo_id].get('sync-uri', None)
    else:
        # fallback to PORTDIR and PORTDIR_OVERLAY settings
        main_repo = normpath(os.environ.get(
            "PORTDIR", conf_dict.pop("PORTDIR", "/usr/portage")).strip())
        overlay_repos = os.environ.get(
            "PORTDIR_OVERLAY", conf_dict.pop("PORTDIR_OVERLAY", "")).split()
        overlay_repos = [normpath(x) for x in overlay_repos]
        main_syncer = conf_dict.pop("SYNC", None)

        if overlay_repos and '-layman-sync' not in features:
            overlay_syncers = add_layman_syncers(
                new_config, rsync_opts, overlay_repos, config_root=config_root)

    if main_syncer is not None:
        make_syncer(new_config, main_repo, main_syncer, rsync_opts)

    if overlay_repos and '-autodetect-sync' not in features:
        for path in overlay_repos:
            if path not in overlay_syncers:
                overlay_syncers[path] = make_autodetect_syncer(new_config, path)

    repos = [main_repo] + overlay_repos
    default_repos = list(reversed(repos))

    new_config['ebuild-repo-common'] = basics.AutoConfigSection({
        'class': 'pkgcore.ebuild.repository.slavedtree',
        'default_mirrors': gentoo_mirrors,
        'inherit-only': True,
        'ignore_paludis_versioning': ('ignore-paludis-versioning' in features),
    })

    rsync_portdir_cache = 'metadata-transfer' not in features
    # if a metadata cache exists, use it.
    if rsync_portdir_cache:
        for cache_type, frag in (('flat_hash.md5_cache', 'md5-cache'),
                                 ('metadata.database', 'cache')):
            if not os.path.exists(pjoin(main_repo, 'metadata', frag)):
                continue
            new_config["cache:%s/metadata/cache" % (main_repo,)] = basics.AutoConfigSection({
                'class': 'pkgcore.cache.' + cache_type,
                'readonly': True,
                'location': main_repo,
            })
            break
        else:
            rsync_portdir_cache = False

    repo_map = {}

    for tree_loc in repos:
        # XXX: Hack for portage-2 profile format support.
        repo_config = RepoConfig(tree_loc)
        repo_map[repo_config.repo_id] = repo_config

        # repo configs
        conf = {
            'class': 'pkgcore.ebuild.repo_objs.RepoConfig',
            'location': tree_loc,
        }
        if 'sync:%s' % (tree_loc,) in new_config:
            conf['syncer'] = 'sync:%s' % (tree_loc,)
        if tree_loc == main_repo:
            conf['default'] = True
        new_config['raw:' + tree_loc] = basics.AutoConfigSection(conf)

        # repo trees
        kwds = {
            'inherit': ('ebuild-repo-common',),
            'raw_repo': ('raw:' + tree_loc),
        }
        cache_name = 'cache:%s' % (tree_loc,)
        new_config[cache_name] = mk_simple_cache(config_root, tree_loc)
        kwds['cache'] = cache_name
        if tree_loc == main_repo:
            kwds['class'] = 'pkgcore.ebuild.repository.tree'
            if rsync_portdir_cache:
                kwds['cache'] = 'cache:%s/metadata/cache %s' % (main_repo, cache_name)
        else:
            kwds['parent_repo'] = main_repo
        new_config[tree_loc] = basics.AutoConfigSection(kwds)

    new_config['portdir'] = basics.section_alias(main_repo, 'repo')

    # XXX: Hack for portage-2 profile format support. We need to figure out how
    # to dynamically create this from the config at runtime on attr access.
    profiles.ProfileNode._repo_map = ImmutableDict(repo_map)

    if overlay_repos:
        new_config['repo-stack'] = basics.FakeIncrementalDictConfigSection(
            my_convert_hybrid, {
                'class': 'pkgcore.repository.multiplex.config_tree',
                'repositories': tuple(default_repos)})
    else:
        new_config['repo-stack'] = basics.section_alias(main_repo, 'repo')

    new_config['vuln'] = basics.AutoConfigSection({
        'class': SecurityUpgradesViaProfile,
        'ebuild_repo': 'repo-stack',
        'vdb': 'vdb',
        'profile': 'profile',
    })
    new_config['glsa'] = basics.section_alias(
        'vuln', SecurityUpgradesViaProfile.pkgcore_config_type.typename)

    # binpkg.
    buildpkg = 'buildpkg' in features or kwargs.get('buildpkg', False)
    pkgdir = os.environ.get("PKGDIR", conf_dict.pop('PKGDIR', None))
    if pkgdir is not None:
        try:
            pkgdir = abspath(pkgdir)
        except OSError as oe:
            if oe.errno != errno.ENOENT:
                raise
            if buildpkg or set(features).intersection(
                    ('pristine-buildpkg', 'buildsyspkg', 'unmerge-backup')):
                logger.warning("disabling buildpkg related features since PKGDIR doesn't exist")
            pkgdir = None
        else:
            if not ensure_dirs(pkgdir, mode=0755, minimal=True):
                logger.warning("disabling buildpkg related features since PKGDIR either doesn't "
                               "exist, or lacks 0755 minimal permissions")
                pkgdir = None
Example #7
0
def config_from_make_conf(location=None, profile_override=None, **kwargs):
    """generate a config using portage's config files

    Args:
        location (optional[str]): path to the portage config directory,
            (defaults to /etc/portage)
        profile_override (optional[str]): profile to use instead of the current system
            profile, i.e. the target of the /etc/portage/make.profile symlink
        configroot (optional[str]): location for various portage config files (defaults to /)
        root (optional[str]): target root filesystem (defaults to /)
        buildpkg (optional[bool]): forcibly disable/enable building binpkgs, otherwise
            FEATURES=buildpkg from make.conf is used

    Returns:
        dict: config settings
    """

    # this actually differs from portage parsing- we allow
    # make.globals to provide vars used in make.conf, portage keeps
    # them separate (kind of annoying)

    config_dir = location if location is not None else '/etc/portage'
    config_dir = pjoin(
        os.environ.get('PORTAGE_CONFIGROOT', kwargs.pop('configroot', '/')),
        config_dir.lstrip('/'))

    # this isn't preserving incremental behaviour for features/use unfortunately

    make_conf = {}
    try:
        load_make_conf(make_conf, pjoin(const.CONFIG_PATH, 'make.globals'))
    except IGNORED_EXCEPTIONS:
        raise
    except:
        raise_from(errors.ParsingError("failed to load make.globals"))
    load_make_conf(make_conf,
                   pjoin(config_dir, 'make.conf'),
                   required=False,
                   allow_sourcing=True,
                   incrementals=True)

    root = os.environ.get("ROOT", kwargs.pop('root',
                                             make_conf.get("ROOT", "/")))
    gentoo_mirrors = [
        x.rstrip("/") + "/distfiles"
        for x in make_conf.pop("GENTOO_MIRRORS", "").split()
    ]

    # this is flawed... it'll pick up -some-feature
    features = make_conf.get("FEATURES", "").split()

    config = {}
    triggers = []

    def add_trigger(name, kls_path, **extra_args):
        d = extra_args.copy()
        d['class'] = kls_path
        config[name] = basics.ConfigSectionFromStringDict(d)
        triggers.append(name)

    # sets...
    add_sets(config, root, config_dir)

    add_profile(config, config_dir, profile_override)

    kwds = {
        "class": "pkgcore.vdb.ondisk.tree",
        "location": pjoin(root, 'var', 'db', 'pkg'),
        "cache_location": '/var/cache/edb/dep/var/db/pkg',
    }
    config["vdb"] = basics.AutoConfigSection(kwds)

    try:
        repos_conf_defaults, repos_conf = load_repos_conf(
            pjoin(config_dir, 'repos.conf'))
    except errors.ParsingError as e:
        if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT:
            raise
        try:
            # fallback to defaults provided by pkgcore
            repos_conf_defaults, repos_conf = load_repos_conf(
                pjoin(const.CONFIG_PATH, 'repos.conf'))
        except IGNORED_EXCEPTIONS:
            raise
        except:
            raise_from(
                errors.ParsingError("failed to find a usable repos.conf"))

    make_repo_syncers(config, repos_conf, make_conf)

    config['ebuild-repo-common'] = basics.AutoConfigSection({
        'class':
        'pkgcore.ebuild.repository.tree',
        'default_mirrors':
        gentoo_mirrors,
        'inherit-only':
        True,
        'ignore_paludis_versioning': ('ignore-paludis-versioning' in features),
    })

    default_repo_path = repos_conf[
        repos_conf_defaults['main-repo']]['location']
    repo_map = {}

    for repo_name, repo_opts in repos_conf.iteritems():
        repo_path = repo_opts['location']

        # XXX: Hack for portage-2 profile format support.
        repo_config = RepoConfig(repo_path, repo_name)
        repo_map[repo_config.repo_id] = repo_config

        # repo configs
        repo_conf = {
            'class': 'pkgcore.ebuild.repo_objs.RepoConfig',
            'config_name': repo_name,
            'location': repo_path,
            'syncer': 'sync:' + repo_name,
        }

        # repo trees
        repo = {
            'inherit': ('ebuild-repo-common', ),
            'repo_config': 'conf:' + repo_name,
        }

        # metadata cache
        if repo_config.cache_format is not None:
            cache_name = 'cache:' + repo_name
            config[cache_name] = make_cache(repo_config.cache_format,
                                            repo_path)
            repo['cache'] = cache_name

        if repo_path == default_repo_path:
            repo_conf['default'] = True

        config['conf:' + repo_name] = basics.AutoConfigSection(repo_conf)
        config[repo_name] = basics.AutoConfigSection(repo)

    # XXX: Hack for portage-2 profile format support. We need to figure out how
    # to dynamically create this from the config at runtime on attr access.
    profiles.ProfileNode._repo_map = ImmutableDict(repo_map)

    repos = [name for name in repos_conf.iterkeys()]
    if len(repos) > 1:
        config['repo-stack'] = basics.FakeIncrementalDictConfigSection(
            my_convert_hybrid, {
                'class': 'pkgcore.repository.multiplex.config_tree',
                'repositories': tuple(repos)
            })
    else:
        config['repo-stack'] = basics.section_alias(repos[0], 'repo')

    config['vuln'] = basics.AutoConfigSection({
        'class': SecurityUpgradesViaProfile,
        'ebuild_repo': 'repo-stack',
        'vdb': 'vdb',
        'profile': 'profile',
    })
    config['glsa'] = basics.section_alias(
        'vuln', SecurityUpgradesViaProfile.pkgcore_config_type.typename)

    # binpkg.
    buildpkg = 'buildpkg' in features or kwargs.pop('buildpkg', False)
    pkgdir = os.environ.get("PKGDIR", make_conf.pop('PKGDIR', None))
    if pkgdir is not None:
        try:
            pkgdir = abspath(pkgdir)
        except OSError as oe:
            if oe.errno != errno.ENOENT:
                raise
            if buildpkg or set(features).intersection(
                ('pristine-buildpkg', 'buildsyspkg', 'unmerge-backup')):
                logger.warning(
                    "disabling buildpkg related features since PKGDIR doesn't exist"
                )
            pkgdir = None
        else:
            if not ensure_dirs(pkgdir, mode=0755, minimal=True):
                logger.warning(
                    "disabling buildpkg related features since PKGDIR either doesn't "
                    "exist, or lacks 0755 minimal permissions")
                pkgdir = None
Example #8
0
def config_from_make_conf(location=None, profile_override=None, **kwargs):
    """generate a config using portage's config files

    Args:
        location (optional[str]): path to the portage config directory,
            (defaults to /etc/portage)
        profile_override (optional[str]): profile to use instead of the current system
            profile, i.e. the target of the /etc/portage/make.profile symlink
        configroot (optional[str]): location for various portage config files (defaults to /)
        root (optional[str]): target root filesystem (defaults to /)
        buildpkg (optional[bool]): forcibly disable/enable building binpkgs, otherwise
            FEATURES=buildpkg from make.conf is used

    Returns:
        dict: config settings
    """

    # this actually differs from portage parsing- we allow
    # make.globals to provide vars used in make.conf, portage keeps
    # them separate (kind of annoying)

    config_dir = location if location is not None else '/etc/portage'
    config_dir = pjoin(
        os.environ.get('PORTAGE_CONFIGROOT', kwargs.pop('configroot', '/')),
        config_dir.lstrip('/'))

    # this isn't preserving incremental behaviour for features/use unfortunately

    make_conf = {}
    try:
        load_make_conf(make_conf, pjoin(const.CONFIG_PATH, 'make.globals'))
    except IGNORED_EXCEPTIONS:
        raise
    except:
        raise_from(errors.ParsingError("failed to load make.globals"))
    load_make_conf(
        make_conf, pjoin(config_dir, 'make.conf'), required=False,
        allow_sourcing=True, incrementals=True)

    root = os.environ.get("ROOT", kwargs.pop('root', make_conf.get("ROOT", "/")))
    gentoo_mirrors = [
        x.rstrip("/") + "/distfiles" for x in make_conf.pop("GENTOO_MIRRORS", "").split()]

    # this is flawed... it'll pick up -some-feature
    features = make_conf.get("FEATURES", "").split()

    config = {}
    triggers = []

    def add_trigger(name, kls_path, **extra_args):
        d = extra_args.copy()
        d['class'] = kls_path
        config[name] = basics.ConfigSectionFromStringDict(d)
        triggers.append(name)

    # sets...
    add_sets(config, root, config_dir)

    add_profile(config, config_dir, profile_override)

    kwds = {
        "class": "pkgcore.vdb.ondisk.tree",
        "location": pjoin(root, 'var', 'db', 'pkg'),
        "cache_location": '/var/cache/edb/dep/var/db/pkg',
    }
    config["vdb"] = basics.AutoConfigSection(kwds)

    try:
        repos_conf_defaults, repos_conf = load_repos_conf(pjoin(config_dir, 'repos.conf'))
    except errors.ParsingError as e:
        if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT:
            raise
        try:
            # fallback to defaults provided by pkgcore
            repos_conf_defaults, repos_conf = load_repos_conf(
                pjoin(const.CONFIG_PATH, 'repos.conf'))
        except IGNORED_EXCEPTIONS:
            raise
        except:
            raise_from(errors.ParsingError(
                "failed to find a usable repos.conf"))

    make_repo_syncers(config, repos_conf, make_conf)

    config['ebuild-repo-common'] = basics.AutoConfigSection({
        'class': 'pkgcore.ebuild.repository.tree',
        'default_mirrors': gentoo_mirrors,
        'inherit-only': True,
        'ignore_paludis_versioning': ('ignore-paludis-versioning' in features),
    })

    default_repo_path = repos_conf[repos_conf_defaults['main-repo']]['location']
    repo_map = {}

    for repo_name, repo_opts in repos_conf.iteritems():
        repo_path = repo_opts['location']

        # XXX: Hack for portage-2 profile format support.
        repo_config = RepoConfig(repo_path, repo_name)
        repo_map[repo_config.repo_id] = repo_config

        # repo configs
        repo_conf = {
            'class': 'pkgcore.ebuild.repo_objs.RepoConfig',
            'config_name': repo_name,
            'location': repo_path,
            'syncer': 'sync:' + repo_name,
        }

        # repo trees
        repo = {
            'inherit': ('ebuild-repo-common',),
            'repo_config': 'conf:' + repo_name,
        }

        # metadata cache
        if repo_config.cache_format is not None:
            cache_name = 'cache:' + repo_name
            config[cache_name] = make_cache(repo_config.cache_format, repo_path)
            repo['cache'] = cache_name

        if repo_path == default_repo_path:
            repo_conf['default'] = True

        config['conf:' + repo_name] = basics.AutoConfigSection(repo_conf)
        config[repo_name] = basics.AutoConfigSection(repo)

    # XXX: Hack for portage-2 profile format support. We need to figure out how
    # to dynamically create this from the config at runtime on attr access.
    profiles.ProfileNode._repo_map = ImmutableDict(repo_map)

    repos = [name for name in repos_conf.iterkeys()]
    if len(repos) > 1:
        config['repo-stack'] = basics.FakeIncrementalDictConfigSection(
            my_convert_hybrid, {
                'class': 'pkgcore.repository.multiplex.config_tree',
                'repositories': tuple(repos)})
    else:
        config['repo-stack'] = basics.section_alias(repos[0], 'repo')

    config['vuln'] = basics.AutoConfigSection({
        'class': SecurityUpgradesViaProfile,
        'ebuild_repo': 'repo-stack',
        'vdb': 'vdb',
        'profile': 'profile',
    })
    config['glsa'] = basics.section_alias(
        'vuln', SecurityUpgradesViaProfile.pkgcore_config_type.typename)

    # binpkg.
    buildpkg = 'buildpkg' in features or kwargs.pop('buildpkg', False)
    pkgdir = os.environ.get("PKGDIR", make_conf.pop('PKGDIR', None))
    if pkgdir is not None:
        try:
            pkgdir = abspath(pkgdir)
        except OSError as oe:
            if oe.errno != errno.ENOENT:
                raise
            if buildpkg or set(features).intersection(
                    ('pristine-buildpkg', 'buildsyspkg', 'unmerge-backup')):
                logger.warning("disabling buildpkg related features since PKGDIR doesn't exist")
            pkgdir = None
        else:
            if not ensure_dirs(pkgdir, mode=0755, minimal=True):
                logger.warning("disabling buildpkg related features since PKGDIR either doesn't "
                               "exist, or lacks 0755 minimal permissions")
                pkgdir = None
Example #9
0
    def __init__(self, location=None, profile_override=None, **kwargs):
        """
        Args:
            location (optional[str]): path to the portage config directory,
                (defaults to /etc/portage)
            profile_override (optional[str]): profile to use instead of the current system
                profile, i.e. the target of the /etc/portage/make.profile symlink
            configroot (optional[str]): location for various portage config files (defaults to /)
            root (optional[str]): target root filesystem (defaults to /)
            buildpkg (optional[bool]): forcibly disable/enable building binpkgs, otherwise
                FEATURES=buildpkg from make.conf is used

        Returns:
            dict: config settings
        """
        self._config = {}
        location = location if location is not None else '/etc/portage'
        self.dir = pjoin(
            os.environ.get('PORTAGE_CONFIGROOT', kwargs.pop('configroot',
                                                            '/')),
            location.lstrip('/'))

        # this actually differs from portage parsing- we allow
        # make.globals to provide vars used in make.conf, portage keeps
        # them separate (kind of annoying)
        #
        # this isn't preserving incremental behaviour for features/use unfortunately

        make_conf = {}
        try:
            self.load_make_conf(make_conf,
                                pjoin(const.CONFIG_PATH, 'make.globals'))
        except IGNORED_EXCEPTIONS:
            raise
        except Exception as e:
            raise errors.ParsingError("failed to load make.globals") from e
        self.load_make_conf(make_conf,
                            pjoin(self.dir, 'make.conf'),
                            required=False,
                            allow_sourcing=True,
                            incrementals=True)

        self.root = os.environ.get(
            "ROOT", kwargs.pop('root', make_conf.get("ROOT", "/")))
        gentoo_mirrors = [
            x.rstrip("/") + "/distfiles"
            for x in make_conf.pop("GENTOO_MIRRORS", "").split()
        ]

        self.features = frozenset(
            optimize_incrementals(make_conf.get('FEATURES', '').split()))

        self._add_sets()
        self._add_profile(profile_override)

        self['vdb'] = basics.AutoConfigSection({
            'class':
            'pkgcore.vdb.ondisk.tree',
            'location':
            pjoin(self.root, 'var', 'db', 'pkg'),
            'cache_location':
            '/var/cache/edb/dep/var/db/pkg',
        })

        try:
            repos_conf_defaults, repos_conf = self.load_repos_conf(
                pjoin(self.dir, 'repos.conf'))
        except errors.ParsingError as e:
            if not getattr(getattr(e, 'exc', None), 'errno',
                           None) == errno.ENOENT:
                raise
            try:
                # fallback to defaults provided by pkgcore
                repos_conf_defaults, repos_conf = self.load_repos_conf(
                    pjoin(const.CONFIG_PATH, 'repos.conf'))
            except IGNORED_EXCEPTIONS:
                raise
            except Exception as e:
                raise errors.ParsingError(
                    'failed to find a usable repos.conf') from e

        self['ebuild-repo-common'] = basics.AutoConfigSection({
            'class':
            'pkgcore.ebuild.repository.tree',
            'default_mirrors':
            gentoo_mirrors,
            'inherit-only':
            True,
        })

        repo_map = {}

        for repo_name, repo_opts in list(repos_conf.items()):
            repo_cls = repo_opts.pop('repo-type')
            try:
                repo = repo_cls(self,
                                repo_name=repo_name,
                                repo_opts=repo_opts,
                                repo_map=repo_map,
                                defaults=repos_conf_defaults)
            except repo_errors.UnsupportedRepo as e:
                logger.warning(
                    f'skipping {repo_name!r} repo: unsupported EAPI {str(e.repo.eapi)!r}'
                )
                del repos_conf[repo_name]
                continue

            self[repo_name] = basics.AutoConfigSection(repo)

        # XXX: Hack for portage-2 profile format support. We need to figure out how
        # to dynamically create this from the config at runtime on attr access.
        profiles.ProfileNode._repo_map = ImmutableDict(repo_map)

        self._make_repo_syncers(repos_conf, make_conf)
        repos = [name for name in repos_conf.keys()]
        if repos:
            if len(repos) > 1:
                self['repo-stack'] = basics.FakeIncrementalDictConfigSection(
                    my_convert_hybrid, {
                        'class': 'pkgcore.repository.multiplex.config_tree',
                        'repos': tuple(repos)
                    })
            else:
                self['repo-stack'] = basics.section_alias(repos[0], 'repo')

            self['vuln'] = basics.AutoConfigSection({
                'class': SecurityUpgradesViaProfile,
                'ebuild_repo': 'repo-stack',
                'vdb': 'vdb',
                'profile': 'profile',
            })
            self['glsa'] = basics.section_alias(
                'vuln',
                SecurityUpgradesViaProfile.pkgcore_config_type.typename)

        # check if package building was forced on by the user
        forced_buildpkg = kwargs.pop('buildpkg', False)
        if forced_buildpkg:
            make_conf['FEATURES'] += ' buildpkg'

        # now add the fetcher- we delay it till here to clean out the environ
        # it passes to the command.
        # *everything* in make_conf must be str values also.
        self._add_fetcher(make_conf)

        # finally... domain.
        make_conf.update({
            'class': 'pkgcore.ebuild.domain.domain',
            'repos': tuple(repos),
            'fetcher': 'fetcher',
            'default': True,
            'vdb': ('vdb', ),
            'profile': 'profile',
            'name': 'livefs',
            'root': self.root,
            'config_dir': self.dir,
        })

        self['livefs'] = basics.FakeIncrementalDictConfigSection(
            my_convert_hybrid, make_conf)
Example #10
0
    def __init__(self, location=None, profile_override=None, **kwargs):
        """
        Args:
            location (optional[str]): path to the portage config directory,
                (defaults to /etc/portage)
            profile_override (optional[str]): profile to use instead of the current system
                profile, i.e. the target of the /etc/portage/make.profile symlink
            configroot (optional[str]): location for various portage config files (defaults to /)
            root (optional[str]): target root filesystem (defaults to /)
            buildpkg (optional[bool]): forcibly disable/enable building binpkgs, otherwise
                FEATURES=buildpkg from make.conf is used

        Returns:
            dict: config settings
        """
        self._config = {}
        location = location if location is not None else '/etc/portage'
        self.dir = pjoin(
            os.environ.get('PORTAGE_CONFIGROOT', kwargs.pop('configroot', '/')),
            location.lstrip('/'))

        # this actually differs from portage parsing- we allow
        # make.globals to provide vars used in make.conf, portage keeps
        # them separate (kind of annoying)
        #
        # this isn't preserving incremental behaviour for features/use unfortunately

        make_conf = {}
        try:
            self.load_make_conf(make_conf, pjoin(const.CONFIG_PATH, 'make.globals'))
        except IGNORED_EXCEPTIONS:
            raise
        except Exception as e:
            raise errors.ParsingError("failed to load make.globals") from e
        self.load_make_conf(
            make_conf, pjoin(self.dir, 'make.conf'), required=False,
            allow_sourcing=True, incrementals=True)

        self.root = os.environ.get("ROOT", kwargs.pop('root', make_conf.get("ROOT", "/")))
        gentoo_mirrors = [
            x.rstrip("/") + "/distfiles" for x in make_conf.pop("GENTOO_MIRRORS", "").split()]

        self.features = frozenset(
            optimize_incrementals(make_conf.get('FEATURES', '').split()))

        self._add_sets()
        self._add_profile(profile_override)

        self['vdb'] = basics.AutoConfigSection({
            'class': 'pkgcore.vdb.ondisk.tree',
            'location': pjoin(self.root, 'var', 'db', 'pkg'),
            'cache_location': '/var/cache/edb/dep/var/db/pkg',
        })

        try:
            repos_conf_defaults, repos_conf = self.load_repos_conf(
                pjoin(self.dir, 'repos.conf'))
        except errors.ParsingError as e:
            if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT:
                raise
            try:
                # fallback to defaults provided by pkgcore
                repos_conf_defaults, repos_conf = self.load_repos_conf(
                    pjoin(const.CONFIG_PATH, 'repos.conf'))
            except IGNORED_EXCEPTIONS:
                raise
            except Exception as e:
                raise errors.ParsingError('failed to find a usable repos.conf') from e

        self['ebuild-repo-common'] = basics.AutoConfigSection({
            'class': 'pkgcore.ebuild.repository.tree',
            'default_mirrors': gentoo_mirrors,
            'inherit-only': True,
        })

        repo_map = {}

        for repo_name, repo_opts in list(repos_conf.items()):
            repo_cls = repo_opts.pop('repo-type')
            try:
                repo = repo_cls(
                    self, repo_name=repo_name, repo_opts=repo_opts,
                    repo_map=repo_map, defaults=repos_conf_defaults)
            except repo_errors.UnsupportedRepo as e:
                logger.warning(
                    f'skipping {repo_name!r} repo: unsupported EAPI {str(e.repo.eapi)!r}')
                del repos_conf[repo_name]
                continue

            self[repo_name] = basics.AutoConfigSection(repo)

        # XXX: Hack for portage-2 profile format support. We need to figure out how
        # to dynamically create this from the config at runtime on attr access.
        profiles.ProfileNode._repo_map = ImmutableDict(repo_map)

        self._make_repo_syncers(repos_conf, make_conf)
        repos = [name for name in repos_conf.keys()]
        if repos:
            if len(repos) > 1:
                self['repo-stack'] = basics.FakeIncrementalDictConfigSection(
                    my_convert_hybrid, {
                        'class': 'pkgcore.repository.multiplex.config_tree',
                        'repos': tuple(repos)})
            else:
                self['repo-stack'] = basics.section_alias(repos[0], 'repo')

            self['vuln'] = basics.AutoConfigSection({
                'class': SecurityUpgradesViaProfile,
                'ebuild_repo': 'repo-stack',
                'vdb': 'vdb',
                'profile': 'profile',
            })
            self['glsa'] = basics.section_alias(
                'vuln', SecurityUpgradesViaProfile.pkgcore_config_type.typename)

        # check if package building was forced on by the user
        forced_buildpkg = kwargs.pop('buildpkg', False)
        if forced_buildpkg:
            make_conf['FEATURES'] += ' buildpkg'

        # now add the fetcher- we delay it till here to clean out the environ
        # it passes to the command.
        # *everything* in make_conf must be str values also.
        self._add_fetcher(make_conf)

        # finally... domain.
        make_conf.update({
            'class': 'pkgcore.ebuild.domain.domain',
            'repos': tuple(repos),
            'fetcher': 'fetcher',
            'default': True,
            'vdb': ('vdb',),
            'profile': 'profile',
            'name': 'livefs',
            'root': self.root,
            'config_dir': self.dir,
        })

        self['livefs'] = basics.FakeIncrementalDictConfigSection(
            my_convert_hybrid, make_conf)
Example #11
0
        kwds = {
                'inherit': ('ebuild-repo-common',),
                'raw_repo': ('raw:' + tree_loc),
        }
        cache_name = 'cache:%s' % (tree_loc,)
        new_config[cache_name] = mk_simple_cache(config_root, tree_loc)
        kwds['cache'] = cache_name
        if tree_loc == portdir:
            kwds['class'] = 'pkgcore.ebuild.repository.tree'
            if rsync_portdir_cache:
                kwds['cache'] = 'cache:%s/metadata/cache %s' % (portdir, cache_name)
        else:
            kwds['parent_repo'] = portdir
        new_config[tree_loc] = basics.AutoConfigSection(kwds)

    new_config['portdir'] = basics.section_alias(portdir, 'repo')

    base_portdir_config = {}
    if portdir_syncer is not None:
        make_syncer(new_config, portdir, portdir_syncer, rsync_opts)

    if portdir_overlays:
        new_config['repo-stack'] = basics.FakeIncrementalDictConfigSection(
            my_convert_hybrid, {
                'class': 'pkgcore.repository.multiplex.config_tree',
                'repositories': tuple(reversed([portdir] + portdir_overlays))})
    else:
        new_config['repo-stack'] = basics.section_alias(portdir, 'repo')

    for tree_loc in [portdir] + portdir_overlays:
        conf = {'class':'pkgcore.ebuild.repo_objs.RepoConfig',