def matches_finalize(targets, namespace):
repos = multiplex.tree(*namespace.repos)
# If current working dir is in a repo, build a path restriction; otherwise
# match everything.
if not targets:
cwd = os.getcwd()
if cwd in repos:
targets = [cwd]
else:
return []
restrictions = []
for target in targets:
try:
restrictions.append(parserestrict.parse_match(target))
except parserestrict.ParseError as e:
if os.path.exists(target):
try:
restrictions.append(repos.path_restrict(target))
except ValueError as e:
argparser.error(e)
else:
argparser.error(e)
if restrictions:
return packages.OrRestriction(*restrictions)
return []
def _dist_validate_args(parser, namespace):
distdir = namespace.domain.fetcher.distdir
repo = multiplex.tree(*get_virtual_repos(namespace.domain.repos, False))
if not namespace.restrict:
namespace.restrict = packages.AlwaysTrue
files = set(os.path.basename(f) for f in listdir_files(distdir))
pfiles = set()
for pkg in repo.itermatch(namespace.restrict, sorter=sorted):
if ((namespace.installed and pkg.versioned_atom in namespace.installed)
or (namespace.fetch_restricted and 'fetch' in pkg.restrict)):
continue
try:
pfiles.update(fetchable.filename
for fetchable in iflatten_instance(
pkg.fetchables, fetch.fetchable))
except errors.MetadataException as e:
if not namespace.ignore_failures:
dist.error("got corruption error '%s', with package %s " %
(e, pkg.cpvstr))
except Exception as e:
dist.error("got error '%s', parsing package %s in repo '%s'" %
(e, pkg.cpvstr, pkg.repo))
distfiles = (pjoin(distdir, f) for f in files.intersection(pfiles))
removal_func = partial(os.remove)
namespace.remove = (
(removal_func, distfile)
for distfile in ifilter(namespace.filters.run, distfiles))
def _setup_scan(parser, namespace, args):
# determine target repo early in order to load relevant config settings if they exist
namespace, _ = parser._parse_known_args(args, namespace)
# load default args from system/user configs if config-loading is allowed
if namespace.config_file is None:
namespace = parser.parse_config_options(namespace)
# Get the current working directory for repo detection and restriction
# creation, fallback to the root dir if it's be removed out from under us.
try:
namespace.cwd = abspath(os.getcwd())
except FileNotFoundError:
namespace.cwd = '/'
# if we have no target repo figure out what to use
if namespace.target_repo is None:
target_repo = _determine_target_repo(namespace, parser)
# fallback to the default repo
if target_repo is None:
target_repo = namespace.config.get_default('repo')
namespace.target_repo = target_repo
# use filtered repo if requested
if namespace.filter == 'repo':
namespace.target_repo = namespace.domain.ebuild_repos[
namespace.target_repo.repo_id]
# determine if we're running in the gentoo repo or a clone
namespace.gentoo_repo = 'gentoo' in namespace.target_repo.aliases
# multiplex of target repo and its masters used for package existence queries
namespace.search_repo = multiplex.tree(*namespace.target_repo.trees)
# support loading repo-specific config settings from metadata/pkgcheck.conf
repo_config_file = os.path.join(namespace.target_repo.location, 'metadata',
'pkgcheck.conf')
configs = ()
if os.path.isfile(repo_config_file):
# repo settings take precedence over system/user settings
configs += (repo_config_file, )
if namespace.config_file is not None:
# and custom user settings take precedence over everything
if not namespace.config_file:
configs = ()
else:
configs += (namespace.config_file, )
if configs:
parser.parse_config(parser.configs + configs)
namespace = parser.parse_config_options(namespace)
# load repo-specific args from config if they exist, command line args override these
for section in namespace.target_repo.aliases:
if section in parser.config:
namespace = parser.parse_config_options(namespace, section)
break
return namespace, args
def prefer_reuse_strategy(cls, dbs):
return multiplex.tree(
misc.multiplex_sorting_repo(highest_iter_sort,
cls.just_livefs_dbs(dbs)),
misc.multiplex_sorting_repo(highest_iter_sort,
cls.just_nonlivefs_dbs(dbs)),
)
def _dist_validate_args(parser, namespace):
distdir = namespace.domain.fetcher.distdir
repo = multiplex.tree(*get_virtual_repos(namespace.domain.repos, False))
if not namespace.restrict:
namespace.restrict = packages.AlwaysTrue
files = set(os.path.basename(f) for f in listdir_files(distdir))
pfiles = set()
for pkg in repo.itermatch(namespace.restrict, sorter=sorted):
if ((namespace.installed and pkg.versioned_atom in namespace.installed) or
(namespace.fetch_restricted and 'fetch' in pkg.restrict)):
continue
try:
pfiles.update(
fetchable.filename for fetchable in
iflatten_instance(pkg.fetchables, fetch.fetchable))
except errors.MetadataException as e:
if not namespace.ignore_failures:
dist.error(
"got corruption error '%s', with package %s " %
(e, pkg.cpvstr))
except Exception as e:
dist.error(
"got error '%s', parsing package %s in repo '%s'" %
(e, pkg.cpvstr, pkg.repo))
distfiles = (pjoin(distdir, f) for f in files.intersection(pfiles))
removal_func = partial(os.remove)
namespace.remove = (
(removal_func, distfile) for distfile in
ifilter(namespace.filters.run, distfiles))
def prefer_reuse_strategy(self, dbs):
return multiplex.tree(
misc.multiplex_sorting_repo(highest_iter_sort,
*list(self.just_livefs_dbs(dbs))),
misc.multiplex_sorting_repo(highest_iter_sort,
*list(self.just_nonlivefs_dbs(dbs)))
)
def __init__(self,
dbs,
per_repo_strategy,
global_strategy=None,
depset_reorder_strategy=None,
process_built_depends=False,
drop_cycles=False,
debug=False,
debug_handle=None):
if debug_handle is None:
debug_handle = sys.stdout
self.debug_handler = debug_handle
self._dprint = partial(dprint, debug_handle)
if not isinstance(dbs, (list, tuple)):
dbs = [dbs]
if global_strategy is None:
global_strategy = self.default_global_strategy
if depset_reorder_strategy is None:
depset_reorder_strategy = self.default_depset_reorder_strategy
self.depset_reorder = depset_reorder_strategy
self.per_repo_strategy = per_repo_strategy
self.total_ordering_strategy = global_strategy
self.all_raw_dbs = [
misc.caching_repo(x, self.per_repo_strategy) for x in dbs
]
self.all_dbs = global_strategy(self.all_raw_dbs)
self.default_dbs = self.all_dbs
self.state = state.plan_state()
vdb_state_filter_restrict = MutableContainmentRestriction(
self.state.vdb_filter)
self.livefs_dbs = multiplex.tree(*[
visibility.filterTree(x, vdb_state_filter_restrict)
for x in self.all_raw_dbs if x.livefs
])
self.insoluble = set()
self.vdb_preloaded = False
self._ensure_livefs_is_loaded = \
self._ensure_livefs_is_loaded_nonpreloaded
self.drop_cycles = drop_cycles
self.skipdeps = ()
self.process_built_depends = process_built_depends
self._debugging = debug
if debug:
self._rec_add_atom = partial(self._stack_debugging_rec_add_atom,
self._rec_add_atom)
self._debugging_depth = 0
self._debugging_drop_cycles = False
def __init__(self, dbs, *args, **kwds):
"""
:param args: see :obj:`pkgcore.resolver.plan.merge_plan.__init__`
for valid args
:param kwds: see :obj:`pkgcore.resolver.plan.merge_plan.__init__`
for valid args
"""
plan.merge_plan.__init__(self, dbs, *args, **kwds)
# XXX *cough*, hack.
self.default_dbs = multiplex.tree(
*[x for x in self.all_raw_dbs if not x.livefs])
def __init__(self, dbs, *args, **kwds):
"""
:param args: see :obj:`pkgcore.resolver.plan.merge_plan.__init__`
for valid args
:param kwds: see :obj:`pkgcore.resolver.plan.merge_plan.__init__`
for valid args
"""
plan.merge_plan.__init__(self, dbs, *args, **kwds)
# XXX *cough*, hack.
self.default_dbs = multiplex.tree(*
[x for x in self.all_raw_dbs if not x.livefs])
def __init__(self,
dbs,
per_repo_strategy,
global_strategy=None,
depset_reorder_strategy=None,
process_built_depends=False,
drop_cycles=False,
debug=False,
debug_handle=None):
if debug:
if debug_handle is None:
debug_handle = sys.stdout
self._dprint = partial(dprint, debug_handle)
else:
# don't run debug func when debugging is disabled
self._dprint = lambda *args, **kwargs: None
if not isinstance(dbs, (util.RepositoryGroup, list, tuple)):
dbs = [dbs]
if global_strategy is None:
global_strategy = self.default_global_strategy
if depset_reorder_strategy is None:
depset_reorder_strategy = self.default_depset_reorder_strategy
self.depset_reorder = depset_reorder_strategy
self.all_raw_dbs = [
misc.caching_repo(x, per_repo_strategy) for x in dbs
]
self.all_dbs = global_strategy(self.all_raw_dbs)
self.default_dbs = self.all_dbs
self.state = state.plan_state()
vdb_state_filter_restrict = MutableContainmentRestriction(
self.state.vdb_filter)
self.livefs_dbs = multiplex.tree(*[
filtered.tree(x, vdb_state_filter_restrict)
for x in self.all_raw_dbs if x.livefs
])
self.insoluble = set()
self.vdb_preloaded = False
self._ensure_livefs_is_loaded = \
self._ensure_livefs_is_loaded_nonpreloaded
self.drop_cycles = drop_cycles
self.process_built_depends = process_built_depends
self._debugging = debug
if debug:
self._rec_add_atom = partial(self._stack_debugging_rec_add_atom,
self._rec_add_atom)
self._debugging_depth = 0
self._debugging_drop_cycles = False
def __getitem__(self, key):
if key not in self._supported_attrs:
raise KeyError
try:
return self.unfiltered_repos[key]
except KeyError:
repos = []
kwargs = {key: ()}
for repo in self.domain.ebuild_repos_unfiltered:
repos.append(self.domain.filter_repo(repo, **kwargs))
unfiltered_repo = multiplex.tree(*repos)
self.unfiltered_repos[key] = unfiltered_repo
return unfiltered_repo
def __getitem__(self, key):
if key not in self._supported_attrs:
raise KeyError
try:
return self.unfiltered_repos[key]
except KeyError:
repos = []
kwargs = {key: ()}
for repo in self.domain.ebuild_repos_unfiltered:
repos.append(self.domain.filter_repo(repo, **kwargs))
unfiltered_repo = multiplex.tree(*repos)
self.unfiltered_repos[key] = unfiltered_repo
return unfiltered_repo
def __init__(self, dbs, per_repo_strategy,
global_strategy=None,
depset_reorder_strategy=None,
process_built_depends=False,
drop_cycles=False, debug=False, debug_handle=None):
if debug_handle is None:
debug_handle = sys.stdout
self.debug_handler = debug_handle
self._dprint = partial(dprint, debug_handle)
if not isinstance(dbs, (list, tuple)):
dbs = [dbs]
if global_strategy is None:
global_strategy = self.default_global_strategy
if depset_reorder_strategy is None:
depset_reorder_strategy = self.default_depset_reorder_strategy
self.depset_reorder = depset_reorder_strategy
self.per_repo_strategy = per_repo_strategy
self.total_ordering_strategy = global_strategy
self.all_raw_dbs = [misc.caching_repo(x, self.per_repo_strategy) for x in dbs]
self.all_dbs = global_strategy(self, self.all_raw_dbs)
self.default_dbs = self.all_dbs
self.state = state.plan_state()
vdb_state_filter_restrict = MutableContainmentRestriction(self.state.vdb_filter)
self.livefs_dbs = multiplex.tree(
*[visibility.filterTree(x, vdb_state_filter_restrict)
for x in self.all_raw_dbs if x.livefs])
self.insoluble = set()
self.vdb_preloaded = False
self._ensure_livefs_is_loaded = \
self._ensure_livefs_is_loaded_nonpreloaded
self.drop_cycles = drop_cycles
self.process_built_depends = process_built_depends
self._debugging = debug
if debug:
self._rec_add_atom = partial(self._stack_debugging_rec_add_atom,
self._rec_add_atom)
self._debugging_depth = 0
self._debugging_drop_cycles = False
def cached_repo(self, repo_cls, target_repo=None):
cached_repo = None
if target_repo is None:
target_repo = self.options.target_repo
if self.options.cache['git']:
git_repos = []
for repo in target_repo.trees:
git_repo = self._cached_repos.get(repo.location, None)
# only enable repo queries if history was found, e.g. a
# shallow clone with a depth of 1 won't have any history
if git_repo:
git_repos.append(repo_cls(git_repo, repo_id=f'{repo.repo_id}-history'))
else:
logger.warning('skipping git checks for %s repo', repo)
break
else:
if len(git_repos) > 1:
cached_repo = multiplex.tree(*git_repos)
elif len(git_repos) == 1:
cached_repo = git_repos[0]
return cached_repo
def _dist_validate_args(parser, namespace):
distdir = namespace.domain.fetcher.distdir
repo = namespace.repo
if repo is None:
repo = multiplex.tree(*get_virtual_repos(namespace.domain.source_repos, False))
all_dist_files = set(os.path.basename(f) for f in listdir_files(distdir))
target_files = set()
installed_dist = set()
exists_dist = set()
excludes_dist = set()
restricted_dist = set()
# exclude distfiles used by installed packages -- note that this uses the
# distfiles attr with USE settings bound to it
if namespace.exclude_installed:
for pkg in namespace.domain.all_installed_repos:
installed_dist.update(iflatten_instance(pkg.distfiles))
# exclude distfiles for existing ebuilds or fetch restrictions
if namespace.exclude_fetch_restricted or (namespace.exclude_exists and not namespace.restrict):
for pkg in repo:
exists_dist.update(iflatten_instance(getattr(pkg, '_raw_pkg', pkg).distfiles))
if 'fetch' in pkg.restrict:
restricted_dist.update(iflatten_instance(getattr(pkg, '_raw_pkg', pkg).distfiles))
# exclude distfiles from specified restrictions
if namespace.exclude_restrict:
for pkg in repo.itermatch(namespace.exclude_restrict, sorter=sorted):
excludes_dist.update(iflatten_instance(getattr(pkg, '_raw_pkg', pkg).distfiles))
# determine dist files for custom restrict targets
if namespace.restrict:
target_dist = defaultdict(lambda: defaultdict(set))
for pkg in repo.itermatch(namespace.restrict, sorter=sorted):
s = set(iflatten_instance(getattr(pkg, '_raw_pkg', pkg).distfiles))
target_dist[pkg.unversioned_atom][pkg].update(s)
if namespace.exclude_exists:
exists_dist.update(s)
extra_regex_prefixes = defaultdict(set)
pkg_regex_prefixes = set()
for catpn, pkgs in target_dist.items():
pn_regex = r'\W'.join(re.split(r'\W', catpn.package))
pkg_regex = re.compile(r'(%s)(\W\w+)+([\W?(0-9)+])*(\W\w+)*(\.\w+)*' % pn_regex,
re.IGNORECASE)
pkg_regex_prefixes.add(pn_regex)
for pkg, files in pkgs.items():
files = sorted(files)
for f in files:
if (pkg_regex.match(f) or (
extra_regex_prefixes and
re.match(r'(%s)([\W?(0-9)+])+(\W\w+)*(\.\w+)+' % '|'.join(extra_regex_prefixes[catpn]), f))):
continue
else:
pieces = re.split(r'([\W?(0-9)+])+(\W\w+)*(\.\w+)+', f)
if pieces[-1] == '':
pieces.pop()
if len(pieces) > 1:
extra_regex_prefixes[catpn].add(pieces[0])
if target_dist:
regexes = []
# build regexes to match distfiles for older ebuilds no longer in the tree
if pkg_regex_prefixes:
pkg_regex_prefixes_str = '|'.join(sorted(pkg_regex_prefixes))
regexes.append(re.compile(r'(%s)(\W\w+)+([\W?(0-9)+])*(\W\w+)*(\.\w+)*' % (
pkg_regex_prefixes_str,)))
if extra_regex_prefixes:
extra_regex_prefixes_str = '|'.join(sorted(chain.from_iterable(
v for k, v in extra_regex_prefixes.items())))
regexes.append(re.compile(r'(%s)([\W?(0-9)+])+(\W\w+)*(\.\w+)+' % (
extra_regex_prefixes_str,)))
if regexes:
for f in all_dist_files:
if any(r.match(f) for r in regexes):
target_files.add(f)
else:
target_files = all_dist_files
# exclude files tagged for saving
saving_files = installed_dist | exists_dist | excludes_dist | restricted_dist
target_files.difference_update(saving_files)
targets = (pjoin(distdir, f) for f in sorted(all_dist_files.intersection(target_files)))
removal_func = partial(os.remove)
namespace.remove = (
(removal_func, f) for f in
filter(namespace.file_filters.run, targets))
def __init__(self,
profile,
repositories,
vdb,
name=None,
root='/',
prefix='/',
incrementals=const.incrementals,
triggers=(),
**settings):
# voodoo, unfortunately (so it goes)
# break this up into chunks once it's stabilized (most of code
# here has already, but still more to add)
self._triggers = triggers
# prevent critical variables from being changed by the user in make.conf
for k in set(profile.profile_only_variables).intersection(
settings.keys()):
del settings[k]
if 'CHOST' in settings and 'CBUILD' not in settings:
settings['CBUILD'] = settings['CHOST']
# map out sectionname -> config manager immediately.
repositories_collapsed = [r.collapse() for r in repositories]
repositories = [r.instantiate() for r in repositories_collapsed]
self.fetcher = settings.pop("fetcher")
self.default_licenses_manager = OverlayedLicenses(*repositories)
vdb_collapsed = [r.collapse() for r in vdb]
vdb = [r.instantiate() for r in vdb_collapsed]
self.repos_raw = {
collapsed.name: repo
for (collapsed, repo) in izip(repositories_collapsed, repositories)
}
self.repos_raw.update(
(collapsed.name, repo)
for (collapsed, repo) in izip(vdb_collapsed, vdb))
self.repos_raw.pop(None, None)
if profile.provides_repo is not None:
self.repos_raw['package.provided'] = profile.provides_repo
vdb.append(profile.provides_repo)
self.profile = profile
pkg_maskers, pkg_unmaskers, pkg_keywords, pkg_licenses = [], [], [], []
pkg_use, self.bashrcs = [], []
self.ebuild_hook_dir = settings.pop("ebuild_hook_dir", None)
for key, val, action in (
("package.mask", pkg_maskers, parse_match),
("package.unmask", pkg_unmaskers, parse_match),
("package.keywords", pkg_keywords, package_keywords_splitter),
("package.accept_keywords", pkg_keywords,
package_keywords_splitter),
("package.license", pkg_licenses, package_keywords_splitter),
("package.use", pkg_use, package_keywords_splitter),
("package.env", self.bashrcs, package_env_splitter),
):
for fp in settings.pop(key, ()):
try:
if key == "package.env":
base = self.ebuild_hook_dir
if base is None:
base = os.path.dirname(fp)
action = partial(action, base)
for fs_obj in iter_scan(fp, follow_symlinks=True):
if not fs_obj.is_reg or '/.' in fs_obj.location:
continue
val.extend(
action(x) for x in iter_read_bash(fs_obj.location))
except EnvironmentError as e:
if e.errno == errno.ENOENT:
raise MissingFile(fp, key)
raise_from(Failure("failed reading '%s': %s" % (fp, e)))
except ValueError as e:
raise_from(Failure("failed reading '%s': %s" % (fp, e)))
self.name = name
settings.setdefault("PKGCORE_DOMAIN", name)
for x in incrementals:
if isinstance(settings.get(x), basestring):
settings[x] = tuple(settings[x].split())
# roughly... all incremental stacks should be interpreted left -> right
# as such we start with the profile settings, and append ours onto it.
for k, v in profile.default_env.iteritems():
if k not in settings:
settings[k] = v
continue
if k in incrementals:
settings[k] = v + tuple(settings[k])
# next we finalize incrementals.
for incremental in incrementals:
# Skip USE/ACCEPT_LICENSE for the time being; hack; we need the
# negations currently so that pkg iuse induced enablings can be
# disabled by negations. For example, think of the profile doing
# USE=-cdr for brasero w/ IUSE=+cdr. Similarly, ACCEPT_LICENSE is
# skipped because negations are required for license filtering.
if incremental not in settings or incremental in (
"USE", "ACCEPT_LICENSE"):
continue
s = set()
incremental_expansion(s, settings[incremental],
'While expanding %s ' % (incremental, ))
settings[incremental] = tuple(s)
# use is collapsed; now stack use_expand.
use = settings['USE'] = set(
optimize_incrementals(settings.get("USE", ())))
self._extend_use_for_features(use, settings.get("FEATURES", ()))
self.use_expand = frozenset(profile.use_expand)
self.use_expand_hidden = frozenset(profile.use_expand_hidden)
for u in profile.use_expand:
v = settings.get(u)
if v is None:
continue
u2 = u.lower() + "_"
use.update(u2 + x for x in v.split())
if not 'ACCEPT_KEYWORDS' in settings:
raise Failure("No ACCEPT_KEYWORDS setting detected from profile, "
"or user config")
s = set()
default_keywords = []
incremental_expansion(s, settings['ACCEPT_KEYWORDS'],
'while expanding ACCEPT_KEYWORDS')
default_keywords.extend(s)
settings['ACCEPT_KEYWORDS'] = set(default_keywords)
self.use = use
if "ARCH" not in settings:
raise Failure(
"No ARCH setting detected from profile, or user config")
self.arch = self.stable_arch = settings["ARCH"]
self.unstable_arch = "~%s" % self.arch
# ~amd64 -> [amd64, ~amd64]
for x in default_keywords[:]:
if x.startswith("~"):
default_keywords.append(x.lstrip("~"))
default_keywords = unstable_unique(default_keywords + [self.arch])
accept_keywords = pkg_keywords + list(profile.accept_keywords)
vfilters = [
self.make_keywords_filter(self.arch,
default_keywords,
accept_keywords,
profile.keywords,
incremental="package.keywords"
in incrementals)
]
del default_keywords, accept_keywords
# we can finally close that fricking
# "DISALLOW NON FOSS LICENSES" bug via this >:)
master_license = []
master_license.extend(settings.get('ACCEPT_LICENSE', ()))
if master_license or pkg_licenses:
vfilters.append(
self.make_license_filter(master_license, pkg_licenses))
del master_license
# if it's made it this far...
self.root = settings["ROOT"] = root
self.prefix = prefix
self.settings = ProtectedDict(settings)
for data in self.settings.get('bashrc', ()):
source = local_source(data)
# this is currently local-only so a path check is ok
# TODO make this more general
if not os.path.exists(source.path):
raise Failure('user-specified bashrc %r does not exist' %
(data, ))
self.bashrcs.append((packages.AlwaysTrue, source))
# stack use stuff first, then profile.
self.enabled_use = ChunkedDataDict()
self.enabled_use.add_bare_global(*split_negations(self.use))
self.enabled_use.merge(profile.pkg_use)
self.enabled_use.update_from_stream(
chunked_data(k, *split_negations(v)) for k, v in pkg_use)
for attr in ('', 'stable_'):
c = ChunkedDataDict()
c.merge(getattr(profile, attr + 'forced_use'))
c.add_bare_global((), (self.arch, ))
setattr(self, attr + 'forced_use', c)
c = ChunkedDataDict()
c.merge(getattr(profile, attr + 'masked_use'))
setattr(self, attr + 'disabled_use', c)
self.repos = []
self.vdb = []
self.repos_configured = {}
self.repos_configured_filtered = {}
rev_names = {repo: name for name, repo in self.repos_raw.iteritems()}
profile_masks = profile._incremental_masks()
profile_unmasks = profile._incremental_unmasks()
repo_masks = {
r.repo_id: r._visibility_limiters()
for r in repositories
}
for l, repos, filtered in ((self.repos, repositories, True),
(self.vdb, vdb, False)):
for repo in repos:
if not repo.configured:
pargs = [repo]
try:
for x in repo.configurables:
if x == "domain":
pargs.append(self)
elif x == "settings":
pargs.append(settings)
elif x == "profile":
pargs.append(profile)
else:
pargs.append(getattr(self, x))
except AttributeError as ae:
raise_from(
Failure("failed configuring repo '%s': "
"configurable missing: %s" % (repo, ae)))
wrapped_repo = repo.configure(*pargs)
else:
wrapped_repo = repo
key = rev_names.get(repo)
self.repos_configured[key] = wrapped_repo
if filtered:
config = getattr(repo, 'config', None)
masters = getattr(config, 'masters', ())
if masters is None:
# tough cookies. If a user has an overlay, no masters
# defined, we're not applying the portdir masks.
# we do this both since that's annoying, and since
# frankly there isn't any good course of action.
masters = ()
masks = [
repo_masks.get(master, [(), ()]) for master in masters
]
masks.append(repo_masks[repo.repo_id])
masks.extend(profile_masks)
mask_atoms = set()
for neg, pos in masks:
mask_atoms.difference_update(neg)
mask_atoms.update(pos)
mask_atoms.update(pkg_maskers)
unmask_atoms = set(chain(pkg_unmaskers, *profile_unmasks))
filtered = self.generate_filter(
generate_masking_restrict(mask_atoms),
generate_unmasking_restrict(unmask_atoms), *vfilters)
if filtered:
wrapped_repo = visibility.filterTree(
wrapped_repo, filtered, True)
self.repos_configured_filtered[key] = wrapped_repo
l.append(wrapped_repo)
if profile.virtuals:
l = [
x for x in (getattr(v, 'old_style_virtuals', None)
for v in self.vdb) if x is not None
]
profile_repo = profile.make_virtuals_repo(
multiplex.tree(*repositories), *l)
self.repos_raw["profile virtuals"] = profile_repo
self.repos_configured_filtered["profile virtuals"] = profile_repo
self.repos_configured["profile virtuals"] = profile_repo
self.repos = [profile_repo] + self.repos
self.use_expand_re = re.compile(
"^(?:[+-])?(%s)_(.*)$" %
"|".join(x.lower() for x in sorted(self.use_expand, reverse=True)))
def __init__(self, repos):
self.all_ebuild_repos_raw = multiplex.tree(*repos)
self.root = None
def _dist_validate_args(parser, namespace):
distdir = namespace.domain.fetcher.distdir
repo = namespace.repo
if repo is None:
repo = multiplex.tree(
*get_virtual_repos(namespace.domain.source_repos, False))
all_dist_files = set(os.path.basename(f) for f in listdir_files(distdir))
target_files = set()
installed_dist = set()
exists_dist = set()
excludes_dist = set()
restricted_dist = set()
# exclude distfiles used by installed packages -- note that this uses the
# distfiles attr with USE settings bound to it
if namespace.exclude_installed:
for pkg in namespace.domain.all_installed_repos:
installed_dist.update(iflatten_instance(pkg.distfiles))
# exclude distfiles for existing ebuilds or fetch restrictions
if namespace.exclude_fetch_restricted or (namespace.exclude_exists
and not namespace.restrict):
for pkg in repo:
exists_dist.update(
iflatten_instance(getattr(pkg, '_raw_pkg', pkg).distfiles))
if 'fetch' in pkg.restrict:
restricted_dist.update(
iflatten_instance(getattr(pkg, '_raw_pkg', pkg).distfiles))
# exclude distfiles from specified restrictions
if namespace.exclude_restrict:
for pkg in repo.itermatch(namespace.exclude_restrict, sorter=sorted):
excludes_dist.update(
iflatten_instance(getattr(pkg, '_raw_pkg', pkg).distfiles))
# determine dist files for custom restrict targets
if namespace.restrict:
target_dist = defaultdict(lambda: defaultdict(set))
for pkg in repo.itermatch(namespace.restrict, sorter=sorted):
s = set(iflatten_instance(getattr(pkg, '_raw_pkg', pkg).distfiles))
target_dist[pkg.unversioned_atom][pkg].update(s)
if namespace.exclude_exists:
exists_dist.update(s)
extra_regex_prefixes = defaultdict(set)
pkg_regex_prefixes = set()
for catpn, pkgs in target_dist.items():
pn_regex = r'\W'.join(re.split(r'\W', catpn.package))
pkg_regex = re.compile(
r'(%s)(\W\w+)+([\W?(0-9)+])*(\W\w+)*(\.\w+)*' % pn_regex,
re.IGNORECASE)
pkg_regex_prefixes.add(pn_regex)
for pkg, files in pkgs.items():
files = sorted(files)
for f in files:
if (pkg_regex.match(f)
or (extra_regex_prefixes and re.match(
r'(%s)([\W?(0-9)+])+(\W\w+)*(\.\w+)+' %
'|'.join(extra_regex_prefixes[catpn]), f))):
continue
else:
pieces = re.split(r'([\W?(0-9)+])+(\W\w+)*(\.\w+)+', f)
if pieces[-1] == '':
pieces.pop()
if len(pieces) > 1:
extra_regex_prefixes[catpn].add(pieces[0])
if target_dist:
regexes = []
# build regexes to match distfiles for older ebuilds no longer in the tree
if pkg_regex_prefixes:
pkg_regex_prefixes_str = '|'.join(sorted(pkg_regex_prefixes))
regexes.append(
re.compile(r'(%s)(\W\w+)+([\W?(0-9)+])*(\W\w+)*(\.\w+)*' %
(pkg_regex_prefixes_str, )))
if extra_regex_prefixes:
extra_regex_prefixes_str = '|'.join(
sorted(
chain.from_iterable(
v for k, v in extra_regex_prefixes.items())))
regexes.append(
re.compile(r'(%s)([\W?(0-9)+])+(\W\w+)*(\.\w+)+' %
(extra_regex_prefixes_str, )))
if regexes:
for f in all_dist_files:
if any(r.match(f) for r in regexes):
target_files.add(f)
else:
target_files = all_dist_files
# exclude files tagged for saving
saving_files = installed_dist | exists_dist | excludes_dist | restricted_dist
target_files.difference_update(saving_files)
targets = (pjoin(distdir, f)
for f in sorted(all_dist_files.intersection(target_files)))
removal_func = partial(os.remove)
namespace.remove = ((removal_func, f)
for f in filter(namespace.file_filters.run, targets))
def _validate_scan_args(parser, namespace):
namespace.enabled_checks = list(const.CHECKS.values())
namespace.enabled_keywords = list(const.KEYWORDS.values())
# Get the current working directory for repo detection and restriction
# creation, fallback to the root dir if it's be removed out from under us.
try:
cwd = abspath(os.getcwd())
except FileNotFoundError as e:
cwd = '/'
# if we have no target repo figure out what to use
if namespace.target_repo is None:
target_repo = _determine_target_repo(namespace, parser, cwd)
# fallback to the default repo
if target_repo is None:
target_repo = namespace.config.get_default('repo')
namespace.target_repo = target_repo
# use filtered repo if filtering is enabled
if namespace.filtered:
namespace.target_repo = namespace.domain.ebuild_repos[
namespace.target_repo.repo_id]
# determine if we're running in the gentoo repo or a clone
namespace.gentoo_repo = 'gentoo' in namespace.target_repo.aliases
# multiplex of target repo and its masters used for package existence queries
namespace.search_repo = multiplex.tree(*namespace.target_repo.trees)
if namespace.targets:
repo = namespace.target_repo
# read targets from stdin in a non-blocking manner
if len(namespace.targets) == 1 and namespace.targets[0] == '-':
def stdin():
while True:
line = sys.stdin.readline()
if not line:
break
yield line.rstrip()
namespace.targets = stdin()
def restrictions():
for target in namespace.targets:
try:
r = parserestrict.parse_match(target)
except parserestrict.ParseError as e:
if os.path.exists(target):
try:
r = _path_restrict(target, namespace)
except ValueError as e:
parser.error(e)
else:
parser.error(e)
yield _restrict_to_scope(r), r
# Collapse restrictions for passed in targets while keeping the
# generator intact for piped in targets.
namespace.restrictions = restrictions()
if isinstance(namespace.targets, list):
namespace.restrictions = list(namespace.restrictions)
# collapse restrictions in order to run them in parallel
if len(namespace.restrictions) > 1:
# multiple targets are restricted to a single scanning scope
scopes = {scope for scope, restrict in namespace.restrictions}
if len(scopes) > 1:
scan_scopes = ', '.join(sorted(map(str, scopes)))
parser.error(
f'targets specify multiple scan scope levels: {scan_scopes}'
)
combined_restrict = boolean.OrRestriction(
*(r for s, r in namespace.restrictions))
namespace.restrictions = [(scopes.pop(), combined_restrict)]
else:
if cwd in namespace.target_repo:
restrict = _path_restrict(cwd, namespace)
else:
restrict = packages.AlwaysTrue
namespace.restrictions = [(_restrict_to_scope(restrict), restrict)]
if namespace.checkset is None:
namespace.checkset = namespace.config.get_default('pkgcheck_checkset')
if namespace.checkset is not None:
namespace.enabled_checks = list(
namespace.checkset.filter(namespace.enabled_checks))
if namespace.selected_scopes is not None:
disabled_scopes, enabled_scopes = namespace.selected_scopes
# validate selected scopes
selected_scopes = set(disabled_scopes + enabled_scopes)
unknown_scopes = selected_scopes - set(base.scopes)
if unknown_scopes:
unknown = ', '.join(map(repr, unknown_scopes))
available = ', '.join(base.scopes)
parser.error(f'unknown scope{_pl(unknown_scopes)}: '
f'{unknown} (available scopes: {available})')
disabled_scopes = {base.scopes[x] for x in disabled_scopes}
enabled_scopes = {base.scopes[x] for x in enabled_scopes}
# convert scopes to keyword lists
disabled_keywords = [
k.__name__ for k in const.KEYWORDS.values()
if k.scope in disabled_scopes
]
enabled_keywords = [
k.__name__ for k in const.KEYWORDS.values()
if k.scope in enabled_scopes
]
# filter outputted keywords
namespace.enabled_keywords = base.filter_update(
namespace.enabled_keywords, enabled_keywords, disabled_keywords)
if namespace.selected_keywords is not None:
disabled_keywords, enabled_keywords = namespace.selected_keywords
error = (k for k, v in const.KEYWORDS.items()
if issubclass(v, results.Error))
warning = (k for k, v in const.KEYWORDS.items()
if issubclass(v, results.Warning))
info = (k for k, v in const.KEYWORDS.items()
if issubclass(v, results.Info))
alias_map = {'error': error, 'warning': warning, 'info': info}
replace_aliases = lambda x: alias_map.get(x, [x])
# expand keyword aliases to keyword lists
disabled_keywords = list(
chain.from_iterable(map(replace_aliases, disabled_keywords)))
enabled_keywords = list(
chain.from_iterable(map(replace_aliases, enabled_keywords)))
# validate selected keywords
selected_keywords = set(disabled_keywords + enabled_keywords)
available_keywords = set(const.KEYWORDS.keys())
unknown_keywords = selected_keywords - available_keywords
if unknown_keywords:
unknown = ', '.join(map(repr, unknown_keywords))
parser.error(f'unknown keyword{_pl(unknown_keywords)}: {unknown}')
# filter outputted keywords
namespace.enabled_keywords = base.filter_update(
namespace.enabled_keywords, enabled_keywords, disabled_keywords)
namespace.filtered_keywords = set(namespace.enabled_keywords)
if namespace.filtered_keywords == set(const.KEYWORDS.values()):
namespace.filtered_keywords = None
disabled_checks, enabled_checks = ((), ())
if namespace.selected_checks is not None:
disabled_checks, enabled_checks = namespace.selected_checks
available_checks = list(const.CHECKS.keys())
alias_map = {'all': available_checks}
replace_aliases = lambda x: alias_map.get(x, [x])
# expand check aliases to check lists
disabled_checks = list(
chain.from_iterable(map(replace_aliases, disabled_checks)))
enabled_checks = list(
chain.from_iterable(map(replace_aliases, enabled_checks)))
# overwrite selected checks with expanded aliases
namespace.selected_checks = (disabled_checks, enabled_checks)
# validate selected checks
selected_checks = set(disabled_checks + enabled_checks)
unknown_checks = selected_checks.difference(available_checks)
if unknown_checks:
unknown = ', '.join(map(repr, unknown_checks))
parser.error(f'unknown check{_pl(unknown_checks)}: {unknown} ')
elif namespace.filtered_keywords is not None:
# enable checks based on enabled keyword -> check mapping
enabled_checks = []
for check, cls in const.CHECKS.items():
if namespace.filtered_keywords.intersection(cls.known_results):
enabled_checks.append(check)
# filter checks to run
if enabled_checks:
whitelist = base.Whitelist(enabled_checks)
namespace.enabled_checks = list(
whitelist.filter(namespace.enabled_checks))
if disabled_checks:
blacklist = base.Blacklist(disabled_checks)
namespace.enabled_checks = list(
blacklist.filter(namespace.enabled_checks))
# skip checks that may be disabled
namespace.enabled_checks = [
c for c in namespace.enabled_checks if not c.skip(namespace)
]
if not namespace.enabled_checks:
parser.error('no active checks')
namespace.addons = set()
for check in namespace.enabled_checks:
add_addon(check, namespace.addons)
try:
for addon in namespace.addons:
addon.check_args(parser, namespace)
except argparse.ArgumentError as e:
if namespace.debug:
raise
parser.error(str(e))
def __init__(self, profile, repositories, vdb, name=None,
root='/', prefix='/', incrementals=const.incrementals,
triggers=(), **settings):
# voodoo, unfortunately (so it goes)
# break this up into chunks once it's stabilized (most of code
# here has already, but still more to add)
self._triggers = triggers
# prevent critical variables from being changed in make.conf
for k in profile.profile_only_variables.intersection(settings.keys()):
del settings[k]
if 'CHOST' in settings and 'CBUILD' not in settings:
settings['CBUILD'] = settings['CHOST']
# if unset, MAKEOPTS defaults to CPU thread count
if 'MAKEOPTS' not in settings:
settings['MAKEOPTS'] = '-j%i' % get_proc_count()
# map out sectionname -> config manager immediately.
repositories_collapsed = [r.collapse() for r in repositories]
repositories = [r.instantiate() for r in repositories_collapsed]
self.fetcher = settings.pop("fetcher")
self.default_licenses_manager = OverlayedLicenses(*repositories)
vdb_collapsed = [r.collapse() for r in vdb]
vdb = [r.instantiate() for r in vdb_collapsed]
self.repos_raw = {
collapsed.name: repo for (collapsed, repo) in izip(
repositories_collapsed, repositories)}
self.repos_raw.update(
(collapsed.name, repo) for (collapsed, repo) in izip(
vdb_collapsed, vdb))
self.repos_raw.pop(None, None)
if profile.provides_repo is not None:
self.repos_raw['package.provided'] = profile.provides_repo
vdb.append(profile.provides_repo)
self.profile = profile
pkg_maskers, pkg_unmaskers, pkg_keywords, pkg_licenses = [], [], [], []
pkg_use, self.bashrcs = [], []
self.ebuild_hook_dir = settings.pop("ebuild_hook_dir", None)
for key, val, action in (
("package.mask", pkg_maskers, parse_match),
("package.unmask", pkg_unmaskers, parse_match),
("package.keywords", pkg_keywords, package_keywords_splitter),
("package.accept_keywords", pkg_keywords, package_keywords_splitter),
("package.license", pkg_licenses, package_keywords_splitter),
("package.use", pkg_use, package_keywords_splitter),
("package.env", self.bashrcs, package_env_splitter),
):
for fp in settings.pop(key, ()):
try:
if key == "package.env":
base = self.ebuild_hook_dir
if base is None:
base = os.path.dirname(fp)
action = partial(action, base)
for fs_obj in iter_scan(fp, follow_symlinks=True):
if not fs_obj.is_reg or '/.' in fs_obj.location:
continue
val.extend(action(x) for x in iter_read_bash(fs_obj.location))
except EnvironmentError as e:
if e.errno == errno.ENOENT:
raise MissingFile(fp, key)
raise_from(Failure("failed reading '%s': %s" % (fp, e)))
except ValueError as e:
raise_from(Failure("failed reading '%s': %s" % (fp, e)))
self.name = name
settings.setdefault("PKGCORE_DOMAIN", name)
for x in incrementals:
if isinstance(settings.get(x), basestring):
settings[x] = tuple(settings[x].split())
# roughly... all incremental stacks should be interpreted left -> right
# as such we start with the profile settings, and append ours onto it.
for k, v in profile.default_env.iteritems():
if k not in settings:
settings[k] = v
continue
if k in incrementals:
settings[k] = v + tuple(settings[k])
# next we finalize incrementals.
for incremental in incrementals:
# Skip USE/ACCEPT_LICENSE for the time being; hack; we need the
# negations currently so that pkg iuse induced enablings can be
# disabled by negations. For example, think of the profile doing
# USE=-cdr for brasero w/ IUSE=+cdr. Similarly, ACCEPT_LICENSE is
# skipped because negations are required for license filtering.
if incremental not in settings or incremental in ("USE", "ACCEPT_LICENSE"):
continue
s = set()
incremental_expansion(
s, settings[incremental],
'While expanding %s ' % (incremental,))
settings[incremental] = tuple(s)
# use is collapsed; now stack use_expand.
use = settings['USE'] = set(optimize_incrementals(
list(settings.get('USE', ())) + os.environ.get('USE', '').split()))
self._extend_use_for_features(use, settings.get("FEATURES", ()))
for u in profile.use_expand:
v = settings.get(u)
if v is None:
continue
u2 = u.lower()+"_"
use.update(u2 + x for x in v.split())
if 'ACCEPT_KEYWORDS' not in settings:
raise Failure("No ACCEPT_KEYWORDS setting detected from profile, "
"or user config")
s = set()
default_keywords = []
incremental_expansion(
s, settings['ACCEPT_KEYWORDS'],
'while expanding ACCEPT_KEYWORDS')
default_keywords.extend(s)
settings['ACCEPT_KEYWORDS'] = set(default_keywords)
self.use = use
if "ARCH" not in settings:
raise Failure(
"No ARCH setting detected from profile, or user config")
self.arch = self.stable_arch = settings["ARCH"]
self.unstable_arch = "~%s" % self.arch
# ~amd64 -> [amd64, ~amd64]
for x in default_keywords[:]:
if x.startswith("~"):
default_keywords.append(x.lstrip("~"))
default_keywords = unstable_unique(default_keywords + [self.arch])
accept_keywords = pkg_keywords + list(profile.accept_keywords)
vfilters = [self.make_keywords_filter(
self.arch, default_keywords, accept_keywords, profile.keywords,
incremental="package.keywords" in incrementals)]
del default_keywords, accept_keywords
# we can finally close that fricking
# "DISALLOW NON FOSS LICENSES" bug via this >:)
master_license = []
master_license.extend(settings.get('ACCEPT_LICENSE', ()))
if master_license or pkg_licenses:
vfilters.append(self.make_license_filter(master_license, pkg_licenses))
del master_license
# if it's made it this far...
self.root = settings["ROOT"] = root
self.prefix = prefix
self.settings = ProtectedDict(settings)
for data in self.settings.get('bashrc', ()):
source = local_source(data)
# this is currently local-only so a path check is ok
# TODO make this more general
if not os.path.exists(source.path):
raise Failure(
'user-specified bashrc %r does not exist' % (data,))
self.bashrcs.append((packages.AlwaysTrue, source))
# stack use stuff first, then profile.
self.enabled_use = ChunkedDataDict()
self.enabled_use.add_bare_global(*split_negations(self.use))
self.enabled_use.merge(profile.pkg_use)
self.enabled_use.update_from_stream(
chunked_data(k, *split_negations(v)) for k, v in pkg_use)
for attr in ('', 'stable_'):
c = ChunkedDataDict()
c.merge(getattr(profile, attr + 'forced_use'))
c.add_bare_global((), (self.arch,))
setattr(self, attr + 'forced_use', c)
c = ChunkedDataDict()
c.merge(getattr(profile, attr + 'masked_use'))
setattr(self, attr + 'disabled_use', c)
self.repos = []
self.vdb = []
self.repos_configured = {}
self.repos_configured_filtered = {}
rev_names = {repo: name for name, repo in self.repos_raw.iteritems()}
profile_masks = profile._incremental_masks()
profile_unmasks = profile._incremental_unmasks()
repo_masks = {r.repo_id: r._visibility_limiters() for r in repositories}
for l, repos, filtered in ((self.repos, repositories, True),
(self.vdb, vdb, False)):
for repo in repos:
if not repo.configured:
pargs = [repo]
try:
for x in repo.configurables:
if x == "domain":
pargs.append(self)
elif x == "settings":
pargs.append(settings)
elif x == "profile":
pargs.append(profile)
else:
pargs.append(getattr(self, x))
except AttributeError as ae:
raise_from(Failure("failed configuring repo '%s': "
"configurable missing: %s" % (repo, ae)))
wrapped_repo = repo.configure(*pargs)
else:
wrapped_repo = repo
key = rev_names.get(repo)
self.repos_configured[key] = wrapped_repo
if filtered:
config = getattr(repo, 'config', None)
masters = getattr(config, 'masters', ())
if masters is None:
# tough cookies. If a user has an overlay, no masters
# defined, we're not applying the portdir masks.
# we do this both since that's annoying, and since
# frankly there isn't any good course of action.
masters = ()
global_masks = [repo_masks.get(master, [(), ()]) for master in masters]
global_masks.append(repo_masks[repo.repo_id])
global_masks.extend(profile_masks)
masks = set()
for neg, pos in global_masks:
masks.difference_update(neg)
masks.update(pos)
masks.update(pkg_maskers)
unmasks = set(chain(pkg_unmaskers, *profile_unmasks))
filtered = generate_filter(masks, unmasks, *vfilters)
if filtered:
wrapped_repo = visibility.filterTree(wrapped_repo, filtered, True)
self.repos_configured_filtered[key] = wrapped_repo
l.append(wrapped_repo)
if profile.virtuals:
l = [x for x in (getattr(v, 'old_style_virtuals', None)
for v in self.vdb) if x is not None]
profile_repo = profile.make_virtuals_repo(
multiplex.tree(*repositories), *l)
self.repos_raw["profile virtuals"] = profile_repo
self.repos_configured_filtered["profile virtuals"] = profile_repo
self.repos_configured["profile virtuals"] = profile_repo
self.repos = [profile_repo] + self.repos
self.use_expand_re = re.compile(
"^(?:[+-])?(%s)_(.*)$" %
"|".join(x.lower() for x in sorted(profile.use_expand, reverse=True)))
