def _identify_candidates(self, restrict, sorter):
# full expansion
if not isinstance(restrict, boolean.base) or isinstance(restrict, atom):
return self._fast_identify_candidates(restrict, sorter)
dsolutions = [
([c.restriction
for c in collect_package_restrictions(x, ("category",))],
[p.restriction
for p in collect_package_restrictions(x, ("package",))])
for x in restrict.iter_dnf_solutions(True)]
# see if any solution state isn't dependent on cat/pkg in anyway.
# if so, search whole search space.
for x in dsolutions:
if not x[0] and not x[1]:
if sorter is iter:
return self.versions
return (
(c, p)
for c in sorter(self.categories)
for p in sorter(self.packages.get(c, ())))
# simple cases first.
# if one specifies categories, and one doesn't
cat_specified = bool(dsolutions[0][0])
pkg_specified = bool(dsolutions[0][1])
pgetter = self.packages.get
if any(True for x in dsolutions[1:] if bool(x[0]) != cat_specified):
if any(True for x in dsolutions[1:] if bool(x[1]) != pkg_specified):
# merde. so we've got a mix- some specify cats, some
# don't, some specify pkgs, some don't.
# this may be optimizable
return self.versions
# ok. so... one doesn't specify a category, but they all
# specify packages (or don't)
pr = values.OrRestriction(
*tuple(iflatten_instance(
(x[1] for x in dsolutions if x[1]), values.base)))
return (
(c, p)
for c in sorter(self.categories)
for p in sorter(pgetter(c, [])) if pr.match(p))
elif any(True for x in dsolutions[1:] if bool(x[1]) != pkg_specified):
# one (or more) don't specify pkgs, but they all specify cats.
cr = values.OrRestriction(
*tuple(iflatten_instance(
(x[0] for x in dsolutions), values.base)))
cats_iter = (c for c in sorter(self.categories) if cr.match(c))
return (
(c, p)
for c in cats_iter for p in sorter(pgetter(c, [])))
return self._fast_identify_candidates(restrict, sorter)
def _identify_candidates(self, restrict, sorter):
# full expansion
if not isinstance(restrict, boolean.base) or isinstance(restrict, atom):
return self._fast_identify_candidates(restrict, sorter)
dsolutions = [
([c.restriction
for c in collect_package_restrictions(x, ("category",))],
[p.restriction
for p in collect_package_restrictions(x, ("package",))])
for x in restrict.iter_dnf_solutions(True)]
# see if any solution state isn't dependent on cat/pkg in anyway.
# if so, search whole search space.
for x in dsolutions:
if not x[0] and not x[1]:
if sorter is iter:
return self.versions
return (
(c, p)
for c in sorter(self.categories)
for p in sorter(self.packages.get(c, ())))
# simple cases first.
# if one specifies categories, and one doesn't
cat_specified = bool(dsolutions[0][0])
pkg_specified = bool(dsolutions[0][1])
pgetter = self.packages.get
if any(True for x in dsolutions[1:] if bool(x[0]) != cat_specified):
if any(True for x in dsolutions[1:] if bool(x[1]) != pkg_specified):
# merde. so we've got a mix- some specify cats, some
# don't, some specify pkgs, some don't.
# this may be optimizable
return self.versions
# ok. so... one doesn't specify a category, but they all
# specify packages (or don't)
pr = values.OrRestriction(
*tuple(iflatten_instance(
(x[1] for x in dsolutions if x[1]), values.base)))
return (
(c, p)
for c in sorter(self.categories)
for p in sorter(pgetter(c, [])) if pr.match(p))
elif any(True for x in dsolutions[1:] if bool(x[1]) != pkg_specified):
# one (or more) don't specify pkgs, but they all specify cats.
cr = values.OrRestriction(
*tuple(iflatten_instance(
(x[0] for x in dsolutions), values.base)))
cats_iter = (c for c in sorter(self.categories) if cr.match(c))
return (
(c, p)
for c in cats_iter for p in sorter(pgetter(c, [])))
return self._fast_identify_candidates(restrict, sorter)
def test_collect_specific(self):
prs = {}
for x in ("category", "package", "version", "iuse"):
prs[x] = packages.PackageRestriction(x, values.AlwaysTrue)
r = packages.AndRestriction(packages.OrRestriction(*prs.values()),
packages.AlwaysTrue)
for k, v in prs.iteritems():
self.assertEqual(
list(util.collect_package_restrictions(r, attrs=[k])), [v])
r = packages.AndRestriction(packages.OrRestriction(*prs.values()),
*prs.values())
for k, v in prs.iteritems():
self.assertEqual(
list(util.collect_package_restrictions(r, attrs=[k])), [v] * 2)
def test_collect_all(self):
prs = [packages.PackageRestriction("category", values.AlwaysTrue)] * 10
self.assertEqual(
list(util.collect_package_restrictions(packages.AndRestriction(
packages.OrRestriction(), packages.AndRestriction(),
*prs))),
prs)
def test_collect_all(self):
prs = [packages.PackageRestriction("category", values.AlwaysTrue)] * 10
self.assertEqual(
list(util.collect_package_restrictions(packages.AndRestriction(
packages.OrRestriction(), packages.AndRestriction(),
*prs))),
prs)
def _restrict_to_scope(restrict):
"""Determine a given restriction's scope level."""
for scope, attrs in ((base.version_scope, ['fullver', 'version', 'rev']),
(base.package_scope,
['package']), (base.category_scope, ['category'])):
if any(collect_package_restrictions(restrict, attrs)):
return scope
return base.repo_scope
def test_collect_specific(self):
prs = {}
for x in ("category", "package", "version", "iuse"):
prs[x] = packages.PackageRestriction(x, values.AlwaysTrue)
r = packages.AndRestriction(
packages.OrRestriction(*prs.values()), packages.AlwaysTrue)
for k, v in prs.iteritems():
self.assertEqual(
list(util.collect_package_restrictions(r, attrs=[k])),
[v])
r = packages.AndRestriction(packages.OrRestriction(
*prs.values()), *prs.values())
for k, v in prs.iteritems():
self.assertEqual(
list(util.collect_package_restrictions(r, attrs=[k])),
[v] * 2)
def parse_match(text):
"""generate appropriate restriction for text
Parsing basically breaks it down into chunks split by /, with each
chunk allowing for prefix/postfix globbing- note that a postfixed
glob on package token is treated as package attribute matching,
not as necessarily a version match.
If only one chunk is found, it's treated as a package chunk.
Finally, it supports a nonstandard variation of atom syntax where
the category can be dropped.
Examples:
- `*`: match all
- `dev-*/*`: category must start with 'dev-'
- `dev-*`: package must start with 'dev-'
- `*-apps/portage*`: category must end in '-apps', package must start with
'portage'
- `>=portage-2.1`: atom syntax, package 'portage', version greater then or
equal to '2.1'
:param text: string to attempt to parse
:type text: string
:return: :obj:`pkgcore.restrictions.packages` derivative
"""
# Ensure the text var is a string if we're under py3k.
if not is_py3k:
text = text.encode('ascii')
orig_text = text = text.strip()
if "!" in text:
raise ParseError(
"!, or any form of blockers make no sense in this usage: %s" % (
text,))
tsplit = text.rsplit("/", 1)
if len(tsplit) == 1:
ops, text = collect_ops(text)
if not ops:
if "*" in text:
r = convert_glob(text)
if r is None:
return packages.AlwaysTrue
return packages.PackageRestriction("package", r)
elif text.startswith("*"):
raise ParseError(
"cannot do prefix glob matches with version ops: %s" % (
orig_text,))
# ok... fake category. whee.
try:
r = list(util.collect_package_restrictions(
atom.atom("%scategory/%s" % (ops, text)).restrictions,
attrs=("category",), invert=True))
except errors.MalformedAtom, e:
raise_from(ParseError(str(e)))
if len(r) == 1:
return r[0]
return packages.AndRestriction(*r)
def _fast_identify_candidates(self, restrict, sorter):
pkg_restrict = set()
cat_restrict = set()
cat_exact = set()
pkg_exact = set()
for x in collect_package_restrictions(restrict, ("category", "package")):
if x.attr == "category":
cat_restrict.add(x.restriction)
elif x.attr == "package":
pkg_restrict.add(x.restriction)
for e, s in ((pkg_exact, pkg_restrict), (cat_exact, cat_restrict)):
l = [x for x in s if isinstance(x, values.StrExactMatch) and not x.negate]
s.difference_update(l)
e.update(x.exact for x in l)
del l
if restrict.negate:
cat_exact = pkg_exact = ()
if cat_exact:
if not cat_restrict and len(cat_exact) == 1:
# Cannot use pop here, cat_exact is reused below.
c = iter(cat_exact).next()
if not pkg_restrict and len(pkg_exact) == 1:
cp = (c, pkg_exact.pop())
if cp in self.versions:
return [cp]
return []
cats_iter = [c]
else:
cat_restrict.add(values.ContainmentMatch(*cat_exact))
cats_iter = sorter(self._cat_filter(cat_restrict))
elif cat_restrict:
cats_iter = self._cat_filter(cat_restrict, negate=restrict.negate)
else:
cats_iter = sorter(self.categories)
if pkg_exact:
if not pkg_restrict:
if sorter is iter:
pkg_exact = tuple(pkg_exact)
else:
pkg_exact = sorter(pkg_exact)
return ((c, p) for c in cats_iter for p in pkg_exact)
else:
pkg_restrict.add(values.ContainmentMatch(*pkg_exact))
if pkg_restrict:
return self._package_filter(cats_iter, pkg_restrict, negate=restrict.negate)
elif not cat_restrict:
if sorter is iter and not cat_exact:
return self.versions
else:
return ((c, p) for c in cats_iter for p in sorter(self.packages.get(c, ())))
return ((c, p) for c in cats_iter for p in sorter(self.packages.get(c, ())))
def __init__(self, repo, limiter):
self.repo = repo
self.limiter = limiter
for scope, attrs in ((base.version_scope, ['fullver', 'version', 'rev']),
(base.package_scope, ['package']),
(base.category_scope, ['category'])):
if any(util.collect_package_restrictions(limiter, attrs)):
self.scope = scope
return
self.scope = base.repository_scope
def __init__(self, repo, limiter):
self.repo = repo
self.limiter = limiter
for scope, attrs in ((base.version_scope, ['fullver', 'version', 'rev']),
(base.package_scope, ['package']),
(base.category_scope, ['category'])):
if any(util.collect_package_restrictions(limiter, attrs)):
self.scope = scope
return
self.scope = base.repository_scope
def feed(self, pkg):
for vuln in self.vulns.get(pkg.key, []):
if vuln.match(pkg):
arches = set()
for v in collect_package_restrictions(vuln, ['keywords']):
if isinstance(v.restriction, values.ContainmentMatch2):
arches.update(x.lstrip('~') for x in v.restriction.vals)
else:
raise Exception(
f'unexpected restriction sequence- {v.restriction} in {vuln}')
keys = {x.lstrip('~') for x in pkg.keywords if not x.startswith('-')}
if arches:
arches = sorted(arches.intersection(keys))
assert arches
else:
arches = sorted(keys)
yield VulnerablePackage(arches, str(vuln), pkg=pkg)
def parse_match(text):
"""generate appropriate restriction for text
Parsing basically breaks it down into chunks split by /, with each
chunk allowing for prefix/postfix globbing- note that a postfixed
glob on package token is treated as package attribute matching,
not as necessarily a version match.
If only one chunk is found, it's treated as a package chunk.
Finally, it supports a nonstandard variation of atom syntax where
the category can be dropped.
Examples:
- `*`: match all
- `dev-*/*`: category must start with 'dev-'
- `dev-*`: package must start with 'dev-'
- `*-apps/portage*`: category must end in '-apps', package must start with
'portage'
- `>=portage-2.1`: atom syntax, package 'portage', version greater then or
equal to '2.1'
- dev-qt/*:5: all Qt 5 libs
- boost:0/1.60: all packages named boost with a slot/subslot of 0/1.60.0
:param text: string to attempt to parse
:type text: string
:return: :obj:`pkgcore.restrictions.packages` derivative
"""
# Ensure the text var is a string if we're under py3k.
if not is_py3k:
text = text.encode('ascii')
orig_text = text = text.strip()
if "!" in text:
raise ParseError(
"'!' or any form of blockers make no sense in this usage: '%s'" %
(text, ))
restrictions = []
if '::' in text:
text, repo_id = text.rsplit('::', 1)
restrictions.append(restricts.RepositoryDep(repo_id))
if ':' in text:
text, slot = text.rsplit(':', 1)
slot, _sep, subslot = slot.partition('/')
if slot:
restrictions.append(restricts.SlotDep(slot))
if subslot:
restrictions.append(restricts.SubSlotDep(subslot))
tsplit = text.rsplit("/", 1)
if len(tsplit) == 1:
ops, text = collect_ops(text)
if not ops:
if "*" in text:
r = convert_glob(text)
if r is None:
restrictions.append(packages.AlwaysTrue)
else:
restrictions.append(
packages.PackageRestriction("package", r))
if len(restrictions) == 1:
return restrictions[0]
return packages.AndRestriction(*restrictions)
elif text.startswith("*"):
raise ParseError(
"cannot do prefix glob matches with version ops: %s" %
(orig_text, ))
# ok... fake category. whee.
try:
r = list(
collect_package_restrictions(atom.atom(
"%scategory/%s" % (ops, text)).restrictions,
attrs=("category", ),
invert=True))
except errors.MalformedAtom as e:
e.atom = orig_text
raise_from(ParseError(str(e)))
if not restrictions and len(r) == 1:
return r[0]
restrictions.extend(r)
return packages.AndRestriction(*restrictions)
elif text[0] in "=<>~" or "*" not in text:
try:
return atom.atom(orig_text)
except errors.MalformedAtom as e:
raise_from(ParseError(str(e)))
r = map(convert_glob, tsplit)
if not r[0] and not r[1]:
restrictions.append(packages.AlwaysTrue)
elif not r[0]:
restrictions.append(packages.PackageRestriction("package", r[1]))
elif not r[1]:
restrictions.append(packages.PackageRestriction("category", r[0]))
else:
restrictions.extend((
packages.PackageRestriction("category", r[0]),
packages.PackageRestriction("package", r[1]),
))
if len(restrictions) == 1:
return restrictions[0]
return packages.AndRestriction(*restrictions)
def parse_match(text):
"""generate appropriate restriction for text
Parsing basically breaks it down into chunks split by /, with each
chunk allowing for prefix/postfix globbing- note that a postfixed
glob on package token is treated as package attribute matching,
not as necessarily a version match.
If only one chunk is found, it's treated as a package chunk.
Finally, it supports a nonstandard variation of atom syntax where
the category can be dropped.
Examples:
- `*`: match all
- `dev-*/*`: category must start with 'dev-'
- `dev-*`: package must start with 'dev-'
- `*-apps/portage*`: category must end in '-apps', package must start with
'portage'
- `>=portage-2.1`: atom syntax, package 'portage', version greater then or
equal to '2.1'
- dev-qt/*:5: all Qt 5 libs
- boost:0/1.60: all packages named boost with a slot/subslot of 0/1.60.0
:param text: string to attempt to parse
:type text: string
:return: :obj:`pkgcore.restrictions.packages` derivative
"""
orig_text = text = text.strip()
if "!" in text:
raise ParseError(
f"'!' or any form of blockers make no sense in this usage: {text!r}")
restrictions = []
if '::' in text:
text, repo_id = text.rsplit('::', 1)
restrictions.append(restricts.RepositoryDep(repo_id))
if ':' in text:
text, slot = text.rsplit(':', 1)
slot, _sep, subslot = slot.partition('/')
if slot:
if '*' in slot:
r = convert_glob(slot)
if r is not None:
restrictions.append(packages.PackageRestriction("slot", r))
else:
restrictions.append(restricts.SlotDep(slot))
if subslot:
if '*' in subslot:
if r is not None:
restrictions.append(packages.PackageRestriction("subslot", r))
else:
restrictions.append(restricts.SubSlotDep(subslot))
tsplit = text.rsplit("/", 1)
if len(tsplit) == 1:
ops, text = collect_ops(text)
if not ops:
if "*" in text:
r = convert_glob(text)
if r is None:
restrictions.append(packages.AlwaysTrue)
else:
restrictions.append(packages.PackageRestriction("package", r))
if len(restrictions) == 1:
return restrictions[0]
return packages.AndRestriction(*restrictions)
elif text.startswith("*"):
raise ParseError(
f"cannot do prefix glob matches with version ops: {orig_text}")
# ok... fake category. whee.
try:
r = list(collect_package_restrictions(
atom.atom(f"{ops}category/{text}").restrictions,
attrs=("category",), invert=True))
except errors.MalformedAtom as e:
e.atom = orig_text
raise ParseError(str(e)) from e
if not restrictions and len(r) == 1:
return r[0]
restrictions.extend(r)
return packages.AndRestriction(*restrictions)
elif text[0] in atom.valid_ops or '*' not in text:
# possibly a valid atom object
try:
return atom.atom(orig_text)
except errors.MalformedAtom as e:
if '*' not in text:
raise ParseError(str(e)) from e
# support globbed targets with version restrictions
return packages.AndRestriction(*parse_globbed_version(text, orig_text))
r = list(map(convert_glob, tsplit))
if not r[0] and not r[1]:
restrictions.append(packages.AlwaysTrue)
elif not r[0]:
restrictions.append(packages.PackageRestriction("package", r[1]))
elif not r[1]:
restrictions.append(packages.PackageRestriction("category", r[0]))
else:
restrictions.extend((
packages.PackageRestriction("category", r[0]),
packages.PackageRestriction("package", r[1]),
))
if len(restrictions) == 1:
return restrictions[0]
return packages.AndRestriction(*restrictions)
def _fast_identify_candidates(self, restrict, sorter):
pkg_restrict = set()
cat_restrict = set()
cat_exact = set()
pkg_exact = set()
for x in collect_package_restrictions(restrict, (
"category",
"package",
)):
if x.attr == "category":
cat_restrict.add(x.restriction)
elif x.attr == "package":
pkg_restrict.add(x.restriction)
for e, s in ((pkg_exact, pkg_restrict), (cat_exact, cat_restrict)):
l = [
x for x in s
if isinstance(x, values.StrExactMatch) and not x.negate
]
s.difference_update(l)
e.update(x.exact for x in l)
del l
if restrict.negate:
cat_exact = pkg_exact = ()
if cat_exact:
if not cat_restrict and len(cat_exact) == 1:
# Cannot use pop here, cat_exact is reused below.
c = next(iter(cat_exact))
if not pkg_restrict and len(pkg_exact) == 1:
cp = (c, pkg_exact.pop())
if cp in self.versions:
return [cp]
return []
cats_iter = [c]
else:
cat_restrict.add(values.ContainmentMatch2(
frozenset(cat_exact)))
cats_iter = sorter(self._cat_filter(cat_restrict))
elif cat_restrict:
cats_iter = self._cat_filter(cat_restrict, negate=restrict.negate)
else:
cats_iter = sorter(self.categories)
if pkg_exact:
if not pkg_restrict:
if sorter is iter:
pkg_exact = tuple(pkg_exact)
else:
pkg_exact = sorter(pkg_exact)
return ((c, p) for c in cats_iter for p in pkg_exact)
else:
pkg_restrict.add(values.ContainmentMatch2(
frozenset(pkg_exact)))
if pkg_restrict:
return self._package_filter(cats_iter,
pkg_restrict,
negate=restrict.negate)
elif not cat_restrict:
if sorter is iter and not cat_exact:
return self.versions
else:
return ((c, p) for c in cats_iter
for p in sorter(self.packages.get(c, ())))
return ((c, p) for c in cats_iter
for p in sorter(self.packages.get(c, ())))