def test_sorter(self):
get_inst = self._py3k_protection
assert get_inst(range(100, 0, -1), sorted) == tuple(range(1, 101))
c = caching_iter(range(100, 0, -1), sorted)
assert c
assert tuple(c) == tuple(range(1, 101))
c = caching_iter(range(50, 0, -1), sorted)
assert c[10] == 11
assert tuple(range(1, 51)) == tuple(c)
def test_sorter(self):
assert tuple(caching_iter(range(100, 0, -1),
sorted)) == tuple(range(1, 101))
c = caching_iter(range(100, 0, -1), sorted)
assert c
assert tuple(iter(c)) == tuple(range(1, 101))
c = caching_iter(range(50, 0, -1), sorted)
assert c[10] == 11
assert tuple(iter(c)) == tuple(range(1, 51))
def test_sorter(self):
get_inst = self._py3k_protection
assert get_inst(range(100, 0, -1), sorted) == tuple(range(1, 101))
c = caching_iter(range(100, 0, -1), sorted)
assert c
assert tuple(c) == tuple(range(1, 101))
c = caching_iter(range(50, 0, -1), sorted)
assert c[10] == 11
assert tuple(range(1, 51)) == tuple(c)
def test_sorter(self):
get_inst = self._py3k_protection
self.assertEqual(get_inst(xrange(100, 0, -1), sorted), tuple(xrange(1, 101)))
c = caching_iter(xrange(100, 0, -1), sorted)
self.assertTrue(c)
if compatibility.is_py3k:
c = tuple(c)
self.assertEqual(c, tuple(xrange(1, 101)))
c = caching_iter(xrange(50, 0, -1), sorted)
self.assertEqual(c[10], 11)
if compatibility.is_py3k:
c = tuple(c)
self.assertEqual(tuple(xrange(1, 51)), c)
def test_sorter(self):
get_inst = self._py3k_protection
self.assertEqual(
get_inst(xrange(100, 0, -1), sorted), tuple(xrange(1, 101)))
c = caching_iter(xrange(100, 0, -1), sorted)
self.assertTrue(c)
if compatibility.is_py3k:
c = tuple(c)
self.assertEqual(c, tuple(xrange(1, 101)))
c = caching_iter(xrange(50, 0, -1), sorted)
self.assertEqual(c[10], 11)
if compatibility.is_py3k:
c = tuple(c)
self.assertEqual(tuple(xrange(1, 51)), c)
def test_getitem(self):
c = caching_iter(range(20))
assert c[-1] == 19
with pytest.raises(IndexError):
operator.getitem(c, -21)
with pytest.raises(IndexError):
operator.getitem(c, 21)
def test_edgecase(self):
c = caching_iter(xrange(5))
self.assertEqual(c[0], 0)
# do an off by one access- this actually has broke before
self.assertEqual(c[2], 2)
self.assertEqual(c[1], 1)
self.assertEqual(list(c), list(xrange(5)))
def match(self, restrict):
v = self.__cache__.get(restrict)
if v is None:
v = self.__cache__[restrict] = \
caching_iter(
self.__db__.itermatch(restrict, sorter=self.__strategy__))
return v
def find_vulnerable_repo_pkgs(glsa_src, repo, grouped=False, arch=None):
"""generator yielding GLSA restrictions, and vulnerable pkgs from a repo.
:param glsa_src: GLSA pkgset to pull vulnerabilities from
:param repo: repo to scan for vulnerable packages
:param grouped: if grouped, combine glsa restrictions into one restriction
(thus yielding a pkg only once)
:param arch: arch to scan for, x86 for example
"""
if grouped:
i = glsa_src.pkg_grouped_iter()
else:
i = iter(glsa_src)
if arch is None:
wrapper = lambda p: p
else:
if isinstance(arch, str):
arch = (arch,)
else:
arch = tuple(arch)
wrapper = lambda p: mutated.MutatedPkg(p, {"keywords": arch})
for restrict in i:
matches = caching_iter(wrapper(x)
for x in repo.itermatch(restrict,
sorter=sorted))
if matches:
yield restrict, matches
def test_getitem(self):
c = caching_iter(range(20))
assert c[-1] == 19
with pytest.raises(IndexError):
operator.getitem(c, -21)
with pytest.raises(IndexError):
operator.getitem(c, 21)
def test_edgecase(self):
c = caching_iter(range(5))
assert c[0] == 0
# do an off by one access- this actually has broke before
assert c[2] == 2
assert c[1] == 1
assert list(c) == list(range(5))
def find_vulnerable_repo_pkgs(glsa_src, repo, grouped=False, arch=None):
"""generator yielding GLSA restrictions, and vulnerable pkgs from a repo.
:param glsa_src: GLSA pkgset to pull vulnerabilities from
:param repo: repo to scan for vulnerable packages
:param grouped: if grouped, combine glsa restrictions into one restriction
(thus yielding a pkg only once)
:param arch: arch to scan for, x86 for example
"""
if grouped:
i = glsa_src.pkg_grouped_iter()
else:
i = iter(glsa_src)
if arch is None:
wrapper = lambda p: p
else:
if isinstance(arch, basestring):
arch = (arch, )
else:
arch = tuple(arch)
wrapper = lambda p: mutated.MutatedPkg(p, {"keywords": arch})
for restrict in i:
matches = caching_iter(
wrapper(x) for x in repo.itermatch(restrict, sorter=sorted))
if matches:
yield restrict, matches
def test_edgecase(self):
c = caching_iter(range(5))
assert c[0] == 0
# do an off by one access- this actually has broke before
assert c[2] == 2
assert c[1] == 1
assert list(c) == list(range(5))
def test_cmp(self):
assert tuple(caching_iter(range(100))) == tuple(range(100))
assert tuple(caching_iter(range(90))) != tuple(range(100))
assert tuple(caching_iter(range(100))) > tuple(range(90))
assert not tuple(caching_iter(range(90))) > tuple(range(100))
assert tuple(caching_iter(range(100))) >= tuple(range(100))
assert tuple(caching_iter(range(90))) < tuple(range(100))
assert not tuple(caching_iter(range(100))) < tuple(range(90))
assert tuple(caching_iter(range(90))) <= tuple(range(100))
def feed(self, pkg, reporter):
# query_cache gets caching_iter partial repo searches shoved into it-
# reason is simple, it's likely that versions of this pkg probably
# use similar deps- so we're forcing those packages that were
# accessed for atom matching to remain in memory.
# end result is less going to disk
if vcs_eclasses.intersection(pkg.inherited):
# vcs ebuild that better not be visible
self.check_visibility_vcs(pkg, reporter)
suppressed_depsets = []
for attr, depset in (("depends", pkg.depends),
("rdepends", pkg.rdepends), ("post_rdepends",
pkg.post_rdepends)):
nonexistent = set()
try:
for orig_node in visit_atoms(pkg, depset):
node = strip_atom_use(orig_node)
if node not in self.query_cache:
if node in self.profiles.global_insoluble:
nonexistent.add(node)
# insert an empty tuple, so that tight loops further
# on don't have to use the slower get method
self.query_cache[node] = ()
else:
matches = caching_iter(
self.options.search_repo.itermatch(node))
if matches:
self.query_cache[node] = matches
if orig_node is not node:
self.query_cache[str(orig_node)] = matches
elif not node.blocks:
nonexistent.add(node)
self.query_cache[node] = ()
self.profiles.global_insoluble.add(node)
elif not self.query_cache[node]:
nonexistent.add(node)
except _BlockMemoryExhaustion as e:
reporter.add_report(UncheckableDep(pkg, attr))
suppressed_depsets.append(attr)
if nonexistent:
reporter.add_report(NonExistentDeps(pkg, attr, nonexistent))
del nonexistent
for attr, depset in (("depends", pkg.depends),
("rdepends", pkg.rdepends), ("post_rdepends",
pkg.post_rdepends)):
if attr in suppressed_depsets:
continue
for edepset, profiles in self.depset_cache.collapse_evaluate_depset(
pkg, attr, depset):
self.process_depset(pkg, attr, edepset, profiles, reporter)
def feed(self, pkg, reporter):
# query_cache gets caching_iter partial repo searches shoved into it-
# reason is simple, it's likely that versions of this pkg probably
# use similar deps- so we're forcing those packages that were
# accessed for atom matching to remain in memory.
# end result is less going to disk
if vcs_eclasses.intersection(pkg.inherited):
# vcs ebuild that better not be visible
self.check_visibility_vcs(pkg, reporter)
suppressed_depsets = []
for attr, depset in (("depends", pkg.depends),
("rdepends", pkg.rdepends),
("post_rdepends", pkg.post_rdepends)):
nonexistent = set()
try:
for orig_node in visit_atoms(pkg, depset):
node = strip_atom_use(orig_node)
if node not in self.query_cache:
if node in self.profiles.global_insoluble:
nonexistent.add(node)
# insert an empty tuple, so that tight loops further
# on don't have to use the slower get method
self.query_cache[node] = ()
else:
matches = caching_iter(
self.options.search_repo.itermatch(node))
if matches:
self.query_cache[node] = matches
if orig_node is not node:
self.query_cache[str(orig_node)] = matches
elif not node.blocks:
nonexistent.add(node)
self.query_cache[node] = ()
self.profiles.global_insoluble.add(node)
elif not self.query_cache[node]:
nonexistent.add(node)
except _BlockMemoryExhaustion as e:
reporter.add_report(UncheckableDep(pkg, attr))
suppressed_depsets.append(attr)
if nonexistent:
reporter.add_report(NonExistentDeps(pkg, attr, nonexistent))
del nonexistent
for attr, depset in (("depends", pkg.depends),
("rdepends", pkg.rdepends),
("post_rdepends", pkg.post_rdepends)):
if attr in suppressed_depsets:
continue
for edepset, profiles in self.depset_cache.collapse_evaluate_depset(pkg, attr, depset):
self.process_depset(pkg, attr, edepset, profiles, reporter)
def _viable(self, stack, mode, atom, dbs, drop_cycles, limit_to_vdb):
"""
internal function to discern if an atom is viable, returning
the choicepoint/matches iterator if viable.
:param stack: current stack
:type stack: :obj:`resolver_stack`
:param mode: type of dependency (depend/rdepend)
:type mode: str
:param atom: atom for the current package
:type atom: :obj:`pkgcore.ebuild.atom.atom`
:param dbs: db list to walk
:param drop_cycles: boolean controlling whether to drop dep cycles
:param limit_to_vdb: boolean controlling considering pkgs only from the vdb
:return: 3 possible; None (not viable), True (presolved),
:obj:`caching_iter` (not solved, but viable), :obj:`choice_point`
"""
choices = ret = None
if atom in self.insoluble:
ret = ((False, "globally insoluble"),{})
matches = ()
else:
matches = self.state.match_atom(atom)
if matches:
ret = ((True,), {"pre_solved":True})
else:
# not in the plan thus far.
matches = caching_iter(dbs.itermatch(atom))
if matches:
choices = choice_point(atom, matches)
# ignore what dropped out, at this juncture we don't care.
choices.reduce_atoms(self.insoluble)
if not choices:
# and was intractable because it has a hard dep on an
# unsolvable atom.
ret = ((False, "pruning of insoluble deps "
"left no choices"), {})
else:
ret = ((False, "no matches"), {})
if choices is None:
choices = choice_point(atom, matches)
stack.add_frame(mode, atom, choices, dbs,
self.state.current_state, drop_cycles, vdb_limited=limit_to_vdb)
if not limit_to_vdb and not matches:
self.insoluble.add(atom)
if ret is not None:
self.notify_viable(stack, atom, *ret[0], **ret[1])
if ret[0][0] == True:
state.add_backref_op(choices, choices.current_pkg).apply(self.state)
return True
return None
return choices, matches
def _viable(self, stack, mode, atom, dbs, drop_cycles, limit_to_vdb):
"""
internal function to discern if an atom is viable, returning
the choicepoint/matches iterator if viable.
:param stack: current stack
:type stack: :obj:`resolver_stack`
:param mode: type of dependency (depends/rdepends)
:type mode: str
:param atom: atom for the current package
:type atom: :obj:`pkgcore.ebuild.atom.atom`
:param dbs: db list to walk
:param drop_cycles: boolean controlling whether to drop dep cycles
:param limit_to_vdb: boolean controlling considering pkgs only from the vdb
:return: 3 possible; None (not viable), True (presolved),
:obj:`caching_iter` (not solved, but viable), :obj:`choice_point`
"""
choices = ret = None
if atom in self.insoluble:
ret = ((False, "globally insoluble"),{})
matches = ()
else:
matches = self.state.match_atom(atom)
if matches:
ret = ((True,), {"pre_solved":True})
else:
# not in the plan thus far.
matches = caching_iter(dbs.itermatch(atom))
if matches:
choices = choice_point(atom, matches)
# ignore what dropped out, at this juncture we don't care.
choices.reduce_atoms(self.insoluble)
if not choices:
# and was intractable because it has a hard dep on an
# unsolvable atom.
ret = ((False, "pruning of insoluble deps "
"left no choices"), {})
else:
ret = ((False, "no matches"), {})
if choices is None:
choices = choice_point(atom, matches)
stack.add_frame(mode, atom, choices, dbs,
self.state.current_state, drop_cycles, vdb_limited=limit_to_vdb)
if not limit_to_vdb and not matches:
self.insoluble.add(atom)
if ret is not None:
self.notify_viable(stack, atom, *ret[0], **ret[1])
if ret[0][0] == True:
state.add_backref_op(choices, choices.current_pkg).apply(self.state)
return True
return None
return choices, matches
def test_iter_consumption(self):
i = iter(range(100))
c = caching_iter(i)
i2 = iter(c)
for _ in range(20):
next(i2)
assert next(i) == 20
# note we consumed one ourselves
assert c[20] == 21
list(c)
pytest.raises(StopIteration, i.__next__)
assert list(c) == list(range(20)) + list(range(21, 100))
def test_iter_consumption(self):
i = iter(xrange(100))
c = caching_iter(i)
i2 = iter(c)
for _ in xrange(20):
i2.next()
self.assertEqual(i.next(), 20)
# note we consumed one ourselves
self.assertEqual(c[20], 21)
list(c)
self.assertRaises(StopIteration, i.next)
self.assertEqual(list(c), range(20) + range(21, 100))
def test_iter_consumption(self):
i = iter(range(100))
c = caching_iter(i)
i2 = iter(c)
for _ in range(20):
next(i2)
assert next(i) == 20
# note we consumed one ourselves
assert c[20] == 21
list(c)
pytest.raises(StopIteration, i.__next__)
assert list(c) == list(range(20)) + list(range(21, 100))
def _viable(self, stack, mode, atom, dbs, drop_cycles, limit_to_vdb):
"""
internal function to discern if an atom is viable, returning
the choicepoint/matches iter if viable.
:return: 3 possible; None (not viable), True (presolved),
:obj:`caching_iter` (not solved, but viable), :obj:`choice_point`
"""
choices = ret = None
if atom in self.insoluble:
ret = ((False, "globally insoluble"),{})
matches = ()
else:
matches = self.state.match_atom(atom)
if matches:
ret = ((True,), {"pre_solved":True})
else:
# not in the plan thus far.
matches = caching_iter(dbs.itermatch(atom))
if matches:
choices = choice_point(atom, matches)
# ignore what dropped out, at this juncture we don't care.
choices.reduce_atoms(self.insoluble)
if not choices:
# and was intractable because it has a hard dep on an
# unsolvable atom.
ret = ((False, "pruning of insoluble deps "
"left no choices"), {})
else:
ret = ((False, "no matches"), {})
if choices is None:
choices = choice_point(atom, matches)
stack.add_frame(mode, atom, choices, dbs,
self.state.current_state, drop_cycles, vdb_limited=limit_to_vdb)
if not limit_to_vdb and not matches:
self.insoluble.add(atom)
if ret is not None:
self.notify_viable(stack, atom, *ret[0], **ret[1])
if ret[0][0] == True:
state.add_backref_op(choices, choices.current_pkg).apply(self.state)
return True
return None
return choices, matches
def _py3k_protection(*args, **kwds):
return tuple(caching_iter(*args, **kwds))
def _py3k_protection(*args, **kwds):
ci = caching_iter(*args, **kwds)
if compatibility.is_py3k:
ci = tuple(ci)
return ci
def feed(self, pkg):
super().feed(pkg)
# query_cache gets caching_iter partial repo searches shoved into it-
# reason is simple, it's likely that versions of this pkg probably
# use similar deps- so we're forcing those packages that were
# accessed for atom matching to remain in memory.
# end result is less going to disk
if pkg.live:
# vcs ebuild that better not be visible
yield from self.check_visibility_vcs(pkg)
suppressed_depsets = []
for attr in (x.lower() for x in pkg.eapi.dep_keys):
nonexistent = set()
try:
for orig_node in visit_atoms(pkg, getattr(pkg, attr)):
node = orig_node.no_usedeps
if node not in self.query_cache:
if node in self.profiles.global_insoluble:
nonexistent.add(node)
# insert an empty tuple, so that tight loops further
# on don't have to use the slower get method
self.query_cache[node] = ()
else:
matches = caching_iter(
self.options.search_repo.itermatch(node))
if matches:
self.query_cache[node] = matches
if orig_node is not node:
self.query_cache[str(orig_node)] = matches
elif not node.blocks:
nonexistent.add(node)
self.query_cache[node] = ()
self.profiles.global_insoluble.add(node)
elif not self.query_cache[node]:
nonexistent.add(node)
except _BlockMemoryExhaustion as e:
yield UncheckableDep(attr, pkg=pkg)
suppressed_depsets.append(attr)
if nonexistent:
nonexistent = map(str, sorted(nonexistent))
yield NonexistentDeps(attr.upper(), nonexistent, pkg=pkg)
del nonexistent
for attr in (x.lower() for x in pkg.eapi.dep_keys):
if attr in suppressed_depsets:
continue
depset = getattr(pkg, attr)
profile_failures = defaultdict(lambda: defaultdict(set))
for edepset, profiles in self.collapse_evaluate_depset(
pkg, attr, depset):
for profile, failures in self.process_depset(
pkg, attr, depset, edepset, profiles):
failures = tuple(map(str, stable_unique(failures)))
profile_failures[failures][profile.status].add(profile)
if profile_failures:
if self.options.verbosity > 0:
# report all failures across all profiles in verbose mode
for failures, profiles in profile_failures.items():
for profile_status, cls in self.report_cls_map.items():
for profile in sorted(
profiles.get(profile_status, ()),
key=attrgetter('key', 'name')):
yield cls(attr,
profile.key,
profile.name,
failures,
profile_status,
profile.deprecated,
pkg=pkg)
else:
# only report one failure per depset per profile type in regular mode
for failures, profiles in profile_failures.items():
for profile_status, cls in self.report_cls_map.items():
status_profiles = sorted(
profiles.get(profile_status, ()),
key=attrgetter('key', 'name'))
if status_profiles:
profile = status_profiles[0]
yield cls(attr,
profile.key,
profile.name,
failures,
profile_status,
profile.deprecated,
len(status_profiles),
pkg=pkg)
def test_getitem(self):
c = caching_iter(xrange(20))
self.assertEqual(19, c[-1])
self.assertRaises(IndexError, operator.getitem, c, -21)
self.assertRaises(IndexError, operator.getitem, c, 21)
def test_hash(self):
self.assertEqual(hash(caching_iter(xrange(100))),
hash(tuple(range(100))))
def test_setitem(self):
self.assertRaises(
TypeError, operator.setitem, caching_iter(xrange(10)), 3, 4)
def test_full_consumption(self):
i = iter(range(100))
c = caching_iter(i)
assert list(c) == list(range(100))
# do it twice, to verify it returns properly
assert list(c) == list(range(100))
def test_hash(self):
assert hash(caching_iter(range(100))) == hash(tuple(range(100)))
def test_setitem(self):
with pytest.raises(TypeError):
operator.setitem(caching_iter(range(10)), 3, 4)
def test_full_consumption(self):
i = iter(xrange(100))
c = caching_iter(i)
self.assertEqual(list(c), range(100))
# do it twice, to verify it returns properly
self.assertEqual(list(c), range(100))
def test_nonzero(self):
c = caching_iter(xrange(100))
self.assertEqual(bool(c), True)
# repeat to check if it works when cached.
self.assertEqual(bool(c), True)
self.assertEqual(bool(caching_iter(iter([]))), False)
def test_str(self):
# Just make sure this works at all.
assert str(caching_iter(range(10)))
def test_setitem(self):
with pytest.raises(TypeError):
operator.setitem(caching_iter(range(10)), 3, 4)
def test_str(self):
# Just make sure this works at all.
self.assertTrue(str(caching_iter(xrange(10))))
def test_init(self):
self.assertEqual(caching_iter(list(xrange(100)))[0], 0)
def test_init(self):
assert caching_iter(list(range(100)))[0] == 0
def test_full_consumption(self):
i = iter(range(100))
c = caching_iter(i)
assert list(c) == list(range(100))
# do it twice, to verify it returns properly
assert list(c) == list(range(100))
def test_str(self):
# Just make sure this works at all.
assert str(caching_iter(range(10)))
def test_len(self):
assert 100 == len(caching_iter(range(100)))
def test_init(self):
assert caching_iter(list(range(100)))[0] == 0
def test_hash(self):
assert hash(caching_iter(range(100))) == hash(tuple(range(100)))
def test_len(self):
assert 100 == len(caching_iter(range(100)))
def test_bool(self):
c = caching_iter(range(100))
assert bool(c)
# repeat to check if it works when cached.
assert bool(c)
assert not bool(caching_iter(iter([])))
def test_bool(self):
c = caching_iter(range(100))
assert bool(c) == True
# repeat to check if it works when cached.
assert bool(c) == True
assert bool(caching_iter(iter([]))) == False
def test_len(self):
self.assertEqual(100, len(caching_iter(xrange(100))))
