def test_sorted_results_4(self):
"""Test resolution when leaf nodes are shared."""
direct_dependencies = [("flask", "0.12.2", "https://pypi.org/simple")]
paths = [
[
("flask", "0.12.2", "https://pypi.org/simple"),
("jinja2", "2.7", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
],
[
("flask", "0.12.2", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
],
]
expected_result = [
{
("flask", "0.12.2", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
},
{
("flask", "0.12.2", "https://pypi.org/simple"),
("jinja2", "2.7", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
},
]
dependency_graph = DependencyGraphWalker(
direct_dependencies=direct_dependencies, paths=paths)
walk_result = list(set(item) for item in dependency_graph.walk())
assert walk_result == expected_result
def test_sorted_results_3(self):
"""Test sorting of results - multiple direct dependencies in different versions."""
direct_dependencies = [
("flask", "0.12.2", "https://pypi.org/simple"),
("flask", "0.12.3", "https://pypi.org/simple"),
]
paths = [
[
("flask", "0.12.2", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
],
[
("flask", "0.12.2", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.1", "https://pypi.org/simple"),
],
[
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
],
[
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.1", "https://pypi.org/simple"),
],
]
expected_result = [
{
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.1", "https://pypi.org/simple"),
},
{
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
},
{
("flask", "0.12.2", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.1", "https://pypi.org/simple"),
},
{
("flask", "0.12.2", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
},
]
dependency_graph = DependencyGraphWalker(
direct_dependencies=direct_dependencies, paths=paths)
walk_result = list(set(item) for item in dependency_graph.walk())
assert walk_result == expected_result
def test_sorted_results_2(self):
"""Test resolution of stacks based with two separable paths."""
direct_dependencies = [("flask", "0.12.3", "https://pypi.org/simple")]
paths = [
[
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
],
[
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.1", "https://pypi.org/simple"),
],
[
("flask", "0.12.3", "https://pypi.org/simple"),
("werkzeug", "0.14.1", "https://pypi.org/simple"),
],
[
("flask", "0.12.3", "https://pypi.org/simple"),
("werkzeug", "0.14.2", "https://pypi.org/simple"),
],
]
expected_result = [
{
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.1", "https://pypi.org/simple"),
("werkzeug", "0.14.2", "https://pypi.org/simple"),
},
{
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
("werkzeug", "0.14.2", "https://pypi.org/simple"),
},
{
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.1", "https://pypi.org/simple"),
("werkzeug", "0.14.1", "https://pypi.org/simple"),
},
{
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
("werkzeug", "0.14.1", "https://pypi.org/simple"),
},
]
dependency_graph = DependencyGraphWalker(
direct_dependencies=direct_dependencies, paths=paths)
walk_result = list(set(item) for item in dependency_graph.walk())
assert walk_result == expected_result
def test_one_direct(self):
"""Test a stack with just one direct dependency."""
direct_dependencies = [
("numpy", "1.15.4", "https://pypi.org/simple"),
]
expected_result = [direct_dependencies]
dependency_graph = DependencyGraphWalker(
direct_dependencies=direct_dependencies, paths=[])
walk_result = list(dependency_graph.walk())
assert walk_result == expected_result
def _do_strides(self,
dependency_graph: DependencyGraphWalker,
stack_candidates: StackCandidates,
limit: int = None) -> None:
"""Run dependency graph and do strides on the generated candidates."""
stacks_seen = 0
stacks_added = 0
strides = self._instantiate_strides()
if len(strides) > 0:
_LOGGER.info("Running strides on stack candidates")
try:
for stack_candidate in dependency_graph.walk():
stacks_seen += 1
if stacks_seen == 1:
_LOGGER.info(
"Stack producer started producing stacks, scoring and filtering produced stacks "
"from stack stream in strides...")
stride_context = StrideContext(stack_candidate)
stride_context.stats.start_timer()
for stride_instance in strides:
_LOGGER.info("Running stride %r", stride_instance.name)
stride_context.stats.reset_stats(stride_instance)
try:
stride_instance.run(stride_context)
except StrideRemoveStack as exc:
_LOGGER.debug(
"Stride %r filtered out stack %r: %s",
stride_instance.name,
stack_candidate,
str(exc),
)
break
else:
stack_candidates.add(stride_context)
stacks_added += 1
if limit is not None and stacks_added >= limit:
break
if len(strides) > 0:
stride_context.stats.log_report()
except PrematureStreamEndError as exc:
_LOGGER.debug("Stack stream closed prematurely: %s", str(exc))
reported_message = self._get_premature_stream_log_msg()
_LOGGER.warning(reported_message)
self._stack_info.append({
"type": "WARNING",
"justification": reported_message
})
finally:
if len(strides) == 0:
_LOGGER.debug("No strides were configured")
_LOGGER.info(
"Pipeline produced %d stacks in total, %d stacks were discarded by strides",
stacks_added,
stacks_seen - stacks_added,
)
def test_too_many_dependencies_error(self):
"""Test there is raised an exception if there are no dependencies specified."""
direct_dependencies = [
("flask", "1.0." + str(i), "https://pypi.org/simple")
for i in range(DependencyGraphWalker.MAX_DEPENDENCIES_COUNT + 1)
]
with pytest.raises(DependenciesCountOverflow):
DependencyGraphWalker(direct_dependencies=direct_dependencies,
paths=[])
def test_only_direct(self):
"""Test a stack with just direct dependencies - the returned values should be ordered correctly."""
direct_dependencies = [
("numpy", "1.15.4", "https://pypi.org/simple"),
("numpy", "1.16.0", "https://pypi.org/simple"),
("numpy", "1.16.1", "https://pypi.org/simple"),
("numpy", "1.16.2", "https://pypi.org/simple"),
]
expected_result = [
[("numpy", "1.16.2", "https://pypi.org/simple")],
[("numpy", "1.16.1", "https://pypi.org/simple")],
[("numpy", "1.16.0", "https://pypi.org/simple")],
[("numpy", "1.15.4", "https://pypi.org/simple")],
]
dependency_graph = DependencyGraphWalker(
direct_dependencies=direct_dependencies, paths=[])
walk_result = list(dependency_graph.walk())
assert walk_result == expected_result
def test_sorted_results_1(self):
"""Test a stack resolution with sorted dependencies - leaf ones."""
direct_dependencies = [("flask", "0.12.3", "https://pypi.org/simple")]
paths = [
[
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.9", "https://pypi.org/simple"),
("markupsafe", "1.1", "https://pypi.org/simple"),
],
[
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.9", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
],
]
dependency_graph = DependencyGraphWalker(
direct_dependencies=direct_dependencies, paths=paths)
walk_result = list(dependency_graph.walk())
assert list(set(item) for item in walk_result) == list(
reversed([set(item) for item in paths]))
def test_sorted_results_duplicate(self):
"""Test removal of duplicates in direct dependencies and paths."""
direct_dependencies = [
("flask", "0.12.3", "https://pypi.org/simple"),
("flask", "0.12.3", "https://pypi.org/simple"),
]
paths = [
[
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
],
[
("flask", "0.12.3", "https://pypi.org/simple"),
("jinja2", "2.8", "https://pypi.org/simple"),
("markupsafe", "1.0", "https://pypi.org/simple"),
],
]
dependency_graph = DependencyGraphWalker(
direct_dependencies=direct_dependencies, paths=paths)
walk_result = list(dependency_graph.walk())
assert len(walk_result) == 1
assert set(walk_result[0]) == set(paths[0])
def test_no_dependency_error(self):
"""Test there is raised an exception if there are no dependencies specified."""
with pytest.raises(NoDependenciesError):
DependencyGraphWalker(direct_dependencies=[], paths=[])
def _finalize_stepping(
self, step_context: StepContext,
count: int) -> Tuple[DependencyGraphWalker, StackCandidates]:
"""Finalize pipeline - run dependency graph to resolve fully pinned down stacks."""
_LOGGER.debug("Finalizing stepping...")
direct_dependencies_map = {
v.to_tuple(): v
for v in step_context.iter_direct_dependencies()
}
transitive_dependencies_map = {
v.to_tuple(): v
for v in step_context.iter_transitive_dependencies()
}
stack_candidates = StackCandidates(
input_project=self.project,
count=count,
direct_dependencies_map=direct_dependencies_map,
transitive_dependencies_map=transitive_dependencies_map,
)
scored_package_tuple_pairs = (step_context.dependency_graph_adaptation.
to_scored_package_tuple_pairs())
# It's important to have this sort stable so that we reflect relative ordering of paths
# based on for example semver sort which have same score.
paths = list(
sorted(scored_package_tuple_pairs, key=operator.itemgetter(0)))
# The actual score is not required. Also, make sure direct dependencies are passed sorted based on the score.
direct_dependencies = []
walker_paths = []
for path in paths:
if path[1][0] is None:
# This is direct dependency, add it.
direct_dependency = path[1][1]
if direct_dependency[1] is None or direct_dependency[2] is None:
raise NotResolvedError(
f"Direct dependency {direct_dependency!r} is not resolved, "
"cannot use it in dependency graph traversals")
direct_dependencies.append(direct_dependency)
continue
package, dependency = path[1]
if package[1] is None or package[2] is None:
raise NotResolvedError(
f"Package {package!r} is not resolved, cannot use it in dependency graph traversals"
)
if dependency[1] is None or dependency[2] is None:
raise NotResolvedError(
f"Package {dependency!r} is not resolved, cannot use it in dependency graph traversals"
)
walker_paths.append(path[1])
dependency_graph = DependencyGraphWalker(
direct_dependencies=direct_dependencies, paths=walker_paths)
estimated = dependency_graph.stacks_estimated
_LOGGER.info(
"Estimated number of stacks: %.2E (precise number: %d)",
estimated,
estimated,
)
return dependency_graph, stack_candidates