def incompatibilities_for(
self, package: DependencyPackage) -> list[Incompatibility]:
"""
Returns incompatibilities that encapsulate a given package's dependencies,
or that it can't be safely selected.
If multiple subsequent versions of this package have the same
dependencies, this will return incompatibilities that reflect that. It
won't return incompatibilities that have already been returned by a
previous call to _incompatibilities_for().
"""
if package.is_root():
dependencies = package.all_requires
else:
dependencies = package.requires
if not package.python_constraint.allows_all(
self._python_constraint):
transitive_python_constraint = get_python_constraint_from_marker(
package.dependency.transitive_marker)
intersection = package.python_constraint.intersect(
transitive_python_constraint)
difference = transitive_python_constraint.difference(
intersection)
# The difference is only relevant if it intersects
# the root package python constraint
difference = difference.intersect(self._python_constraint)
if (transitive_python_constraint.is_any()
or self._python_constraint.intersect(
package.dependency.python_constraint).is_empty()
or intersection.is_empty()
or not difference.is_empty()):
return [
Incompatibility(
[Term(package.to_dependency(), True)],
PythonCause(package.python_versions,
str(self._python_constraint)),
)
]
_dependencies = [
dep for dep in dependencies if dep.name not in self.UNSAFE_PACKAGES
and self._python_constraint.allows_any(dep.python_constraint) and (
not self._env or dep.marker.validate(self._env.marker_env))
]
dependencies = self._get_dependencies_with_overrides(
_dependencies, package)
return [
Incompatibility(
[Term(package.to_dependency(), True),
Term(dep, False)],
DependencyCause(),
) for dep in dependencies
]
def solve_in_compatibility_mode(self, overrides, use_latest=None):
locked = {}
for package in self._locked.packages:
locked[package.name] = DependencyPackage(package.to_dependency(),
package)
packages = []
depths = []
for override in overrides:
self._provider.debug(
"<comment>Retrying dependency resolution "
"with the following overrides ({}).</comment>".format(
override))
self._provider.set_overrides(override)
_packages, _depths = self._solve(use_latest=use_latest)
for index, package in enumerate(_packages):
if package not in packages:
packages.append(package)
depths.append(_depths[index])
continue
else:
idx = packages.index(package)
pkg = packages[idx]
depths[idx] = max(depths[idx], _depths[index])
for dep in package.requires:
if dep not in pkg.requires:
pkg.requires.append(dep)
return packages, depths
def check_solver_result(
root, provider, result=None, error=None, tries=None, locked=None, use_latest=None
):
if locked is not None:
locked = {k: DependencyPackage(l.to_dependency(), l) for k, l in locked.items()}
solver = VersionSolver(root, provider, locked=locked, use_latest=use_latest)
try:
solution = solver.solve()
except SolveFailure as e:
if error:
assert str(e) == error
if tries is not None:
assert solver.solution.attempted_solutions == tries
return
raise
packages = {}
for package in solution.packages:
packages[package.name] = str(package.version)
assert result == packages
if tries is not None:
assert solution.attempted_solutions == tries
def solve_in_compatibility_mode(self, constraints, use_latest=None):
locked = {}
for package in self._locked.packages:
locked[package.name] = DependencyPackage(package.to_dependency(),
package)
packages = []
depths = []
for constraint in constraints:
constraint = parse_constraint(constraint)
intersection = constraint.intersect(
self._package.python_constraint)
self._provider.debug(
"<comment>Retrying dependency resolution "
"for Python ({}).</comment>".format(intersection))
with self._package.with_python_versions(str(intersection)):
_packages, _depths = self._solve(use_latest=use_latest)
for index, package in enumerate(_packages):
if package not in packages:
packages.append(package)
depths.append(_depths[index])
continue
else:
idx = packages.index(package)
pkg = packages[idx]
depths[idx] = max(depths[idx], _depths[index])
pkg.marker = pkg.marker.union(package.marker)
for dep in package.requires:
if dep not in pkg.requires:
pkg.requires.append(dep)
return packages, depths
def _solve(self, use_latest=None):
if self._provider._overrides:
self._overrides.append(self._provider._overrides)
locked = {}
for package in self._locked.packages:
locked[package.name] = DependencyPackage(package.to_dependency(),
package)
try:
result = resolve_version(self._package,
self._provider,
locked=locked,
use_latest=use_latest)
packages = result.packages
except OverrideNeeded as e:
return self.solve_in_compatibility_mode(e.overrides,
use_latest=use_latest)
except SolveFailure as e:
raise SolverProblemError(e)
results = dict(
depth_first_search(PackageNode(self._package, packages),
aggregate_package_nodes))
# Return the packages in their original order with associated depths
final_packages = packages
depths = [results[package] for package in packages]
return final_packages, depths
def _solve(
self,
use_latest: list[str] | None = None
) -> tuple[list[Package], list[int]]:
if self._provider._overrides:
self._overrides.append(self._provider._overrides)
locked: dict[str, list[DependencyPackage]] = defaultdict(list)
for package in self._locked.packages:
locked[package.name].append(
DependencyPackage(package.to_dependency(), package))
for dependency_packages in locked.values():
dependency_packages.sort(
key=lambda p: p.package.version,
reverse=True,
)
try:
result = resolve_version(self._package,
self._provider,
locked=locked,
use_latest=use_latest)
packages = result.packages
except OverrideNeeded as e:
return self.solve_in_compatibility_mode(e.overrides,
use_latest=use_latest)
except SolveFailure as e:
raise SolverProblemError(e)
combined_nodes = depth_first_search(
PackageNode(self._package, packages))
results = dict(
aggregate_package_nodes(nodes) for nodes in combined_nodes)
# Merging feature packages with base packages
final_packages = []
depths = []
for package in packages:
if package.features:
for _package in packages:
if (not _package.features and _package.name == package.name
and _package.version == package.version):
for dep in package.requires:
# Prevent adding base package as a dependency to itself
if _package.name == dep.name:
continue
if dep not in _package.requires:
_package.add_dependency(dep)
else:
final_packages.append(package)
depths.append(results[package])
# Return the packages in their original order with associated depths
return final_packages, depths
def _solve(
self,
use_latest: List[str] = None) -> Tuple[List["Package"], List[int]]:
if self._provider._overrides:
self._overrides.append(self._provider._overrides)
locked = {
package.name: DependencyPackage(package.to_dependency(), package)
for package in self._locked.packages
}
try:
result = resolve_version(self._package,
self._provider,
locked=locked,
use_latest=use_latest)
packages = result.packages
except OverrideNeeded as e:
return self.solve_in_compatibility_mode(e.overrides,
use_latest=use_latest)
except SolveFailure as e:
raise SolverProblemError(e)
# NOTE passing explicit empty array for seen to reset between invocations during
# update + install cycle
results = dict(
depth_first_search(PackageNode(self._package, packages, seen=[]),
aggregate_package_nodes))
# Merging feature packages with base packages
final_packages = []
depths = []
for package in packages:
if package.features:
for _package in packages:
if (_package.name == package.name
and not _package.is_same_package_as(package)
and _package.version == package.version):
for dep in package.requires:
if dep.is_same_package_as(_package):
continue
if dep not in _package.requires:
_package.add_dependency(dep)
continue
final_packages.append(package)
depths.append(results[package])
# Return the packages in their original order with associated depths
return final_packages, depths
def get_project_dependency_packages(
self,
project_requires: list[Dependency],
project_python_marker: BaseMarker | None = None,
extras: bool | Sequence[str] | None = None,
) -> Iterator[DependencyPackage]:
# Apply the project python marker to all requirements.
if project_python_marker is not None:
marked_requires: list[Dependency] = []
for require in project_requires:
require = deepcopy(require)
require.marker = require.marker.intersect(
project_python_marker)
marked_requires.append(require)
project_requires = marked_requires
repository = self.locked_repository()
# Build a set of all packages required by our selected extras
extra_package_names: set[str] | None = None
if extras is not True:
extra_package_names = set(
get_extra_package_names(
repository.packages,
self.lock_data.get("extras", {}),
extras or (),
))
# If a package is optional and we haven't opted in to it, do not select
selected = []
for dependency in project_requires:
try:
package = repository.find_packages(dependency=dependency)[0]
except IndexError:
continue
if extra_package_names is not None and (
package.optional
and package.name not in extra_package_names):
# a package is locked as optional, but is not activated via extras
continue
selected.append(dependency)
for package, dependency in self.get_project_dependencies(
project_requires=selected,
locked_packages=repository.packages,
):
for extra in dependency.extras:
package.requires_extras.append(extra)
yield DependencyPackage(dependency=dependency, package=package)
def get_project_dependency_packages(
self,
project_requires: List[Dependency],
dev: bool = False,
extras: Optional[Union[bool, Sequence[str]]] = None,
) -> Iterator[DependencyPackage]:
repository = self.locked_repository(with_dev_reqs=dev)
# Build a set of all packages required by our selected extras
extra_package_names = (
None if (isinstance(extras, bool) and extras is True) else ()
)
if extra_package_names is not None:
extra_package_names = set(
get_extra_package_names(
repository.packages,
self.lock_data.get("extras", {}),
extras or (),
)
)
# If a package is optional and we haven't opted in to it, do not select
selected = []
for dependency in project_requires:
try:
package = repository.find_packages(dependency=dependency)[0]
except IndexError:
continue
if extra_package_names is not None and (
package.optional and package.name not in extra_package_names
):
# a package is locked as optional, but is not activated via extras
continue
selected.append(dependency)
for dependency in self.get_project_dependencies(
project_requires=selected,
locked_packages=repository.packages,
with_nested=True,
):
try:
package = repository.find_packages(dependency=dependency)[0]
except IndexError:
continue
for extra in dependency.extras:
package.requires_extras.append(extra)
yield DependencyPackage(dependency=dependency, package=package)
def _solve(
self,
use_latest: List[str] = None) -> Tuple[List[Package], List[int]]:
if self._provider._overrides:
self._overrides.append(self._provider._overrides)
locked = {}
for package in self._locked.packages:
locked[package.name] = DependencyPackage(package.to_dependency(),
package)
try:
result = resolve_version(self._package,
self._provider,
locked=locked,
use_latest=use_latest)
packages = result.packages
except OverrideNeeded as e:
return self.solve_in_compatibility_mode(e.overrides,
use_latest=use_latest)
except SolveFailure as e:
raise SolverProblemError(e)
results = dict(
depth_first_search(PackageNode(self._package, packages),
aggregate_package_nodes))
# Merging feature packages with base packages
final_packages = []
depths = []
for package in packages:
if package.features:
for _package in packages:
if (_package.name == package.name
and not _package.is_same_package_as(package)
and _package.version == package.version):
for dep in package.requires:
if dep.is_same_package_as(_package):
continue
if dep not in _package.requires:
_package.requires.append(dep)
continue
final_packages.append(package)
depths.append(results[package])
# Return the packages in their original order with associated depths
return final_packages, depths
def _get_locked(
self, dependency: Dependency, *, allow_similar: bool = False
) -> DependencyPackage | None:
if dependency.name in self._use_latest:
return None
locked = self._locked.get(dependency.name, [])
for package in locked:
if (allow_similar or dependency.is_same_package_as(package.package)) and (
dependency.constraint.allows(package.version)
or package.is_prerelease()
and dependency.constraint.allows(package.version.next_patch())
):
return DependencyPackage(dependency, package.package)
return None
def test_complete_package_preserves_source_type_with_subdirectories(
provider: Provider, root: ProjectPackage) -> None:
dependency_one = Factory.create_dependency(
"one",
{
"git": "https://github.com/demo/subdirectories.git",
"subdirectory": "one",
},
)
dependency_one_copy = Factory.create_dependency(
"one",
{
"git": "https://github.com/demo/subdirectories.git",
"subdirectory": "one-copy",
},
)
dependency_two = Factory.create_dependency(
"two",
{
"git": "https://github.com/demo/subdirectories.git",
"subdirectory": "two"
},
)
root.add_dependency(
Factory.create_dependency(
"one",
{
"git": "https://github.com/demo/subdirectories.git",
"subdirectory": "one",
},
))
root.add_dependency(dependency_one_copy)
root.add_dependency(dependency_two)
complete_package = provider.complete_package(
DependencyPackage(root.to_dependency(), root))
requires = complete_package.package.all_requires
assert len(requires) == 3
assert {r.to_pep_508()
for r in requires} == {
dependency_one.to_pep_508(),
dependency_one_copy.to_pep_508(),
dependency_two.to_pep_508(),
}
def test_complete_package_preserves_source_type(provider: Provider,
root: ProjectPackage) -> None:
fixtures = Path(__file__).parent.parent / "fixtures"
project_dir = fixtures.joinpath("with_conditional_path_deps")
for folder in ["demo_one", "demo_two"]:
path = (project_dir / folder).as_posix()
root.add_dependency(Factory.create_dependency("demo", {"path": path}))
complete_package = provider.complete_package(
DependencyPackage(root.to_dependency(), root))
requires = complete_package.package.all_requires
assert len(requires) == 2
assert {requires[0].source_url, requires[1].source_url} == {
project_dir.joinpath("demo_one").as_posix(),
project_dir.joinpath("demo_two").as_posix(),
}
def check_solver_result(
root: "ProjectPackage",
provider: "Provider",
result: Optional[Dict[str, str]] = None,
error: Optional[str] = None,
tries: Optional[int] = None,
locked: Optional[Dict[str, Package]] = None,
use_latest: Optional[List[str]] = None,
) -> None:
if locked is not None:
locked = {
k: DependencyPackage(l.to_dependency(), l)
for k, l in locked.items()
}
solver = VersionSolver(root,
provider,
locked=locked,
use_latest=use_latest)
try:
solution = solver.solve()
except SolveFailure as e:
if error:
assert str(e) == error
if tries is not None:
assert solver.solution.attempted_solutions == tries
return
raise
except AssertionError as e:
if error:
assert str(e) == error
return
raise
packages = {}
for package in solution.packages:
packages[package.name] = str(package.version)
assert result == packages
if tries is not None:
assert solution.attempted_solutions == tries
def _solve(self, use_latest=None):
self._branches.append(self._package.python_versions)
locked = {}
for package in self._locked.packages:
locked[package.name] = DependencyPackage(package.to_dependency(),
package)
try:
result = resolve_version(self._package,
self._provider,
locked=locked,
use_latest=use_latest)
packages = result.packages
except CompatibilityError as e:
return self.solve_in_compatibility_mode(e.constraints,
use_latest=use_latest)
except SolveFailure as e:
raise SolverProblemError(e)
graph = self._build_graph(self._package, packages)
depths = []
final_packages = []
for package in packages:
category, optional, marker, depth = self._get_tags_for_package(
package, graph)
if marker is None:
marker = AnyMarker()
if marker.is_empty():
continue
package.category = category
package.optional = optional
package.marker = marker
depths.append(depth)
final_packages.append(package)
return final_packages, depths
def _solve(self, use_latest=None):
if self._provider._overrides:
self._overrides.append(self._provider._overrides)
locked = {}
for package in self._locked.packages:
locked[package.name] = DependencyPackage(package.to_dependency(),
package)
try:
result = resolve_version(self._package,
self._provider,
locked=locked,
use_latest=use_latest)
packages = result.packages
except OverrideNeeded as e:
return self.solve_in_compatibility_mode(e.overrides,
use_latest=use_latest)
except SolveFailure as e:
raise SolverProblemError(e)
graph = self._build_graph(self._package, packages)
depths = []
final_packages = []
for package in packages:
category, optional, depth = self._get_tags_for_package(
package, graph)
package.category = category
package.optional = optional
depths.append(depth)
final_packages.append(package)
return final_packages, depths
def complete_package(
self, package
): # type: (DependencyPackage) -> DependencyPackage
if package.is_root():
package = package.clone()
requires = package.all_requires
elif not package.is_root() and package.source_type not in {
"directory",
"file",
"url",
"git",
}:
package = DependencyPackage(
package.dependency,
self._pool.package(
package.name,
package.version.text,
extras=list(package.dependency.extras),
repository=package.dependency.source_name,
),
)
requires = package.requires
else:
requires = package.requires
if self._load_deferred:
# Retrieving constraints for deferred dependencies
for r in requires:
if r.is_directory():
self.search_for_directory(r)
elif r.is_file():
self.search_for_file(r)
elif r.is_vcs():
self.search_for_vcs(r)
elif r.is_url():
self.search_for_url(r)
optional_dependencies = []
_dependencies = []
# If some extras/features were required, we need to
# add a special dependency representing the base package
# to the current package
if package.dependency.extras:
for extra in package.dependency.extras:
if extra not in package.extras:
continue
optional_dependencies += [d.name for d in package.extras[extra]]
package = package.with_features(list(package.dependency.extras))
_dependencies.append(package.without_features().to_dependency())
for dep in requires:
if not self._python_constraint.allows_any(dep.python_constraint):
continue
if dep.name in self.UNSAFE_PACKAGES:
continue
if self._env and not dep.marker.validate(self._env.marker_env):
continue
if not package.is_root():
if (dep.is_optional() and dep.name not in optional_dependencies) or (
dep.in_extras
and not set(dep.in_extras).intersection(package.dependency.extras)
):
continue
_dependencies.append(dep)
overrides = self._overrides.get(package, {})
dependencies = []
overridden = []
for dep in _dependencies:
if dep.name in overrides:
if dep.name in overridden:
continue
dependencies.append(overrides[dep.name])
overridden.append(dep.name)
continue
dependencies.append(dep)
# Searching for duplicate dependencies
#
# If the duplicate dependencies have the same constraint,
# the requirements will be merged.
#
# For instance:
# - enum34; python_version=="2.7"
# - enum34; python_version=="3.3"
#
# will become:
# - enum34; python_version=="2.7" or python_version=="3.3"
#
# If the duplicate dependencies have different constraints
# we have to split the dependency graph.
#
# An example of this is:
# - pypiwin32 (220); sys_platform == "win32" and python_version >= "3.6"
# - pypiwin32 (219); sys_platform == "win32" and python_version < "3.6"
duplicates = dict()
for dep in dependencies:
if dep.name not in duplicates:
duplicates[dep.name] = []
duplicates[dep.name].append(dep)
dependencies = []
for dep_name, deps in duplicates.items():
if len(deps) == 1:
dependencies.append(deps[0])
continue
self.debug("<debug>Duplicate dependencies for {}</debug>".format(dep_name))
# Regrouping by constraint
by_constraint = dict()
for dep in deps:
if dep.constraint not in by_constraint:
by_constraint[dep.constraint] = []
by_constraint[dep.constraint].append(dep)
# We merge by constraint
for constraint, _deps in by_constraint.items():
new_markers = []
for dep in _deps:
marker = dep.marker.without_extras()
if marker.is_any():
# No marker or only extras
continue
new_markers.append(marker)
if not new_markers:
continue
dep = _deps[0]
dep.marker = dep.marker.union(MarkerUnion(*new_markers))
by_constraint[constraint] = [dep]
continue
if len(by_constraint) == 1:
self.debug(
"<debug>Merging requirements for {}</debug>".format(str(deps[0]))
)
dependencies.append(list(by_constraint.values())[0][0])
continue
# We leave dependencies as-is if they have the same
# python/platform constraints.
# That way the resolver will pickup the conflict
# and display a proper error.
_deps = [value[0] for value in by_constraint.values()]
seen = set()
for _dep in _deps:
pep_508_dep = _dep.to_pep_508(False)
if ";" not in pep_508_dep:
_requirements = ""
else:
_requirements = pep_508_dep.split(";")[1].strip()
if _requirements not in seen:
seen.add(_requirements)
if len(_deps) != len(seen):
for _dep in _deps:
dependencies.append(_dep)
continue
# At this point, we raise an exception that will
# tell the solver to make new resolutions with specific overrides.
#
# For instance, if the foo (1.2.3) package has the following dependencies:
# - bar (>=2.0) ; python_version >= "3.6"
# - bar (<2.0) ; python_version < "3.6"
#
# then the solver will need to make two new resolutions
# with the following overrides:
# - {<Package foo (1.2.3): {"bar": <Dependency bar (>=2.0)>}
# - {<Package foo (1.2.3): {"bar": <Dependency bar (<2.0)>}
markers = []
for constraint, _deps in by_constraint.items():
markers.append(_deps[0].marker)
_deps = [_dep[0] for _dep in by_constraint.values()]
self.debug(
"<warning>Different requirements found for {}.</warning>".format(
", ".join(
"<c1>{}</c1> <fg=default>(<c2>{}</c2>)</> with markers <b>{}</b>".format(
d.name,
d.pretty_constraint,
d.marker if not d.marker.is_any() else "*",
)
for d in _deps[:-1]
)
+ " and "
+ "<c1>{}</c1> <fg=default>(<c2>{}</c2>)</> with markers <b>{}</b>".format(
_deps[-1].name,
_deps[-1].pretty_constraint,
_deps[-1].marker if not _deps[-1].marker.is_any() else "*",
)
)
)
# We need to check if one of the duplicate dependencies
# has no markers. If there is one, we need to change its
# environment markers to the inverse of the union of the
# other dependencies markers.
# For instance, if we have the following dependencies:
# - ipython
# - ipython (1.2.4) ; implementation_name == "pypy"
#
# the marker for `ipython` will become `implementation_name != "pypy"`.
any_markers_dependencies = [d for d in _deps if d.marker.is_any()]
other_markers_dependencies = [d for d in _deps if not d.marker.is_any()]
if any_markers_dependencies:
marker = other_markers_dependencies[0].marker
for other_dep in other_markers_dependencies[1:]:
marker = marker.union(other_dep.marker)
for i, d in enumerate(_deps):
if d.marker.is_any():
_deps[i].marker = marker.invert()
overrides = []
for _dep in _deps:
current_overrides = self._overrides.copy()
package_overrides = current_overrides.get(package, {}).copy()
package_overrides.update({_dep.name: _dep})
current_overrides.update({package: package_overrides})
overrides.append(current_overrides)
raise OverrideNeeded(*overrides)
# Modifying dependencies as needed
clean_dependencies = []
for dep in dependencies:
if not package.dependency.transitive_marker.without_extras().is_any():
marker_intersection = (
package.dependency.transitive_marker.without_extras().intersect(
dep.marker.without_extras()
)
)
if marker_intersection.is_empty():
# The dependency is not needed, since the markers specified
# for the current package selection are not compatible with
# the markers for the current dependency, so we skip it
continue
dep.transitive_marker = marker_intersection
if not package.dependency.python_constraint.is_any():
python_constraint_intersection = dep.python_constraint.intersect(
package.dependency.python_constraint
)
if python_constraint_intersection.is_empty():
# This dependency is not needed under current python constraint.
continue
dep.transitive_python_versions = str(python_constraint_intersection)
clean_dependencies.append(dep)
package.requires = clean_dependencies
return package
def complete_package(self,
package: DependencyPackage) -> DependencyPackage:
if package.is_root():
package = package.clone()
requires = package.all_requires
elif not package.is_root() and package.source_type not in {
"directory",
"file",
"url",
"git",
}:
package = DependencyPackage(
package.dependency,
self._pool.package(
package.name,
package.version.text,
extras=list(package.dependency.extras),
repository=package.dependency.source_name,
),
)
requires = package.requires
else:
requires = package.requires
if self._load_deferred:
# Retrieving constraints for deferred dependencies
for r in requires:
if r.is_directory():
self.search_for_directory(r)
elif r.is_file():
self.search_for_file(r)
elif r.is_vcs():
self.search_for_vcs(r)
elif r.is_url():
self.search_for_url(r)
optional_dependencies = []
_dependencies = []
# If some extras/features were required, we need to
# add a special dependency representing the base package
# to the current package
if package.dependency.extras:
for extra in package.dependency.extras:
if extra not in package.extras:
continue
optional_dependencies += [
d.name for d in package.extras[extra]
]
package = package.with_features(list(package.dependency.extras))
_dependencies.append(package.without_features().to_dependency())
for dep in requires:
if not self._python_constraint.allows_any(dep.python_constraint):
continue
if dep.name in self.UNSAFE_PACKAGES:
continue
if self._env and not dep.marker.validate(self._env.marker_env):
continue
if not package.is_root() and (
(dep.is_optional() and dep.name not in optional_dependencies)
or (dep.in_extras and not set(dep.in_extras).intersection(
package.dependency.extras))):
continue
_dependencies.append(dep)
dependencies = self._get_dependencies_with_overrides(
_dependencies, package)
# Searching for duplicate dependencies
#
# If the duplicate dependencies have the same constraint,
# the requirements will be merged.
#
# For instance:
# - enum34; python_version=="2.7"
# - enum34; python_version=="3.3"
#
# will become:
# - enum34; python_version=="2.7" or python_version=="3.3"
#
# If the duplicate dependencies have different constraints
# we have to split the dependency graph.
#
# An example of this is:
# - pypiwin32 (220); sys_platform == "win32" and python_version >= "3.6"
# - pypiwin32 (219); sys_platform == "win32" and python_version < "3.6"
duplicates: dict[str, list[Dependency]] = {}
for dep in dependencies:
if dep.complete_name not in duplicates:
duplicates[dep.complete_name] = []
duplicates[dep.complete_name].append(dep)
dependencies = []
for dep_name, deps in duplicates.items():
if len(deps) == 1:
dependencies.append(deps[0])
continue
self.debug(f"<debug>Duplicate dependencies for {dep_name}</debug>")
deps = self._merge_dependencies_by_marker(deps)
deps = self._merge_dependencies_by_constraint(deps)
if len(deps) == 1:
self.debug(
f"<debug>Merging requirements for {deps[0]!s}</debug>")
dependencies.append(deps[0])
continue
# We leave dependencies as-is if they have the same
# python/platform constraints.
# That way the resolver will pickup the conflict
# and display a proper error.
seen = set()
for dep in deps:
pep_508_dep = dep.to_pep_508(False)
if ";" not in pep_508_dep:
_requirements = ""
else:
_requirements = pep_508_dep.split(";")[1].strip()
if _requirements not in seen:
seen.add(_requirements)
if len(deps) != len(seen):
for dep in deps:
dependencies.append(dep)
continue
# At this point, we raise an exception that will
# tell the solver to make new resolutions with specific overrides.
#
# For instance, if the foo (1.2.3) package has the following dependencies:
# - bar (>=2.0) ; python_version >= "3.6"
# - bar (<2.0) ; python_version < "3.6"
#
# then the solver will need to make two new resolutions
# with the following overrides:
# - {<Package foo (1.2.3): {"bar": <Dependency bar (>=2.0)>}
# - {<Package foo (1.2.3): {"bar": <Dependency bar (<2.0)>}
def fmt_warning(d: Dependency) -> str:
marker = d.marker if not d.marker.is_any() else "*"
return (
f"<c1>{d.name}</c1> <fg=default>(<c2>{d.pretty_constraint}</c2>)</>"
f" with markers <b>{marker}</b>")
warnings = ", ".join(fmt_warning(d) for d in deps[:-1])
warnings += f" and {fmt_warning(deps[-1])}"
self.debug(
f"<warning>Different requirements found for {warnings}.</warning>"
)
# We need to check if one of the duplicate dependencies
# has no markers. If there is one, we need to change its
# environment markers to the inverse of the union of the
# other dependencies markers.
# For instance, if we have the following dependencies:
# - ipython
# - ipython (1.2.4) ; implementation_name == "pypy"
#
# the marker for `ipython` will become `implementation_name != "pypy"`.
#
# Further, we have to merge the constraints of the requirements
# without markers into the constraints of the requirements with markers.
# for instance, if we have the following dependencies:
# - foo (>= 1.2)
# - foo (!= 1.2.1) ; python == 3.10
#
# the constraint for the second entry will become (!= 1.2.1, >= 1.2)
any_markers_dependencies = [d for d in deps if d.marker.is_any()]
other_markers_dependencies = [
d for d in deps if not d.marker.is_any()
]
marker = other_markers_dependencies[0].marker
for other_dep in other_markers_dependencies[1:]:
marker = marker.union(other_dep.marker)
inverted_marker = marker.invert()
if any_markers_dependencies:
for dep_any in any_markers_dependencies:
dep_any.marker = inverted_marker
for dep_other in other_markers_dependencies:
dep_other.set_constraint(
dep_other.constraint.intersect(dep_any.constraint))
elif not inverted_marker.is_empty(
) and self._python_constraint.allows_any(
get_python_constraint_from_marker(inverted_marker)):
# if there is no any marker dependency
# and the inverted marker is not empty,
# a dependency with the inverted union of all markers is required
# in order to not miss other dependencies later, for instance:
# - foo (1.0) ; python == 3.7
# - foo (2.0) ; python == 3.8
# - bar (2.0) ; python == 3.8
# - bar (3.0) ; python == 3.9
#
# the last dependency would be missed without this,
# because the intersection with both foo dependencies is empty
inverted_marker_dep = deps[0].with_constraint(
EmptyConstraint())
inverted_marker_dep.marker = inverted_marker
deps.append(inverted_marker_dep)
overrides = []
overrides_marker_intersection: BaseMarker = AnyMarker()
for dep_overrides in self._overrides.values():
for dep in dep_overrides.values():
overrides_marker_intersection = (
overrides_marker_intersection.intersect(dep.marker))
for dep in deps:
if not overrides_marker_intersection.intersect(
dep.marker).is_empty():
current_overrides = self._overrides.copy()
package_overrides = current_overrides.get(package,
{}).copy()
package_overrides.update({dep.name: dep})
current_overrides.update({package: package_overrides})
overrides.append(current_overrides)
if overrides:
raise OverrideNeeded(*overrides)
# Modifying dependencies as needed
clean_dependencies = []
for dep in dependencies:
if not package.dependency.transitive_marker.without_extras(
).is_any():
marker_intersection = (package.dependency.transitive_marker.
without_extras().intersect(
dep.marker.without_extras()))
if marker_intersection.is_empty():
# The dependency is not needed, since the markers specified
# for the current package selection are not compatible with
# the markers for the current dependency, so we skip it
continue
dep.transitive_marker = marker_intersection
if not package.dependency.python_constraint.is_any():
python_constraint_intersection = dep.python_constraint.intersect(
package.dependency.python_constraint)
if python_constraint_intersection.is_empty():
# This dependency is not needed under current python constraint.
continue
dep.transitive_python_versions = str(
python_constraint_intersection)
clean_dependencies.append(dep)
package = DependencyPackage(
package.dependency, package.with_dependency_groups([], only=True))
for dep in clean_dependencies:
package.add_dependency(dep)
return package
def complete_package(
self, package): # type: (DependencyPackage) -> DependencyPackage
if package.is_root():
package = package.clone()
if not package.is_root() and package.source_type not in {
"directory",
"file",
"url",
"git",
}:
package = DependencyPackage(
package.dependency,
self._pool.package(
package.name,
package.version.text,
extras=package.requires_extras,
repository=package.dependency.source_name,
),
)
dependencies = [
r for r in package.requires
if self._package.python_constraint.allows_any(r.python_constraint)
]
# Searching for duplicate dependencies
#
# If the duplicate dependencies have the same constraint,
# the requirements will be merged.
#
# For instance:
# - enum34; python_version=="2.7"
# - enum34; python_version=="3.3"
#
# will become:
# - enum34; python_version=="2.7" or python_version=="3.3"
#
# If the duplicate dependencies have different constraints
# we have to split the dependency graph.
#
# An example of this is:
# - pypiwin32 (220); sys_platform == "win32" and python_version >= "3.6"
# - pypiwin32 (219); sys_platform == "win32" and python_version < "3.6"
duplicates = OrderedDict()
for dep in dependencies:
if dep.name not in duplicates:
duplicates[dep.name] = []
duplicates[dep.name].append(dep)
dependencies = []
for dep_name, deps in duplicates.items():
if len(deps) == 1:
dependencies.append(deps[0])
continue
self.debug("<debug>Duplicate dependencies for {}</debug>".format(
dep_name))
# Regrouping by constraint
by_constraint = OrderedDict()
for dep in deps:
if dep.constraint not in by_constraint:
by_constraint[dep.constraint] = []
by_constraint[dep.constraint].append(dep)
# We merge by constraint
for constraint, _deps in by_constraint.items():
new_markers = []
for dep in _deps:
marker = dep.marker.without_extras()
if marker.is_empty():
# No marker or only extras
continue
new_markers.append(marker)
if not new_markers:
continue
dep = _deps[0]
dep.marker = dep.marker.union(MarkerUnion(*new_markers))
by_constraint[constraint] = [dep]
continue
if len(by_constraint) == 1:
self.debug("<debug>Merging requirements for {}</debug>".format(
str(deps[0])))
dependencies.append(list(by_constraint.values())[0][0])
continue
# We leave dependencies as-is if they have the same
# python/platform constraints.
# That way the resolver will pickup the conflict
# and display a proper error.
_deps = [value[0] for value in by_constraint.values()]
seen = set()
for _dep in _deps:
pep_508_dep = _dep.to_pep_508(False)
if ";" not in pep_508_dep:
_requirements = ""
else:
_requirements = pep_508_dep.split(";")[1].strip()
if _requirements not in seen:
seen.add(_requirements)
if len(_deps) != len(seen):
for _dep in _deps:
dependencies.append(_dep)
continue
# At this point, we raise an exception that will
# tell the solver to enter compatibility mode
# which means it will resolve for subsets
# Python constraints
#
# For instance, if our root package requires Python ~2.7 || ^3.6
# And we have one dependency that requires Python <3.6
# and the other Python >=3.6 than the solver will solve
# dependencies for Python >=2.7,<2.8 || >=3.4,<3.6
# and Python >=3.6,<4.0
python_constraints = []
for constraint, _deps in by_constraint.items():
python_constraints.append(_deps[0].python_versions)
_deps = [str(_dep[0]) for _dep in by_constraint.values()]
self.debug(
"<warning>Different requirements found for {}.</warning>".
format(", ".join(_deps[:-1]) + " and " + _deps[-1]))
raise CompatibilityError(*python_constraints)
# Modifying dependencies as needed
for dep in dependencies:
if not package.dependency.python_constraint.is_any():
dep.transitive_python_versions = str(
dep.python_constraint.intersect(
package.dependency.python_constraint))
if (package.dependency.is_directory()
or package.dependency.is_file()) and (dep.is_directory()
or dep.is_file()):
if dep.path.as_posix().startswith(package.source_url):
relative = (Path(package.source_url) /
dep.path).relative_to(package.source_url)
else:
relative = Path(package.source_url) / dep.path
# TODO: Improve the way we set the correct relative path for dependencies
dep._path = relative
package.requires = dependencies
return package
