def complete_package( self, package ): # type: (DependencyPackage) -> DependencyPackage if package.is_root(): package = package.clone() requires = package.all_requires elif not package.is_root() and package.source_type not in { "directory", "file", "url", "git", }: package = DependencyPackage( package.dependency, self._pool.package( package.name, package.version.text, extras=list(package.dependency.extras), repository=package.dependency.source_name, ), ) requires = package.requires else: requires = package.requires if self._load_deferred: # Retrieving constraints for deferred dependencies for r in requires: if r.is_directory(): self.search_for_directory(r) elif r.is_file(): self.search_for_file(r) elif r.is_vcs(): self.search_for_vcs(r) elif r.is_url(): self.search_for_url(r) optional_dependencies = [] _dependencies = [] # If some extras/features were required, we need to # add a special dependency representing the base package # to the current package if package.dependency.extras: for extra in package.dependency.extras: if extra not in package.extras: continue optional_dependencies += [d.name for d in package.extras[extra]] package = package.with_features(list(package.dependency.extras)) _dependencies.append(package.without_features().to_dependency()) for dep in requires: if not self._python_constraint.allows_any(dep.python_constraint): continue if dep.name in self.UNSAFE_PACKAGES: continue if self._env and not dep.marker.validate(self._env.marker_env): continue if not package.is_root(): if (dep.is_optional() and dep.name not in optional_dependencies) or ( dep.in_extras and not set(dep.in_extras).intersection(package.dependency.extras) ): continue _dependencies.append(dep) overrides = self._overrides.get(package, {}) dependencies = [] overridden = [] for dep in _dependencies: if dep.name in overrides: if dep.name in overridden: continue dependencies.append(overrides[dep.name]) overridden.append(dep.name) continue dependencies.append(dep) # Searching for duplicate dependencies # # If the duplicate dependencies have the same constraint, # the requirements will be merged. # # For instance: # - enum34; python_version=="2.7" # - enum34; python_version=="3.3" # # will become: # - enum34; python_version=="2.7" or python_version=="3.3" # # If the duplicate dependencies have different constraints # we have to split the dependency graph. # # An example of this is: # - pypiwin32 (220); sys_platform == "win32" and python_version >= "3.6" # - pypiwin32 (219); sys_platform == "win32" and python_version < "3.6" duplicates = dict() for dep in dependencies: if dep.name not in duplicates: duplicates[dep.name] = [] duplicates[dep.name].append(dep) dependencies = [] for dep_name, deps in duplicates.items(): if len(deps) == 1: dependencies.append(deps[0]) continue self.debug("<debug>Duplicate dependencies for {}</debug>".format(dep_name)) # Regrouping by constraint by_constraint = dict() for dep in deps: if dep.constraint not in by_constraint: by_constraint[dep.constraint] = [] by_constraint[dep.constraint].append(dep) # We merge by constraint for constraint, _deps in by_constraint.items(): new_markers = [] for dep in _deps: marker = dep.marker.without_extras() if marker.is_any(): # No marker or only extras continue new_markers.append(marker) if not new_markers: continue dep = _deps[0] dep.marker = dep.marker.union(MarkerUnion(*new_markers)) by_constraint[constraint] = [dep] continue if len(by_constraint) == 1: self.debug( "<debug>Merging requirements for {}</debug>".format(str(deps[0])) ) dependencies.append(list(by_constraint.values())[0][0]) continue # We leave dependencies as-is if they have the same # python/platform constraints. # That way the resolver will pickup the conflict # and display a proper error. _deps = [value[0] for value in by_constraint.values()] seen = set() for _dep in _deps: pep_508_dep = _dep.to_pep_508(False) if ";" not in pep_508_dep: _requirements = "" else: _requirements = pep_508_dep.split(";")[1].strip() if _requirements not in seen: seen.add(_requirements) if len(_deps) != len(seen): for _dep in _deps: dependencies.append(_dep) continue # At this point, we raise an exception that will # tell the solver to make new resolutions with specific overrides. # # For instance, if the foo (1.2.3) package has the following dependencies: # - bar (>=2.0) ; python_version >= "3.6" # - bar (<2.0) ; python_version < "3.6" # # then the solver will need to make two new resolutions # with the following overrides: # - {<Package foo (1.2.3): {"bar": <Dependency bar (>=2.0)>} # - {<Package foo (1.2.3): {"bar": <Dependency bar (<2.0)>} markers = [] for constraint, _deps in by_constraint.items(): markers.append(_deps[0].marker) _deps = [_dep[0] for _dep in by_constraint.values()] self.debug( "<warning>Different requirements found for {}.</warning>".format( ", ".join( "<c1>{}</c1> <fg=default>(<c2>{}</c2>)</> with markers <b>{}</b>".format( d.name, d.pretty_constraint, d.marker if not d.marker.is_any() else "*", ) for d in _deps[:-1] ) + " and " + "<c1>{}</c1> <fg=default>(<c2>{}</c2>)</> with markers <b>{}</b>".format( _deps[-1].name, _deps[-1].pretty_constraint, _deps[-1].marker if not _deps[-1].marker.is_any() else "*", ) ) ) # We need to check if one of the duplicate dependencies # has no markers. If there is one, we need to change its # environment markers to the inverse of the union of the # other dependencies markers. # For instance, if we have the following dependencies: # - ipython # - ipython (1.2.4) ; implementation_name == "pypy" # # the marker for `ipython` will become `implementation_name != "pypy"`. any_markers_dependencies = [d for d in _deps if d.marker.is_any()] other_markers_dependencies = [d for d in _deps if not d.marker.is_any()] if any_markers_dependencies: marker = other_markers_dependencies[0].marker for other_dep in other_markers_dependencies[1:]: marker = marker.union(other_dep.marker) for i, d in enumerate(_deps): if d.marker.is_any(): _deps[i].marker = marker.invert() overrides = [] for _dep in _deps: current_overrides = self._overrides.copy() package_overrides = current_overrides.get(package, {}).copy() package_overrides.update({_dep.name: _dep}) current_overrides.update({package: package_overrides}) overrides.append(current_overrides) raise OverrideNeeded(*overrides) # Modifying dependencies as needed clean_dependencies = [] for dep in dependencies: if not package.dependency.transitive_marker.without_extras().is_any(): marker_intersection = ( package.dependency.transitive_marker.without_extras().intersect( dep.marker.without_extras() ) ) if marker_intersection.is_empty(): # The dependency is not needed, since the markers specified # for the current package selection are not compatible with # the markers for the current dependency, so we skip it continue dep.transitive_marker = marker_intersection if not package.dependency.python_constraint.is_any(): python_constraint_intersection = dep.python_constraint.intersect( package.dependency.python_constraint ) if python_constraint_intersection.is_empty(): # This dependency is not needed under current python constraint. continue dep.transitive_python_versions = str(python_constraint_intersection) clean_dependencies.append(dep) package.requires = clean_dependencies return package
def complete_package(self, package: DependencyPackage) -> DependencyPackage: if package.is_root(): package = package.clone() requires = package.all_requires elif not package.is_root() and package.source_type not in { "directory", "file", "url", "git", }: package = DependencyPackage( package.dependency, self._pool.package( package.name, package.version.text, extras=list(package.dependency.extras), repository=package.dependency.source_name, ), ) requires = package.requires else: requires = package.requires if self._load_deferred: # Retrieving constraints for deferred dependencies for r in requires: if r.is_directory(): self.search_for_directory(r) elif r.is_file(): self.search_for_file(r) elif r.is_vcs(): self.search_for_vcs(r) elif r.is_url(): self.search_for_url(r) optional_dependencies = [] _dependencies = [] # If some extras/features were required, we need to # add a special dependency representing the base package # to the current package if package.dependency.extras: for extra in package.dependency.extras: if extra not in package.extras: continue optional_dependencies += [ d.name for d in package.extras[extra] ] package = package.with_features(list(package.dependency.extras)) _dependencies.append(package.without_features().to_dependency()) for dep in requires: if not self._python_constraint.allows_any(dep.python_constraint): continue if dep.name in self.UNSAFE_PACKAGES: continue if self._env and not dep.marker.validate(self._env.marker_env): continue if not package.is_root() and ( (dep.is_optional() and dep.name not in optional_dependencies) or (dep.in_extras and not set(dep.in_extras).intersection( package.dependency.extras))): continue _dependencies.append(dep) dependencies = self._get_dependencies_with_overrides( _dependencies, package) # Searching for duplicate dependencies # # If the duplicate dependencies have the same constraint, # the requirements will be merged. # # For instance: # - enum34; python_version=="2.7" # - enum34; python_version=="3.3" # # will become: # - enum34; python_version=="2.7" or python_version=="3.3" # # If the duplicate dependencies have different constraints # we have to split the dependency graph. # # An example of this is: # - pypiwin32 (220); sys_platform == "win32" and python_version >= "3.6" # - pypiwin32 (219); sys_platform == "win32" and python_version < "3.6" duplicates: dict[str, list[Dependency]] = {} for dep in dependencies: if dep.complete_name not in duplicates: duplicates[dep.complete_name] = [] duplicates[dep.complete_name].append(dep) dependencies = [] for dep_name, deps in duplicates.items(): if len(deps) == 1: dependencies.append(deps[0]) continue self.debug(f"<debug>Duplicate dependencies for {dep_name}</debug>") deps = self._merge_dependencies_by_marker(deps) deps = self._merge_dependencies_by_constraint(deps) if len(deps) == 1: self.debug( f"<debug>Merging requirements for {deps[0]!s}</debug>") dependencies.append(deps[0]) continue # We leave dependencies as-is if they have the same # python/platform constraints. # That way the resolver will pickup the conflict # and display a proper error. seen = set() for dep in deps: pep_508_dep = dep.to_pep_508(False) if ";" not in pep_508_dep: _requirements = "" else: _requirements = pep_508_dep.split(";")[1].strip() if _requirements not in seen: seen.add(_requirements) if len(deps) != len(seen): for dep in deps: dependencies.append(dep) continue # At this point, we raise an exception that will # tell the solver to make new resolutions with specific overrides. # # For instance, if the foo (1.2.3) package has the following dependencies: # - bar (>=2.0) ; python_version >= "3.6" # - bar (<2.0) ; python_version < "3.6" # # then the solver will need to make two new resolutions # with the following overrides: # - {<Package foo (1.2.3): {"bar": <Dependency bar (>=2.0)>} # - {<Package foo (1.2.3): {"bar": <Dependency bar (<2.0)>} def fmt_warning(d: Dependency) -> str: marker = d.marker if not d.marker.is_any() else "*" return ( f"<c1>{d.name}</c1> <fg=default>(<c2>{d.pretty_constraint}</c2>)</>" f" with markers <b>{marker}</b>") warnings = ", ".join(fmt_warning(d) for d in deps[:-1]) warnings += f" and {fmt_warning(deps[-1])}" self.debug( f"<warning>Different requirements found for {warnings}.</warning>" ) # We need to check if one of the duplicate dependencies # has no markers. If there is one, we need to change its # environment markers to the inverse of the union of the # other dependencies markers. # For instance, if we have the following dependencies: # - ipython # - ipython (1.2.4) ; implementation_name == "pypy" # # the marker for `ipython` will become `implementation_name != "pypy"`. # # Further, we have to merge the constraints of the requirements # without markers into the constraints of the requirements with markers. # for instance, if we have the following dependencies: # - foo (>= 1.2) # - foo (!= 1.2.1) ; python == 3.10 # # the constraint for the second entry will become (!= 1.2.1, >= 1.2) any_markers_dependencies = [d for d in deps if d.marker.is_any()] other_markers_dependencies = [ d for d in deps if not d.marker.is_any() ] marker = other_markers_dependencies[0].marker for other_dep in other_markers_dependencies[1:]: marker = marker.union(other_dep.marker) inverted_marker = marker.invert() if any_markers_dependencies: for dep_any in any_markers_dependencies: dep_any.marker = inverted_marker for dep_other in other_markers_dependencies: dep_other.set_constraint( dep_other.constraint.intersect(dep_any.constraint)) elif not inverted_marker.is_empty( ) and self._python_constraint.allows_any( get_python_constraint_from_marker(inverted_marker)): # if there is no any marker dependency # and the inverted marker is not empty, # a dependency with the inverted union of all markers is required # in order to not miss other dependencies later, for instance: # - foo (1.0) ; python == 3.7 # - foo (2.0) ; python == 3.8 # - bar (2.0) ; python == 3.8 # - bar (3.0) ; python == 3.9 # # the last dependency would be missed without this, # because the intersection with both foo dependencies is empty inverted_marker_dep = deps[0].with_constraint( EmptyConstraint()) inverted_marker_dep.marker = inverted_marker deps.append(inverted_marker_dep) overrides = [] overrides_marker_intersection: BaseMarker = AnyMarker() for dep_overrides in self._overrides.values(): for dep in dep_overrides.values(): overrides_marker_intersection = ( overrides_marker_intersection.intersect(dep.marker)) for dep in deps: if not overrides_marker_intersection.intersect( dep.marker).is_empty(): current_overrides = self._overrides.copy() package_overrides = current_overrides.get(package, {}).copy() package_overrides.update({dep.name: dep}) current_overrides.update({package: package_overrides}) overrides.append(current_overrides) if overrides: raise OverrideNeeded(*overrides) # Modifying dependencies as needed clean_dependencies = [] for dep in dependencies: if not package.dependency.transitive_marker.without_extras( ).is_any(): marker_intersection = (package.dependency.transitive_marker. without_extras().intersect( dep.marker.without_extras())) if marker_intersection.is_empty(): # The dependency is not needed, since the markers specified # for the current package selection are not compatible with # the markers for the current dependency, so we skip it continue dep.transitive_marker = marker_intersection if not package.dependency.python_constraint.is_any(): python_constraint_intersection = dep.python_constraint.intersect( package.dependency.python_constraint) if python_constraint_intersection.is_empty(): # This dependency is not needed under current python constraint. continue dep.transitive_python_versions = str( python_constraint_intersection) clean_dependencies.append(dep) package = DependencyPackage( package.dependency, package.with_dependency_groups([], only=True)) for dep in clean_dependencies: package.add_dependency(dep) return package