def wheel_dist_info_dir(source, name): # type: (ZipFile, str) -> str """Returns the name of the contained .dist-info directory. Raises AssertionError or UnsupportedWheel if not found, >1 found, or it doesn't match the provided name. """ # Zip file path separators must be / subdirs = set(p.split("/", 1)[0] for p in source.namelist()) info_dirs = [s for s in subdirs if s.endswith('.dist-info')] if not info_dirs: raise UnsupportedWheel(".dist-info directory not found") if len(info_dirs) > 1: raise UnsupportedWheel( "multiple .dist-info directories found: {}".format( ", ".join(info_dirs))) info_dir = info_dirs[0] info_dir_name = canonicalize_name(info_dir) canonical_name = canonicalize_name(name) if not info_dir_name.startswith(canonical_name): raise UnsupportedWheel( ".dist-info directory {!r} does not start with {!r}".format( info_dir, canonical_name)) # Zip file paths can be unicode or str depending on the zip entry flags, # so normalize it. return ensure_str(info_dir)
def get_requiring_packages(package_name): # type: (str) -> List[str] canonical_name = canonicalize_name(package_name) return [ pkg.project_name for pkg in pkg_resources.working_set if canonical_name in [canonicalize_name(required.name) for required in pkg.requires()] ]
def warn_on_mismatching_name(self): # type: () -> None metadata_name = canonicalize_name(self.metadata["Name"]) if canonicalize_name(self.req.name) == metadata_name: # Everything is fine. return # If we're here, there's a mismatch. Log a warning about it. logger.warning( 'Generating metadata for package %s ' 'produced metadata for project name %s. Fix your ' '#egg=%s fragments.', self.name, metadata_name, self.name ) self.req = Requirement(metadata_name)
def _find_name_version_sep(fragment, canonical_name): # type: (str, str) -> int """Find the separator's index based on the package's canonical name. :param fragment: A <package>+<version> filename "fragment" (stem) or egg fragment. :param canonical_name: The package's canonical name. This function is needed since the canonicalized name does not necessarily have the same length as the egg info's name part. An example:: >>> fragment = 'foo__bar-1.0' >>> canonical_name = 'foo-bar' >>> _find_name_version_sep(fragment, canonical_name) 8 """ # Project name and version must be separated by one single dash. Find all # occurrences of dashes; if the string in front of it matches the canonical # name, this is the one separating the name and version parts. for i, c in enumerate(fragment): if c != "-": continue if canonicalize_name(fragment[:i]) == canonical_name: return i raise ValueError("{} does not match {}".format(fragment, canonical_name))
def _fetch_metadata_using_lazy_wheel(self, link): # type: (Link) -> Optional[Distribution] """Fetch metadata using lazy wheel, if possible.""" if not self.use_lazy_wheel: return None if self.require_hashes: logger.debug('Lazy wheel is not used as hash checking is required') return None if link.is_file or not link.is_wheel: logger.debug( 'Lazy wheel is not used as ' '%r does not points to a remote wheel', link, ) return None wheel = Wheel(link.filename) name = canonicalize_name(wheel.name) logger.info( 'Obtaining dependency information from %s %s', name, wheel.version, ) url = link.url.split('#', 1)[0] try: return dist_from_wheel_url(name, url, self._session) except HTTPRangeRequestUnsupported: logger.debug('%s does not support range requests', url) return None
def __str__(self): # type: () -> str requirements = sorted( (req for req in self.requirements.values() if not req.comes_from), key=lambda req: canonicalize_name(req.name), ) return ' '.join(str(req.req) for req in requirements)
def check_binary_allowed(req): # type: (InstallRequirement) -> bool if req.use_pep517: return True canonical_name = canonicalize_name(req.name) allowed_formats = format_control.get_allowed_formats(canonical_name) return "binary" in allowed_formats
def get_requirement(self, name): # type: (str) -> InstallRequirement project_name = canonicalize_name(name) if project_name in self.requirements: return self.requirements[project_name] raise KeyError("No project with the name {name!r}".format(**locals()))
def _check_metadata_consistency(self, dist): # type: (Distribution) -> None """Check for consistency of project name and version of dist.""" # TODO: (Longer term) Rather than abort, reject this candidate # and backtrack. This would need resolvelib support. name = canonicalize_name(dist.project_name) if self._name is not None and self._name != name: raise MetadataInconsistent(self._ireq, "name", dist.project_name) version = dist.parsed_version if self._version is not None and self._version != version: raise MetadataInconsistent(self._ireq, "version", dist.version)
def mkurl_pypi_url(url): # type: (str) -> str loc = posixpath.join( url, urllib_parse.quote(canonicalize_name(project_name))) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's # behavior. if not loc.endswith('/'): loc = loc + '/' return loc
def _search_distribution(req_name): # type: (str) -> Optional[Distribution] """Find a distribution matching the ``req_name`` in the environment. This searches from *all* distributions available in the environment, to match the behavior of ``pkg_resources.get_distribution()``. """ # Canonicalize the name before searching in the list of # installed distributions and also while creating the package # dictionary to get the Distribution object req_name = canonicalize_name(req_name) packages = get_installed_distributions( local_only=False, skip=(), include_editables=True, editables_only=False, user_only=False, paths=None, ) pkg_dict = {canonicalize_name(p.key): p for p in packages} return pkg_dict.get(req_name)
def _req_set_item_sorter( item, # type: Tuple[str, InstallRequirement] weights, # type: Dict[Optional[str], int] ): # type: (...) -> Tuple[int, str] """Key function used to sort install requirements for installation. Based on the "weight" mapping calculated in ``get_installation_order()``. The canonical package name is returned as the second member as a tie- breaker to ensure the result is predictable, which is useful in tests. """ name = canonicalize_name(item[0]) return weights[name], name
def __repr__(self): # type: () -> str requirements = sorted( self.requirements.values(), key=lambda req: canonicalize_name(req.name), ) format_string = '<{classname} object; {count} requirement(s): {reqs}>' return format_string.format( classname=self.__class__.__name__, count=len(requirements), reqs=', '.join(str(req.req) for req in requirements), )
def make_link_evaluator(self, project_name): # type: (str) -> LinkEvaluator canonical_name = canonicalize_name(project_name) formats = self.format_control.get_allowed_formats(canonical_name) return LinkEvaluator( project_name=project_name, canonical_name=canonical_name, formats=formats, target_python=self._target_python, allow_yanked=self._allow_yanked, ignore_requires_python=self._ignore_requires_python, )
def _create_whitelist(would_be_installed, package_set): # type: (Set[str], PackageSet) -> Set[str] packages_affected = set(would_be_installed) for package_name in package_set: if package_name in packages_affected: continue for req in package_set[package_name].requires: if canonicalize_name(req.name) in packages_affected: packages_affected.add(package_name) break return packages_affected
def run(self, options, args): # type: (Values, List[str]) -> int session = self.get_default_session(options) reqs_to_uninstall = {} for name in args: req = install_req_from_line( name, isolated=options.isolated_mode, ) if req.name: reqs_to_uninstall[canonicalize_name(req.name)] = req for filename in options.requirements: for parsed_req in parse_requirements(filename, options=options, session=session): req = install_req_from_parsed_requirement( parsed_req, isolated=options.isolated_mode) if req.name: reqs_to_uninstall[canonicalize_name(req.name)] = req if not reqs_to_uninstall: raise InstallationError( 'You must give at least one requirement to {self.name} (see ' '"pip help {self.name}")'.format(**locals())) protect_pip_from_modification_on_windows( modifying_pip="pip" in reqs_to_uninstall) for req in reqs_to_uninstall.values(): uninstall_pathset = req.uninstall( auto_confirm=options.yes, verbose=self.verbosity > 0, ) if uninstall_pathset: uninstall_pathset.commit() return SUCCESS
def create_package_set_from_installed(**kwargs): # type: (**Any) -> Tuple[PackageSet, bool] """Converts a list of distributions into a PackageSet. """ # Default to using all packages installed on the system if kwargs == {}: kwargs = {"local_only": False, "skip": ()} package_set = {} problems = False for dist in get_installed_distributions(**kwargs): name = canonicalize_name(dist.project_name) try: package_set[name] = PackageDetails(dist.version, dist.requires()) except RequirementParseError as e: # Don't crash on broken metadata logger.warning("Error parsing requirements for %s: %s", name, e) problems = True return package_set, problems
def check_package_set(package_set, should_ignore=None): # type: (PackageSet, Optional[Callable[[str], bool]]) -> CheckResult """Check if a package set is consistent If should_ignore is passed, it should be a callable that takes a package name and returns a boolean. """ missing = {} conflicting = {} for package_name in package_set: # Info about dependencies of package_name missing_deps = set() # type: Set[Missing] conflicting_deps = set() # type: Set[Conflicting] if should_ignore and should_ignore(package_name): continue for req in package_set[package_name].requires: name = canonicalize_name(req.project_name) # type: str # Check if it's missing if name not in package_set: missed = True if req.marker is not None: missed = req.marker.evaluate() if missed: missing_deps.add((name, req)) continue # Check if there's a conflict version = package_set[name].version # type: str if not req.specifier.contains(version, prereleases=True): conflicting_deps.add((name, version, req)) if missing_deps: missing[package_name] = sorted(missing_deps, key=str) if conflicting_deps: conflicting[package_name] = sorted(conflicting_deps, key=str) return missing, conflicting
def _simulate_installation_of(to_install, package_set): # type: (List[InstallRequirement], PackageSet) -> Set[str] """Computes the version of packages after installing to_install. """ # Keep track of packages that were installed installed = set() # Modify it as installing requirement_set would (assuming no errors) for inst_req in to_install: abstract_dist = make_distribution_for_install_requirement(inst_req) dist = abstract_dist.get_pkg_resources_distribution() assert dist is not None name = canonicalize_name(dist.key) package_set[name] = PackageDetails(dist.version, dist.requires()) installed.add(name) return installed
def __init__( self, link, # type: Link template, # type: InstallRequirement factory, # type: Factory name=None, # type: Optional[str] version=None, # type: Optional[_BaseVersion] ): # type: (...) -> None source_link = link cache_entry = factory.get_wheel_cache_entry(link, name) if cache_entry is not None: logger.debug("Using cached wheel link: %s", cache_entry.link) link = cache_entry.link ireq = make_install_req_from_link(link, template) assert ireq.link == link if ireq.link.is_wheel and not ireq.link.is_file: wheel = Wheel(ireq.link.filename) wheel_name = canonicalize_name(wheel.name) assert name == wheel_name, ("{!r} != {!r} for wheel".format( name, wheel_name)) # Version may not be present for PEP 508 direct URLs if version is not None: assert str(version) == wheel.version, ( "{!r} != {!r} for wheel {}".format(version, wheel.version, name)) if (cache_entry is not None and cache_entry.persistent and template.link is template.original_link): ireq.original_link_is_in_wheel_cache = True super(LinkCandidate, self).__init__( link=link, source_link=source_link, ireq=ireq, factory=factory, name=name, version=version, )
def handle_mutual_excludes(value, target, other): # type: (str, Set[str], Set[str]) -> None if value.startswith('-'): raise CommandError( "--no-binary / --only-binary option requires 1 argument." ) new = value.split(',') while ':all:' in new: other.clear() target.clear() target.add(':all:') del new[:new.index(':all:') + 1] # Without a none, we want to discard everything as :all: covers it if ':none:' not in new: return for name in new: if name == ':none:': target.clear() continue name = canonicalize_name(name) other.discard(name) target.add(name)
def make_install_req_from_dist(dist, template): # type: (Distribution, InstallRequirement) -> InstallRequirement project_name = canonicalize_name(dist.project_name) if template.req: line = str(template.req) elif template.link: line = "{} @ {}".format(project_name, template.link.url) else: line = "{}=={}".format(project_name, dist.parsed_version) ireq = install_req_from_line( line, user_supplied=template.user_supplied, comes_from=template.comes_from, use_pep517=template.use_pep517, isolated=template.isolated, constraint=template.constraint, options=dict(install_options=template.install_options, global_options=template.global_options, hashes=template.hash_options), ) ireq.satisfied_by = dist return ireq
def ensure_build_location(self, build_dir, autodelete, parallel_builds): # type: (str, bool, bool) -> str assert build_dir is not None if self._temp_build_dir is not None: assert self._temp_build_dir.path return self._temp_build_dir.path if self.req is None: # Some systems have /tmp as a symlink which confuses custom # builds (such as numpy). Thus, we ensure that the real path # is returned. self._temp_build_dir = TempDirectory( kind=tempdir_kinds.REQ_BUILD, globally_managed=True ) return self._temp_build_dir.path # When parallel builds are enabled, add a UUID to the build directory # name so multiple builds do not interfere with each other. dir_name = canonicalize_name(self.name) if parallel_builds: dir_name = "{}_{}".format(dir_name, uuid.uuid4().hex) # FIXME: Is there a better place to create the build_dir? (hg and bzr # need this) if not os.path.exists(build_dir): logger.debug('Creating directory %s', build_dir) os.makedirs(build_dir) actual_build_dir = os.path.join(build_dir, dir_name) # `None` indicates that we respect the globally-configured deletion # settings, which is what we actually want when auto-deleting. delete_arg = None if autodelete else False return TempDirectory( path=actual_build_dir, delete=delete_arg, kind=tempdir_kinds.REQ_BUILD, globally_managed=True, ).path
def evaluate_link(self, link): # type: (Link) -> Tuple[bool, Optional[Text]] """ Determine whether a link is a candidate for installation. :return: A tuple (is_candidate, result), where `result` is (1) a version string if `is_candidate` is True, and (2) if `is_candidate` is False, an optional string to log the reason the link fails to qualify. """ version = None if link.is_yanked and not self._allow_yanked: reason = link.yanked_reason or '<none given>' # Mark this as a unicode string to prevent "UnicodeEncodeError: # 'ascii' codec can't encode character" in Python 2 when # the reason contains non-ascii characters. return (False, u'yanked for reason: {}'.format(reason)) if link.egg_fragment: egg_info = link.egg_fragment ext = link.ext else: egg_info, ext = link.splitext() if not ext: return (False, 'not a file') if ext not in SUPPORTED_EXTENSIONS: return (False, 'unsupported archive format: {}'.format(ext)) if "binary" not in self._formats and ext == WHEEL_EXTENSION: reason = 'No binaries permitted for {}'.format( self.project_name) return (False, reason) if "macosx10" in link.path and ext == '.zip': return (False, 'macosx10 one') if ext == WHEEL_EXTENSION: try: wheel = Wheel(link.filename) except InvalidWheelFilename: return (False, 'invalid wheel filename') if canonicalize_name(wheel.name) != self._canonical_name: reason = 'wrong project name (not {})'.format( self.project_name) return (False, reason) supported_tags = self._target_python.get_tags() if not wheel.supported(supported_tags): # Include the wheel's tags in the reason string to # simplify troubleshooting compatibility issues. file_tags = wheel.get_formatted_file_tags() reason = ( "none of the wheel's tags match: {}".format( ', '.join(file_tags) ) ) return (False, reason) version = wheel.version # This should be up by the self.ok_binary check, but see issue 2700. if "source" not in self._formats and ext != WHEEL_EXTENSION: reason = 'No sources permitted for {}'.format(self.project_name) return (False, reason) if not version: version = _extract_version_from_fragment( egg_info, self._canonical_name, ) if not version: reason = 'Missing project version for {}'.format(self.project_name) return (False, reason) match = self._py_version_re.search(version) if match: version = version[:match.start()] py_version = match.group(1) if py_version != self._target_python.py_version: return (False, 'Python version is incorrect') supports_python = _check_link_requires_python( link, version_info=self._target_python.py_version_info, ignore_requires_python=self._ignore_requires_python, ) if not supports_python: # Return None for the reason text to suppress calling # _log_skipped_link(). return (False, None) logger.debug('Found link %s, version: %s', link, version) return (True, version)
def add_named_requirement(self, install_req): # type: (InstallRequirement) -> None assert install_req.name project_name = canonicalize_name(install_req.name) self.requirements[project_name] = install_req
def search_packages_info(query): # type: (List[str]) -> Iterator[Dict[str, str]] """ Gather details from installed distributions. Print distribution name, version, location, and installed files. Installed files requires a pip generated 'installed-files.txt' in the distributions '.egg-info' directory. """ installed = {} for p in pkg_resources.working_set: installed[canonicalize_name(p.project_name)] = p query_names = [canonicalize_name(name) for name in query] missing = sorted([ name for name, pkg in zip(query, query_names) if pkg not in installed ]) if missing: logger.warning('Package(s) not found: %s', ', '.join(missing)) def get_requiring_packages(package_name): # type: (str) -> List[str] canonical_name = canonicalize_name(package_name) return [ pkg.project_name for pkg in pkg_resources.working_set if canonical_name in [canonicalize_name(required.name) for required in pkg.requires()] ] for dist in [installed[pkg] for pkg in query_names if pkg in installed]: package = { 'name': dist.project_name, 'version': dist.version, 'location': dist.location, 'requires': [dep.project_name for dep in dist.requires()], 'required_by': get_requiring_packages(dist.project_name) } file_list = None metadata = '' if isinstance(dist, pkg_resources.DistInfoDistribution): # RECORDs should be part of .dist-info metadatas if dist.has_metadata('RECORD'): lines = dist.get_metadata_lines('RECORD') paths = [line.split(',')[0] for line in lines] paths = [os.path.join(dist.location, p) for p in paths] file_list = [os.path.relpath(p, dist.location) for p in paths] if dist.has_metadata('METADATA'): metadata = dist.get_metadata('METADATA') else: # Otherwise use pip's log for .egg-info's if dist.has_metadata('installed-files.txt'): paths = dist.get_metadata_lines('installed-files.txt') paths = [os.path.join(dist.egg_info, p) for p in paths] file_list = [os.path.relpath(p, dist.location) for p in paths] if dist.has_metadata('PKG-INFO'): metadata = dist.get_metadata('PKG-INFO') if dist.has_metadata('entry_points.txt'): entry_points = dist.get_metadata_lines('entry_points.txt') package['entry_points'] = entry_points if dist.has_metadata('INSTALLER'): for line in dist.get_metadata_lines('INSTALLER'): if line.strip(): package['installer'] = line.strip() break # @todo: Should pkg_resources.Distribution have a # `get_pkg_info` method? feed_parser = FeedParser() feed_parser.feed(metadata) pkg_info_dict = feed_parser.close() for key in ('metadata-version', 'summary', 'home-page', 'author', 'author-email', 'license'): package[key] = pkg_info_dict.get(key) # It looks like FeedParser cannot deal with repeated headers classifiers = [] for line in metadata.splitlines(): if line.startswith('Classifier: '): classifiers.append(line[len('Classifier: '):]) package['classifiers'] = classifiers if file_list: package['files'] = sorted(file_list) yield package
def resolve(self, root_reqs, check_supported_wheels): # type: (List[InstallRequirement], bool) -> RequirementSet constraints = {} # type: Dict[str, SpecifierSet] user_requested = set() # type: Set[str] requirements = [] for req in root_reqs: if req.constraint: # Ensure we only accept valid constraints problem = check_invalid_constraint_type(req) if problem: raise InstallationError(problem) if not req.match_markers(): continue name = canonicalize_name(req.name) if name in constraints: constraints[name] = constraints[name] & req.specifier else: constraints[name] = req.specifier else: if req.user_supplied and req.name: user_requested.add(canonicalize_name(req.name)) r = self.factory.make_requirement_from_install_req( req, requested_extras=(), ) if r is not None: requirements.append(r) provider = PipProvider( factory=self.factory, constraints=constraints, ignore_dependencies=self.ignore_dependencies, upgrade_strategy=self.upgrade_strategy, user_requested=user_requested, ) reporter = BaseReporter() resolver = RLResolver(provider, reporter) try: try_to_avoid_resolution_too_deep = 2000000 self._result = resolver.resolve( requirements, max_rounds=try_to_avoid_resolution_too_deep, ) except ResolutionImpossible as e: error = self.factory.get_installation_error(e) six.raise_from(error, e) req_set = RequirementSet(check_supported_wheels=check_supported_wheels) for candidate in self._result.mapping.values(): ireq = candidate.get_install_requirement() if ireq is None: continue # Check if there is already an installation under the same name, # and set a flag for later stages to uninstall it, if needed. # * There isn't, good -- no uninstalltion needed. # * The --force-reinstall flag is set. Always reinstall. # * The installation is different in version or editable-ness, so # we need to uninstall it to install the new distribution. # * The installed version is the same as the pending distribution. # Skip this distrubiton altogether to save work. installed_dist = self.factory.get_dist_to_uninstall(candidate) if installed_dist is None: ireq.should_reinstall = False elif self.factory.force_reinstall: ireq.should_reinstall = True elif installed_dist.parsed_version != candidate.version: ireq.should_reinstall = True elif dist_is_editable(installed_dist) != candidate.is_editable: ireq.should_reinstall = True else: continue link = candidate.source_link if link and link.is_yanked: # The reason can contain non-ASCII characters, Unicode # is required for Python 2. msg = ( u'The candidate selected for download or install is a ' u'yanked version: {name!r} candidate (version {version} ' u'at {link})\nReason for being yanked: {reason}').format( name=candidate.name, version=candidate.version, link=link, reason=link.yanked_reason or u'<none given>', ) logger.warning(msg) req_set.add_named_requirement(ireq) reqs = req_set.all_requirements self.factory.preparer.prepare_linked_requirements_more(reqs) return req_set
def has_requirement(self, name): # type: (str) -> bool project_name = canonicalize_name(name) return (project_name in self.requirements and not self.requirements[project_name].constraint)
def name(self): # type: () -> str canonical_name = canonicalize_name(self._ireq.req.name) return format_name(canonical_name, self._extras)
def format_name(project, extras): # type: (str, FrozenSet[str]) -> str if not extras: return project canonical_extras = sorted(canonicalize_name(e) for e in extras) return "{}[{}]".format(project, ",".join(canonical_extras))