def install(candidate): pip_logging._log_state.indentation = 0 dependencies = repository.get_dependencies(candidate)[0] key = safe_name(candidate.name).lower() dist = Distribution(key, candidate.version, candidate.req.editable) dist.dependencies = dependencies rv.add_distribution(dist)
def add_dependencies(self, requirements: Dict[str, Requirement], show_message: bool = True) -> None: for name, dep in requirements.items(): if dep.from_section == "default": deps = self.tool_settings["dependencies"] elif dep.from_section == "dev": deps = self.tool_settings["dev-dependencies"] else: section = f"{dep.from_section}-dependencies" if section not in self.tool_settings: self.tool_settings[section] = tomlkit.table() deps = self.tool_settings[section] matched_name = next( filter(lambda k: strip_extras(name)[0] == safe_name(k).lower(), deps.keys()), None, ) name_to_save = dep.name if matched_name is None else matched_name _, req_dict = dep.as_req_dict() if isinstance(req_dict, dict): req = tomlkit.inline_table() req.update(req_dict) req_dict = req deps[name_to_save] = req_dict self.write_pyproject(show_message)
def cached_wheel(cache_dir, link, format_control, package_name): if not cache_dir: return link if not link: return link if link.is_wheel: return link if not package_name: return link canonical_name = pkg_resources.safe_name(package_name).lower() formats = pip.index.fmt_ctl_formats(format_control, canonical_name) if "binary" not in formats: return link root = _cache_for_link(cache_dir, link) try: wheel_names = os.listdir(root) except OSError as e: if e.errno == errno.ENOENT: return link raise candidates = [] for wheel_name in wheel_names: try: wheel = Wheel(wheel_name) except InvalidWheelFilename: continue if not wheel.supported(): # Built for a different python/arch/etc continue candidates.append((wheel.support_index_min(), wheel_name)) if not candidates: return link candidates.sort() path = os.path.join(root, candidates[0][1]) return pip.index.Link(path_to_url(path), trusted=True)
def get_locked_candidates(self, section: Optional[str] = None ) -> Dict[str, Candidate]: if not self.lockfile_file.is_file(): return {} section = section or "default" result = {} for package in [dict(p) for p in self.lockfile.get("package", [])]: if section != "__all__" and section not in package["sections"]: continue version = package.get("version") if version: package["version"] = f"=={version}" package_name = package.pop("name") req = Requirement.from_req_dict(package_name, dict(package)) can = Candidate(req, self.environment, name=package_name, version=version) can.marker = req.marker can.hashes = { item["file"]: item["hash"] for item in self.lockfile["metadata"].get( f"{req.key} {version}", []) } or None result[req.identify()] = can if section in ("default", "__all__") and self.meta.name and self.meta.version: result[safe_name( self.meta.name).lower()] = self.make_self_candidate(True) return result
def update_candidate(self, key: str) -> Tuple[Distribution, Candidate]: """Update candidate""" can = self.candidates[key] dist = self.working_set[safe_name(can.name).lower()] installer = self.get_installer() installer.uninstall(dist) installer.install(can) return dist, can
def name(self): # type: () -> Optional[str] if self.req is None: return None return six.ensure_str(pkg_resources.safe_name(self.req.name))
def find_packages_latest_versions(self, options): index_urls = [options.index_url] + options.extra_index_urls if options.no_index: logger.info('Ignoring indexes: %s', ','.join(index_urls)) index_urls = [] dependency_links = [] for dist in get_installed_distributions(local_only=options.local, user_only=options.user): if dist.has_metadata('dependency_links.txt'): dependency_links.extend( dist.get_metadata_lines('dependency_links.txt'), ) with self._build_session(options) as session: finder = self._build_package_finder(options, index_urls, session) finder.add_dependency_links(dependency_links) installed_packages = get_installed_distributions( local_only=options.local, user_only=options.user, include_editables=False, ) format_control = FormatControl(set(), set()) wheel_cache = WheelCache(options.cache_dir, format_control) for dist in installed_packages: req = InstallRequirement.from_line( dist.key, None, isolated=options.isolated_mode, wheel_cache=wheel_cache ) typ = 'unknown' try: link = finder.find_requirement(req, True) # If link is None, means installed version is most # up-to-date if link is None: continue except DistributionNotFound: continue else: canonical_name = pkg_resources.safe_name(req.name).lower() formats = fmt_ctl_formats(format_control, canonical_name) search = Search( req.name, canonical_name, formats) remote_version = finder._link_package_versions( link, search).version if link.is_wheel: typ = 'wheel' else: typ = 'sdist' yield dist, remote_version, typ
def print_results( ui: termui.UI, hits: SearchResult, working_set: WorkingSet, terminal_width: Optional[int] = None, ) -> None: if not hits: return name_column_width = ( max([len(hit.name) + len(hit.version or "") for hit in hits]) + 4 ) for hit in hits: name = hit.name summary = hit.summary or "" latest = hit.version or "" if terminal_width is not None: target_width = terminal_width - name_column_width - 5 if target_width > 10: # wrap and indent summary to fit terminal summary = textwrap.wrap(summary, target_width) summary = ("\n" + " " * (name_column_width + 2)).join(summary) current_width = len(name) + len(latest) + 4 spaces = " " * (name_column_width - current_width) line = "{name} ({latest}){spaces} - {summary}".format( name=termui.green(name, bold=True), latest=termui.yellow(latest), spaces=spaces, summary=summary, ) try: ui.echo(line) if safe_name(name).lower() in working_set: dist = working_set[safe_name(name).lower()] if dist.version == latest: ui.echo(" INSTALLED: %s (latest)" % dist.version) else: ui.echo(" INSTALLED: %s" % dist.version) ui.echo(" LATEST: %s" % latest) except UnicodeEncodeError: pass
def __init__(self, **kwargs): self._marker = None self.from_section = "default" self.marker_no_python = None # type: Optional[Marker] self.requires_python = PySpecSet() # type: PySpecSet for k, v in kwargs.items(): if k == "specifier": v = get_specifier(v) setattr(self, k, v) if self.name and not self.project_name: self.project_name = safe_name(self.name) self.key = self.project_name.lower()
def find_packages_latest_versions(self, options): index_urls = [options.index_url] + options.extra_index_urls if options.no_index: logger.info('Ignoring indexes: %s', ','.join(index_urls)) index_urls = [] dependency_links = [] for dist in get_installed_distributions(local_only=options.local, user_only=options.user): if dist.has_metadata('dependency_links.txt'): dependency_links.extend( dist.get_metadata_lines('dependency_links.txt'), ) with self._build_session(options) as session: finder = self._build_package_finder(options, index_urls, session) finder.add_dependency_links(dependency_links) installed_packages = get_installed_distributions( local_only=options.local, user_only=options.user, include_editables=False, ) format_control = FormatControl(set(), set()) wheel_cache = WheelCache(options.cache_dir, format_control) for dist in installed_packages: req = InstallRequirement.from_line( dist.key, None, isolated=options.isolated_mode, wheel_cache=wheel_cache) typ = 'unknown' try: link = finder.find_requirement(req, True) # If link is None, means installed version is most # up-to-date if link is None: continue except DistributionNotFound: continue else: canonical_name = pkg_resources.safe_name(req.name).lower() formats = fmt_ctl_formats(format_control, canonical_name) search = Search(req.name, canonical_name, formats) remote_version = finder._link_package_versions( link, search).version if link.is_wheel: typ = 'wheel' else: typ = 'sdist' yield dist, remote_version, typ
def fmt_ctl_handle_mutual_exclude(value, target, other): new = value.split(',') while ':all:' in new: other.clear() target.clear() target.add(':all:') del new[:new.index(':all:') + 1] if ':none:' not in new: # Without a none, we want to discard everything as :all: covers it return for name in new: if name == ':none:': target.clear() continue name = pkg_resources.safe_name(name).lower() other.discard(name) target.add(name)
def update_candidate(self, key: str) -> Tuple[Distribution, Candidate]: """Update candidate""" can = self.candidates[key] dist = self.working_set[safe_name(can.name).lower()] installer = self.get_installer() with stream.open_spinner( f"Updating {stream.green(key, bold=True)} {stream.yellow(dist.version)} " f"-> {stream.yellow(can.version)}...") as spinner: try: installer.uninstall(dist) installer.install(can) except Exception: spinner.fail(f"Update {stream.green(key, bold=True)} " f"-> {stream.yellow(can.version)} failed") raise else: spinner.succeed(f"Update {stream.green(key, bold=True)} " f"-> {stream.yellow(can.version)} successful") return dist, can
def install_candidates(self, candidates: List[Candidate], update: bool = False) -> None: """Install candidates. :param candidates: a list of candidates to be installed. :param update: whether to remove existed packages. """ installer = self.get_installer() working_set = self.environment.get_working_set() for can in candidates: if update: dist = working_set[safe_name(can.name).lower()] context.io.echo( f"Updating {context.io.green(can.name, bold=True)} " f"{context.io.yellow(dist.version)} -> " f"{context.io.yellow(can.version)}") installer.uninstall(dist) else: context.io.echo(f"Installing {can.format()}...") installer.install(can)
def project_name(self) -> Optional[str]: return safe_name(self.name) if self.name else None
def canonicalize_name(name): """Convert an arbitrary string to a canonical name used for comparison""" return pkg_resources.safe_name(name).lower()
def do_update( project: Project, dev: bool = False, sections: Sequence[str] = (), default: bool = True, strategy: str = "reuse", save: str = "compatible", unconstrained: bool = False, top: bool = False, dry_run: bool = False, packages: Sequence[str] = (), ) -> None: """Update specified packages or all packages""" check_project_file(project) if len(packages) > 0 and (top or len(sections) > 1 or not default): raise PdmUsageError( "packages argument can't be used together with multiple -s or " "--no-default and --top.") all_dependencies = project.all_dependencies updated_deps = {} if not packages: sections = translate_sections(project, default, dev, sections or ()) for section in sections: updated_deps.update(all_dependencies[section]) else: section = sections[0] if sections else ("dev" if dev else "default") dependencies = all_dependencies[section] for name in packages: matched_name = next( filter( lambda k: safe_name(strip_extras(k)[0]).lower() == safe_name(name).lower(), dependencies.keys(), ), None, ) if not matched_name: raise ProjectError( "{} does not exist in {} {}dependencies.".format( termui.green(name, bold=True), section, "dev-" if dev else "")) updated_deps[matched_name] = dependencies[matched_name] project.core.ui.echo("Updating packages: {}.".format(", ".join( termui.green(v, bold=True) for v in updated_deps))) if unconstrained: for _, dep in updated_deps.items(): dep.specifier = get_specifier("") reqs = [r for deps in all_dependencies.values() for r in deps.values()] resolved = do_lock( project, strategy if top or packages else "all", updated_deps.keys(), reqs, dry_run=dry_run, ) do_sync( project, sections=sections, dev=dev, default=default, clean=False, dry_run=dry_run, tracked_names=updated_deps.keys() if top else None, ) if unconstrained and not dry_run: # Need to update version constraints save_version_specifiers(updated_deps, resolved, save) project.add_dependencies(updated_deps, section, dev) lockfile = project.lockfile project.write_lockfile(lockfile, False)
def _link_package_versions(self, link, search_name): """ Return an iterable of triples (pkg_resources_version_key, link, python_version) that can be extracted from the given link. Meant to be overridden by subclasses, not called by clients. """ platform = get_platform() version = None if link.egg_fragment: egg_info = link.egg_fragment else: egg_info, ext = link.splitext() if not ext: if link not in self.logged_links: logger.debug('Skipping link %s; not a file', link) self.logged_links.add(link) return if egg_info.endswith('.tar'): # Special double-extension case: egg_info = egg_info[:-4] ext = '.tar' + ext if ext not in self._known_extensions(): if link not in self.logged_links: logger.debug( 'Skipping link %s; unknown archive format: %s', link, ext, ) self.logged_links.add(link) return if "macosx10" in link.path and ext == '.zip': if link not in self.logged_links: logger.debug('Skipping link %s; macosx10 one', link) self.logged_links.add(link) return if ext == wheel_ext: try: wheel = Wheel(link.filename) except InvalidWheelFilename: logger.debug( 'Skipping %s because the wheel filename is invalid', link ) return if (pkg_resources.safe_name(wheel.name).lower() != pkg_resources.safe_name(search_name).lower()): logger.debug( 'Skipping link %s; wrong project name (not %s)', link, search_name, ) return if not wheel.supported(): logger.debug( 'Skipping %s because it is not compatible with this ' 'Python', link, ) return # This is a dirty hack to prevent installing Binary Wheels from # PyPI unless it is a Windows or Mac Binary Wheel. This is # paired with a change to PyPI disabling uploads for the # same. Once we have a mechanism for enabling support for # binary wheels on linux that deals with the inherent problems # of binary distribution this can be removed. comes_from = getattr(link, "comes_from", None) if ( ( not platform.startswith('win') and not platform.startswith('macosx') and not platform == 'cli' ) and comes_from is not None and urllib_parse.urlparse( comes_from.url ).netloc.endswith(PyPI.netloc)): if not wheel.supported(tags=supported_tags_noarch): logger.debug( "Skipping %s because it is a pypi-hosted binary " "Wheel on an unsupported platform", link, ) return version = wheel.version if not version: version = self._egg_info_matches(egg_info, search_name, link) if version is None: logger.debug( 'Skipping link %s; wrong project name (not %s)', link, search_name, ) return if (link.internal is not None and not link.internal and not normalize_name(search_name).lower() in self.allow_external and not self.allow_all_external): # We have a link that we are sure is external, so we should skip # it unless we are allowing externals logger.debug("Skipping %s because it is externally hosted.", link) self.need_warn_external = True return if (link.verifiable is not None and not link.verifiable and not (normalize_name(search_name).lower() in self.allow_unverified)): # We have a link that we are sure we cannot verify its integrity, # so we should skip it unless we are allowing unsafe installs # for this requirement. logger.debug( "Skipping %s because it is an insecure and unverifiable file.", link, ) self.need_warn_unverified = True return match = self._py_version_re.search(version) if match: version = version[:match.start()] py_version = match.group(1) if py_version != sys.version[:3]: logger.debug( 'Skipping %s because Python version is incorrect', link ) return logger.debug('Found link %s, version: %s', link, version) return InstallationCandidate(search_name, version, link)
def project_name(self) -> str | None: return safe_name(self.name) if self.name else None # type: ignore
def _link_package_versions(self, link, search): """Return an InstallationCandidate or None""" platform = get_platform() version = None if link.egg_fragment: egg_info = link.egg_fragment ext = link.ext else: egg_info, ext = link.splitext() if not ext: self._log_skipped_link(link, 'not a file') return if ext not in SUPPORTED_EXTENSIONS: self._log_skipped_link(link, 'unsupported archive format: %s' % ext) return if "binary" not in search.formats and ext == wheel_ext: self._log_skipped_link( link, 'No binaries permitted for %s' % search.supplied) return if "macosx10" in link.path and ext == '.zip': self._log_skipped_link(link, 'macosx10 one') return if ext == wheel_ext: try: wheel = Wheel(link.filename) except InvalidWheelFilename: self._log_skipped_link(link, 'invalid wheel filename') return if (pkg_resources.safe_name(wheel.name).lower() != search.canonical): self._log_skipped_link( link, 'wrong project name (not %s)' % search.supplied) return if not wheel.supported(): self._log_skipped_link( link, 'it is not compatible with this Python') return # This is a dirty hack to prevent installing Binary Wheels from # PyPI unless it is a Windows or Mac Binary Wheel. This is # paired with a change to PyPI disabling uploads for the # same. Once we have a mechanism for enabling support for # binary wheels on linux that deals with the inherent problems # of binary distribution this can be removed. comes_from = getattr(link, "comes_from", None) if ((not platform.startswith('win') and not platform.startswith('macosx') and not platform == 'cli') and comes_from is not None and urllib_parse.urlparse( comes_from.url).netloc.endswith(PyPI.netloc)): if not wheel.supported(tags=supported_tags_noarch): self._log_skipped_link( link, "it is a pypi-hosted binary " "Wheel on an unsupported platform", ) return version = wheel.version # This should be up by the search.ok_binary check, but see issue 2700. if "source" not in search.formats and ext != wheel_ext: self._log_skipped_link( link, 'No sources permitted for %s' % search.supplied) return if not version: version = egg_info_matches(egg_info, search.supplied, link) if version is None: self._log_skipped_link( link, 'wrong project name (not %s)' % search.supplied) return if (link.internal is not None and not link.internal and not normalize_name(search.supplied).lower() in self.allow_external and not self.allow_all_external): # We have a link that we are sure is external, so we should skip # it unless we are allowing externals self._log_skipped_link(link, 'it is externally hosted') self.need_warn_external = True return if (link.verifiable is not None and not link.verifiable and not (normalize_name(search.supplied).lower() in self.allow_unverified)): # We have a link that we are sure we cannot verify its integrity, # so we should skip it unless we are allowing unsafe installs # for this requirement. self._log_skipped_link(link, 'it is an insecure and unverifiable file') self.need_warn_unverified = True return match = self._py_version_re.search(version) if match: version = version[:match.start()] py_version = match.group(1) if py_version != sys.version[:3]: self._log_skipped_link(link, 'Python version is incorrect') return logger.debug('Found link %s, version: %s', link, version) return InstallationCandidate(search.supplied, version, link)
def _find_all_versions(self, project_name): """Find all available versions for project_name This checks index_urls, find_links and dependency_links All versions found are returned See _link_package_versions for details on which files are accepted """ index_locations = self._get_index_urls_locations(project_name) index_file_loc, index_url_loc = self._sort_locations(index_locations) fl_file_loc, fl_url_loc = self._sort_locations( self.find_links, expand_dir=True) dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links) file_locations = ( Link(url) for url in itertools.chain( index_file_loc, fl_file_loc, dep_file_loc) ) # We trust every url that the user has given us whether it was given # via --index-url or --find-links # We explicitly do not trust links that came from dependency_links # We want to filter out any thing which does not have a secure origin. url_locations = [ link for link in itertools.chain( (Link(url, trusted=True) for url in index_url_loc), (Link(url, trusted=True) for url in fl_url_loc), (Link(url) for url in dep_url_loc), ) if self._validate_secure_origin(logger, link) ] logger.debug('%d location(s) to search for versions of %s:', len(url_locations), project_name) for location in url_locations: logger.debug('* %s', location) canonical_name = pkg_resources.safe_name(project_name).lower() formats = fmt_ctl_formats(self.format_control, canonical_name) search = Search(project_name.lower(), canonical_name, formats) find_links_versions = self._package_versions( # We trust every directly linked archive in find_links (Link(url, '-f', trusted=True) for url in self.find_links), search ) page_versions = [] for page in self._get_pages(url_locations, project_name): logger.debug('Analyzing links from page %s', page.url) with indent_log(): page_versions.extend( self._package_versions(page.links, search) ) dependency_versions = self._package_versions( (Link(url) for url in self.dependency_links), search ) if dependency_versions: logger.debug( 'dependency_links found: %s', ', '.join([ version.location.url for version in dependency_versions ]) ) file_versions = self._package_versions(file_locations, search) if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.location.url) for candidate in file_versions ]) ) # This is an intentional priority ordering return ( file_versions + find_links_versions + page_versions + dependency_versions )
def normalize_package(name): return safe_name(name).lower()
def do_update( project: Project, dev: bool = False, sections: Sequence[str] = (), default: bool = True, strategy: str = "reuse", save: str = "compatible", unconstrained: bool = False, packages: Sequence[str] = (), ) -> None: """Update specified packages or all packages :param project: The project instance :param dev: whether to update dev dependencies :param sections: update speicified sections :param default: update default :param strategy: update strategy (reuse/eager) :param save: save strategy (compatible/exact/wildcard) :param unconstrained: ignore version constraint :param packages: specified packages to update :return: None """ check_project_file(project) if len(packages) > 0 and (len(sections) > 1 or not default): raise PdmUsageError( "packages argument can't be used together with multple -s or --no-default." ) if not packages: if unconstrained: raise PdmUsageError( "--unconstrained must be used with package names given.") # pdm update with no packages given, same as 'lock' + 'sync' do_lock(project) do_sync(project, sections, dev, default, clean=False) return section = sections[0] if sections else ("dev" if dev else "default") all_dependencies = project.all_dependencies dependencies = all_dependencies[section] updated_deps = {} tracked_names = set() for name in packages: matched_name = next( filter( lambda k: safe_name(strip_extras(k)[0]).lower() == safe_name( name).lower(), dependencies.keys(), ), None, ) if not matched_name: raise ProjectError("{} does not exist in {} dependencies.".format( stream.green(name, bold=True), section)) if unconstrained: dependencies[matched_name].specifier = get_specifier("") tracked_names.add(matched_name) updated_deps[matched_name] = dependencies[matched_name] stream.echo("Updating packages: {}.".format(", ".join( stream.green(v, bold=True) for v in tracked_names))) reqs = [r for deps in all_dependencies.values() for r in deps.values()] resolved = do_lock(project, strategy, tracked_names, reqs) do_sync(project, sections=(section, ), default=False, clean=False) if unconstrained: # Need to update version constraints save_version_specifiers(updated_deps, resolved, save) project.add_dependencies(updated_deps) lockfile = project.lockfile project.write_lockfile(lockfile, False)
def normalize_package(name: str) -> str: return safe_name(name).lower()
def name(self) -> Optional[str]: if self.req is None: return None return pkg_resources.safe_name(self.req.name)
def _link_package_versions(self, link, search): """Return an InstallationCandidate or None""" platform = get_platform() version = None if link.egg_fragment: egg_info = link.egg_fragment ext = link.ext else: egg_info, ext = link.splitext() if not ext: self._log_skipped_link(link, 'not a file') return if ext not in SUPPORTED_EXTENSIONS: self._log_skipped_link( link, 'unsupported archive format: %s' % ext) return if "binary" not in search.formats and ext == wheel_ext: self._log_skipped_link( link, 'No binaries permitted for %s' % search.supplied) return if "macosx10" in link.path and ext == '.zip': self._log_skipped_link(link, 'macosx10 one') return if ext == wheel_ext: try: wheel = Wheel(link.filename) except InvalidWheelFilename: self._log_skipped_link(link, 'invalid wheel filename') return if (pkg_resources.safe_name(wheel.name).lower() != search.canonical): self._log_skipped_link( link, 'wrong project name (not %s)' % search.supplied) return if not wheel.supported(): self._log_skipped_link( link, 'it is not compatible with this Python') return # This is a dirty hack to prevent installing Binary Wheels from # PyPI unless it is a Windows or Mac Binary Wheel. This is # paired with a change to PyPI disabling uploads for the # same. Once we have a mechanism for enabling support for # binary wheels on linux that deals with the inherent problems # of binary distribution this can be removed. comes_from = getattr(link, "comes_from", None) if ( ( not platform.startswith('win') and not platform.startswith('macosx') and not platform == 'cli' ) and comes_from is not None and urllib_parse.urlparse( comes_from.url ).netloc.endswith(PyPI.netloc)): if not wheel.supported(tags=supported_tags_noarch): self._log_skipped_link( link, "it is a pypi-hosted binary " "Wheel on an unsupported platform", ) return version = wheel.version # This should be up by the search.ok_binary check, but see issue 2700. if "source" not in search.formats and ext != wheel_ext: self._log_skipped_link( link, 'No sources permitted for %s' % search.supplied) return if not version: version = egg_info_matches(egg_info, search.supplied, link) if version is None: self._log_skipped_link( link, 'wrong project name (not %s)' % search.supplied) return if (link.internal is not None and not link.internal and not normalize_name(search.supplied).lower() in self.allow_external and not self.allow_all_external): # We have a link that we are sure is external, so we should skip # it unless we are allowing externals self._log_skipped_link(link, 'it is externally hosted') self.need_warn_external = True return if (link.verifiable is not None and not link.verifiable and not (normalize_name(search.supplied).lower() in self.allow_unverified)): # We have a link that we are sure we cannot verify its integrity, # so we should skip it unless we are allowing unsafe installs # for this requirement. self._log_skipped_link( link, 'it is an insecure and unverifiable file') self.need_warn_unverified = True return match = self._py_version_re.search(version) if match: version = version[:match.start()] py_version = match.group(1) if py_version != sys.version[:3]: self._log_skipped_link( link, 'Python version is incorrect') return logger.debug('Found link %s, version: %s', link, version) return InstallationCandidate(search.supplied, version, link)
def _link_package_versions(self, link, search_name): """ Return an iterable of triples (pkg_resources_version_key, link, python_version) that can be extracted from the given link. Meant to be overridden by subclasses, not called by clients. """ platform = get_platform() version = None if link.egg_fragment: egg_info = link.egg_fragment else: egg_info, ext = link.splitext() if not ext: if link not in self.logged_links: logger.debug('Skipping link %s; not a file', link) self.logged_links.add(link) return if egg_info.endswith('.tar'): # Special double-extension case: egg_info = egg_info[:-4] ext = '.tar' + ext if ext not in self._known_extensions(): if link not in self.logged_links: logger.debug( 'Skipping link %s; unknown archive format: %s', link, ext, ) self.logged_links.add(link) return if "macosx10" in link.path and ext == '.zip': if link not in self.logged_links: logger.debug('Skipping link %s; macosx10 one', link) self.logged_links.add(link) return if ext == wheel_ext: try: wheel = Wheel(link.filename) except InvalidWheelFilename: logger.debug( 'Skipping %s because the wheel filename is invalid', link) return if (pkg_resources.safe_name(wheel.name).lower() != pkg_resources.safe_name(search_name).lower()): logger.debug( 'Skipping link %s; wrong project name (not %s)', link, search_name, ) return if not wheel.supported(): logger.debug( 'Skipping %s because it is not compatible with this ' 'Python', link, ) return # This is a dirty hack to prevent installing Binary Wheels from # PyPI unless it is a Windows or Mac Binary Wheel. This is # paired with a change to PyPI disabling uploads for the # same. Once we have a mechanism for enabling support for # binary wheels on linux that deals with the inherent problems # of binary distribution this can be removed. comes_from = getattr(link, "comes_from", None) if ((not platform.startswith('win') and not platform.startswith('macosx') and not platform == 'cli') and comes_from is not None and urllib_parse.urlparse( comes_from.url).netloc.endswith(PyPI.netloc)): if not wheel.supported(tags=supported_tags_noarch): logger.debug( "Skipping %s because it is a pypi-hosted binary " "Wheel on an unsupported platform", link, ) return version = wheel.version if not version: version = self._egg_info_matches(egg_info, search_name, link) if version is None: logger.debug( 'Skipping link %s; wrong project name (not %s)', link, search_name, ) return if (link.internal is not None and not link.internal and not normalize_name(search_name).lower() in self.allow_external and not self.allow_all_external): # We have a link that we are sure is external, so we should skip # it unless we are allowing externals logger.debug("Skipping %s because it is externally hosted.", link) self.need_warn_external = True return if (link.verifiable is not None and not link.verifiable and not (normalize_name(search_name).lower() in self.allow_unverified)): # We have a link that we are sure we cannot verify its integrity, # so we should skip it unless we are allowing unsafe installs # for this requirement. logger.debug( "Skipping %s because it is an insecure and unverifiable file.", link, ) self.need_warn_unverified = True return match = self._py_version_re.search(version) if match: version = version[:match.start()] py_version = match.group(1) if py_version != sys.version[:3]: logger.debug('Skipping %s because Python version is incorrect', link) return logger.debug('Found link %s, version: %s', link, version) return InstallationCandidate(search_name, version, link)
def _find_all_versions(self, project_name): """Find all available versions for project_name This checks index_urls, find_links and dependency_links All versions found are returned See _link_package_versions for details on which files are accepted """ index_locations = self._get_index_urls_locations(project_name) index_file_loc, index_url_loc = self._sort_locations(index_locations) fl_file_loc, fl_url_loc = self._sort_locations(self.find_links, expand_dir=True) dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links) file_locations = (Link(url) for url in itertools.chain( index_file_loc, fl_file_loc, dep_file_loc)) # We trust every url that the user has given us whether it was given # via --index-url or --find-links # We explicitly do not trust links that came from dependency_links # We want to filter out any thing which does not have a secure origin. url_locations = [ link for link in itertools.chain( (Link(url, trusted=True) for url in index_url_loc), (Link(url, trusted=True) for url in fl_url_loc), (Link(url) for url in dep_url_loc), ) if self._validate_secure_origin(logger, link) ] logger.debug('%d location(s) to search for versions of %s:', len(url_locations), project_name) for location in url_locations: logger.debug('* %s', location) canonical_name = pkg_resources.safe_name(project_name).lower() formats = fmt_ctl_formats(self.format_control, canonical_name) search = Search(project_name.lower(), canonical_name, formats) find_links_versions = self._package_versions( # We trust every directly linked archive in find_links (Link(url, '-f', trusted=True) for url in self.find_links), search) page_versions = [] for page in self._get_pages(url_locations, project_name): logger.debug('Analyzing links from page %s', page.url) with indent_log(): page_versions.extend(self._package_versions( page.links, search)) dependency_versions = self._package_versions( (Link(url) for url in self.dependency_links), search) if dependency_versions: logger.debug( 'dependency_links found: %s', ', '.join( [version.location.url for version in dependency_versions])) file_versions = self._package_versions(file_locations, search) if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.location.url) for candidate in file_versions ])) # This is an intentional priority ordering return (file_versions + find_links_versions + page_versions + dependency_versions)
def add_dependencies(self, name, version, requirements): pypi_data = self._pypi_data[safe_name(name)][version] pypi_data.setdefault("dependencies", []).extend(requirements)
def name(self): # type: () -> Optional[str] if self.req is None: return None return native_str(pkg_resources.safe_name(self.req.name))
def build(self, autobuilding=False): """Build wheels. :param unpack: If True, replace the sdist we built from the with the newly built wheel, in preparation for installation. :return: True if all the wheels built correctly. """ assert self._wheel_dir or (autobuilding and self._cache_root) # unpack sdists and constructs req set self.requirement_set.prepare_files(self.finder) reqset = self.requirement_set.requirements.values() buildset = [] for req in reqset: if req.is_wheel: if not autobuilding: logger.info( 'Skipping %s, due to already being wheel.', req.name) elif req.editable: if not autobuilding: logger.info( 'Skipping bdist_wheel for %s, due to being editable', req.name) elif autobuilding and req.link and not req.link.is_artifact: pass elif autobuilding and not req.source_dir: pass else: if autobuilding: link = req.link base, ext = link.splitext() if pip.index.egg_info_matches(base, None, link) is None: # Doesn't look like a package - don't autobuild a wheel # because we'll have no way to lookup the result sanely continue if "binary" not in pip.index.fmt_ctl_formats( self.finder.format_control, pkg_resources.safe_name(req.name).lower()): logger.info( "Skipping bdist_wheel for %s, due to binaries " "being disabled for it.", req.name) continue buildset.append(req) if not buildset: return True # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for req in buildset]), ) with indent_log(): build_success, build_failure = [], [] for req in buildset: if autobuilding: output_dir = _cache_for_link(self._cache_root, req.link) ensure_dir(output_dir) else: output_dir = self._wheel_dir wheel_file = self._build_one(req, output_dir) if wheel_file: build_success.append(req) if autobuilding: # XXX: This is mildly duplicative with prepare_files, # but not close enough to pull out to a single common # method. # The code below assumes temporary source dirs - # prevent it doing bad things. if req.source_dir and not os.path.exists(os.path.join( req.source_dir, PIP_DELETE_MARKER_FILENAME)): raise AssertionError( "bad source dir - missing marker") # Delete the source we built the wheel from req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. req.source_dir = req.build_location( self.requirement_set.build_dir) # Update the link for this. req.link = pip.index.Link( path_to_url(wheel_file), trusted=True) assert req.link.is_wheel # extract the wheel into the dir unpack_url( req.link, req.source_dir, None, False, session=self.requirement_set.session) else: build_failure.append(req) # notify success/failure if build_success: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_success]), ) if build_failure: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) # Return True if all builds were successful return len(build_failure) == 0
def normalize_name(name: str) -> str: return safe_name(name).lower() # type: ignore
def name(self): if self.req is None: return None return native_str(pkg_resources.safe_name(self.req.name))
def add_candidate(self, candidate): key = safe_name(candidate.name).lower() self._data[key] = Distribution(key, candidate.version)
def build(self, autobuilding=False): """Build wheels. :param unpack: If True, replace the sdist we built from the with the newly built wheel, in preparation for installation. :return: True if all the wheels built correctly. """ assert self._wheel_dir or (autobuilding and self._cache_root) # unpack sdists and constructs req set self.requirement_set.prepare_files(self.finder) reqset = self.requirement_set.requirements.values() buildset = [] for req in reqset: if req.constraint: continue if req.is_wheel: if not autobuilding: logger.info('Skipping %s, due to already being wheel.', req.name) elif req.editable: if not autobuilding: logger.info( 'Skipping bdist_wheel for %s, due to being editable', req.name) elif autobuilding and req.link and not req.link.is_artifact: pass elif autobuilding and not req.source_dir: pass else: if autobuilding: link = req.link base, ext = link.splitext() if pip.index.egg_info_matches(base, None, link) is None: # Doesn't look like a package - don't autobuild a wheel # because we'll have no way to lookup the result sanely continue if "binary" not in pip.index.fmt_ctl_formats( self.finder.format_control, pkg_resources.safe_name(req.name).lower()): logger.info( "Skipping bdist_wheel for %s, due to binaries " "being disabled for it.", req.name) continue buildset.append(req) if not buildset: return True # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for req in buildset]), ) with indent_log(): build_success, build_failure = [], [] for req in buildset: if autobuilding: output_dir = _cache_for_link(self._cache_root, req.link) ensure_dir(output_dir) else: output_dir = self._wheel_dir wheel_file = self._build_one(req, output_dir) if wheel_file: build_success.append(req) if autobuilding: # XXX: This is mildly duplicative with prepare_files, # but not close enough to pull out to a single common # method. # The code below assumes temporary source dirs - # prevent it doing bad things. if req.source_dir and not os.path.exists( os.path.join(req.source_dir, PIP_DELETE_MARKER_FILENAME)): raise AssertionError( "bad source dir - missing marker") # Delete the source we built the wheel from req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. req.source_dir = req.build_location( self.requirement_set.build_dir) # Update the link for this. req.link = pip.index.Link(path_to_url(wheel_file), trusted=True) assert req.link.is_wheel # extract the wheel into the dir unpack_url(req.link, req.source_dir, None, False, session=self.requirement_set.session) else: build_failure.append(req) # notify success/failure if build_success: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_success]), ) if build_failure: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) # Return True if all builds were successful return len(build_failure) == 0
def add_candidate(self, name, version, requires_python=""): pypi_data = self._pypi_data.setdefault(safe_name(name), {}).setdefault(version, {}) pypi_data["requires_python"] = requires_python
def install(candidate): dependencies = repository.get_dependencies(candidate)[0] key = safe_name(candidate.name).lower() dist = Distribution(key, candidate.version) dist.dependencies = dependencies rv.add_distribution(dist)