def package_or_url_from_pep508(requirement: Requirement, remove_version_specifiers=False) -> str: requirement.marker = None requirement.name = canonicalize_name(requirement.name) if remove_version_specifiers: requirement.specifier = SpecifierSet("") return str(requirement)
async def add_requirement( self, requirement: Union[str, Requirement], ctx, transaction ): """Add a requirement to the transaction. See PEP 508 for a description of the requirements. https://www.python.org/dev/peps/pep-0508 """ if isinstance(requirement, Requirement): req = requirement elif requirement.endswith(".whl"): # custom download location name, wheel, version = _parse_wheel_url(requirement) name = name.lower() await self.add_wheel(name, wheel, version, (), ctx, transaction) return else: req = Requirement(requirement) req.name = req.name.lower() # If there's a Pyodide package that matches the version constraint, use # the Pyodide package instead of the one on PyPI if ( req.name in self.builtin_packages and self.builtin_packages[req.name]["version"] in req.specifier ): version = self.builtin_packages[req.name]["version"] transaction["pyodide_packages"].append((req.name, version)) return if req.marker: # handle environment markers # https://www.python.org/dev/peps/pep-0508/#environment-markers if not req.marker.evaluate(ctx): return # Is some version of this package is already installed? if req.name in transaction["locked"]: ver = transaction["locked"][req.name] if ver in req.specifier: # installed version matches, nothing to do return else: raise ValueError( f"Requested '{requirement}', " f"but {req.name}=={ver} is already installed" ) metadata = await _get_pypi_json(req.name) wheel, ver = self.find_wheel(metadata, req) await self.add_wheel(req.name, wheel, ver, req.extras, ctx, transaction)
def _parse_package_lines(package_lines: List[str]) -> Set[Requirement]: """Parse a requirement line ignores commented line and inline comments """ filtered_requirements: Set[Requirement] = set() for package_line in package_lines: package_line = package_line.strip() if not package_line or package_line.startswith("#"): continue package_line, *_ = package_line.split("#", maxsplit=1) requirement = Requirement(package_line.strip()) requirement.name = canonicalize_name(requirement.name) requirement.specifier.prereleases = True filtered_requirements.add(requirement) return filtered_requirements
def fix_package_name(package_or_url: str, package: str) -> str: try: package_req = Requirement(package_or_url) except InvalidRequirement: # not a valid PEP508 package specification return package_or_url if canonicalize_name(package_req.name) != canonicalize_name(package): logging.warning( textwrap.fill( f"{hazard} Name supplied in package specifier was " f"{package_req.name!r} but package found has name " f"{package!r}. Using {package!r}.", subsequent_indent=" ", )) package_req.name = package return str(package_req)
def parse_requirement(requirement: str, *, constraint: bool = False, weak: bool = False) -> Union[Package, str]: """Parse a single requirement. It will handle - requirements handled by packaging.requirements.Requirement - command-line options, which are returned as-is. Anything not handled is returned as a string (with a warning). """ try: req = Requirement(requirement) except ValueError: if not requirement.startswith('-'): warnings.warn('Requirement {} could not be parsed and is not an option' .format(requirement)) return requirement req.name = canonicalize_name(req.name) return Package(req, constraint=constraint, weak=weak)
def _determine_filtered_package_requirements(self) -> List[Requirement]: """ Parse the configuration file for [blocklist]packages Returns ------- list of packaging.requirements.Requirement For all PEP440 package specifiers """ filtered_requirements: Set[Requirement] = set() try: lines = self.blocklist["packages"] package_lines = lines.split("\n") except KeyError: package_lines = [] for package_line in package_lines: package_line = package_line.strip() if not package_line or package_line.startswith("#"): continue requirement = Requirement(package_line) requirement.name = canonicalize_name(requirement.name) requirement.specifier.prereleases = True filtered_requirements.add(requirement) return list(filtered_requirements)
async def add_requirement(self, requirement: str | Requirement, ctx, transaction): """Add a requirement to the transaction. See PEP 508 for a description of the requirements. https://www.python.org/dev/peps/pep-0508 """ if isinstance(requirement, Requirement): req = requirement elif requirement.endswith(".whl"): # custom download location name, wheel, version = _parse_wheel_url(requirement) name = name.lower() if not _is_pure_python_wheel(wheel["filename"]): raise ValueError( f"'{wheel['filename']}' is not a pure Python 3 wheel") await self.add_wheel(name, wheel, version, (), ctx, transaction) return else: req = Requirement(requirement) req.name = req.name.lower() # If there's a Pyodide package that matches the version constraint, use # the Pyodide package instead of the one on PyPI if (req.name in BUILTIN_PACKAGES and BUILTIN_PACKAGES[req.name]["version"] in req.specifier): version = BUILTIN_PACKAGES[req.name]["version"] transaction["pyodide_packages"].append( PackageMetadata(name=req.name, version=version, source="pyodide")) return if req.marker: # handle environment markers # https://www.python.org/dev/peps/pep-0508/#environment-markers if not req.marker.evaluate(ctx): return # Is some version of this package is already installed? if req.name in transaction["locked"]: ver = transaction["locked"][req.name].version if ver in req.specifier: # installed version matches, nothing to do return else: raise ValueError(f"Requested '{requirement}', " f"but {req.name}=={ver} is already installed") metadata = await _get_pypi_json(req.name) maybe_wheel, maybe_ver = self.find_wheel(metadata, req) if maybe_wheel is None or maybe_ver is None: if transaction["keep_going"]: transaction["failed"].append(req) else: raise ValueError( f"Couldn't find a pure Python 3 wheel for '{req}'. " "You can use `micropip.install(..., keep_going=True)` to get a list of all packages with missing wheels." ) else: await self.add_wheel(req.name, maybe_wheel, maybe_ver, req.extras, ctx, transaction)
def run_packages( specs, file, algorithm, python_versions=None, verbose=False, include_prereleases=False, dry_run=False, previous_versions=None, interactive=False, synchronous=False, index_url=DEFAULT_INDEX_URL, ): assert index_url assert isinstance(specs, list), type(specs) all_new_lines = [] first_interactive = True yes_to_all = False lookup_memory = {} if not synchronous and len(specs) > 1: pre_download_packages(lookup_memory, specs, verbose=verbose, index_url=index_url) for spec in specs: package, version, restriction = _explode_package_spec(spec) # It's important to keep a track of what the package was called before # so that if we have to amend the requirements file, we know what to # look for before. previous_name = package # The 'previous_versions' dict is based on the old names. So figure # out what the previous version was *before* the new/"correct" name # is figured out. previous_version = previous_versions.get( package) if previous_versions else None req = Requirement(package) data = get_package_hashes( package=req.name, version=version, verbose=verbose, python_versions=python_versions, algorithm=algorithm, include_prereleases=include_prereleases, lookup_memory=lookup_memory, index_url=index_url, ) package = data["package"] # We need to keep this `req` instance for the sake of turning it into a string # the correct way. But, the name might actually be wrong. Suppose the user # asked for "Django" but on PyPI it's actually called "django", then we want # correct that. # We do that by modifying only the `name` part of the `Requirement` instance. req.name = package if previous_versions is None: # Need to be smart here. It's a little counter-intuitive. # If no previous_versions was supplied that has an implied the fact; # the user was explicit about what they want to install. # The name it was called in the old requirements file doesn't matter. previous_name = package new_version_specifier = SpecifierSet("=={}".format(data["version"])) if previous_version: # We have some form of previous version and a new version. # If they' already equal, just skip this one. if previous_version == new_version_specifier: continue if interactive: try: response = interactive_upgrade_request( package, previous_version, new_version_specifier, print_header=first_interactive, force_yes=yes_to_all, ) first_interactive = False if response == "NO": continue elif response == "ALL": # If you ever answer "all" to the update question, we don't want # stop showing the interactive prompt but we don't need to # ask any questions any more. This way, you get to see the # upgrades that are going to happen. yes_to_all = True elif response == "QUIT": return 1 except KeyboardInterrupt: return 1 maybe_restriction = "" if not restriction else "; {0}".format( restriction) new_lines = "{0}=={1}{2} \\\n".format(req, data["version"], maybe_restriction) padding = " " * 4 for i, release in enumerate( sorted(data["hashes"], key=lambda r: r["hash"])): new_lines += "{0}--hash={1}:{2}".format(padding, algorithm, release["hash"]) if i != len(data["hashes"]) - 1: new_lines += " \\" new_lines += "\n" all_new_lines.append((package, previous_name, new_lines)) if not all_new_lines: # This can happen if you use 'interactive' and said no to everything or # if every single package you listed already has the latest version. return 0 with open(file) as f: old_requirements = f.read() requirements = amend_requirements_content(old_requirements, all_new_lines) if dry_run: if verbose: _verbose("Dry run, not editing ", file) print("".join( difflib.unified_diff( old_requirements.splitlines(True), requirements.splitlines(True), fromfile="Old", tofile="New", ))) else: with open(file, "w") as f: f.write(requirements) if verbose: _verbose("Editing", file) return 0
async def add_requirement_inner( self, req: Requirement, ) -> None: """Add a requirement to the transaction. See PEP 508 for a description of the requirements. https://www.python.org/dev/peps/pep-0508 """ for e in req.extras: self.ctx_extras.append({"extra": e}) if self.pre: req.specifier.prereleases = True if req.marker: # handle environment markers # https://www.python.org/dev/peps/pep-0508/#environment-markers # For a requirement being installed as part of an optional feature # via the extra specifier, the evaluation of the marker requires # the extra key in self.ctx to have the value specified in the # primary requirement. # The req.extras attribute is only set for the primary requirement # and hence has to be available during the evaluation of the # dependencies. Thus, we use the self.ctx_extras attribute above to # store all the extra values we come across during the transaction and # attempt the marker evaluation for all of these values. If any of the # evaluations return true we include the dependency. def eval_marker(e: dict[str, str]) -> bool: self.ctx.update(e) # need the assertion here to make mypy happy: # https://github.com/python/mypy/issues/4805 assert req.marker is not None return req.marker.evaluate(self.ctx) self.ctx.update({"extra": ""}) # The current package may have been brought into the transaction # without any of the optional requirement specification, but has # another marker, such as implementation_name. In this scenario, # self.ctx_extras is empty and hence the eval_marker() function # will not be called at all. if not req.marker.evaluate(self.ctx) and not any( [eval_marker(e) for e in self.ctx_extras]): return # Is some version of this package is already installed? req.name = canonicalize_name(req.name) if self.check_version_satisfied(req): return # If there's a Pyodide package that matches the version constraint, use # the Pyodide package instead of the one on PyPI if req.name in BUILTIN_PACKAGES and req.specifier.contains( BUILTIN_PACKAGES[req.name]["version"], prereleases=True): version = BUILTIN_PACKAGES[req.name]["version"] self.pyodide_packages.append( PackageMetadata(name=req.name, version=str(version), source="pyodide")) return metadata = await _get_pypi_json(req.name, self.fetch_kwargs) try: wheel = find_wheel(metadata, req) except ValueError: self.failed.append(req) if not self.keep_going: raise else: return if self.check_version_satisfied(req): # Maybe while we were downloading pypi_json some other branch # installed the wheel? return await self.add_wheel(wheel, req.extras)