def package_or_url_from_pep508(requirement: Requirement,
remove_version_specifiers=False) -> str:
requirement.marker = None
requirement.name = canonicalize_name(requirement.name)
if remove_version_specifiers:
requirement.specifier = SpecifierSet("")
return str(requirement)
async def add_requirement(
self, requirement: Union[str, Requirement], ctx, transaction
):
"""Add a requirement to the transaction.
See PEP 508 for a description of the requirements.
https://www.python.org/dev/peps/pep-0508
"""
if isinstance(requirement, Requirement):
req = requirement
elif requirement.endswith(".whl"):
# custom download location
name, wheel, version = _parse_wheel_url(requirement)
name = name.lower()
await self.add_wheel(name, wheel, version, (), ctx, transaction)
return
else:
req = Requirement(requirement)
req.name = req.name.lower()
# If there's a Pyodide package that matches the version constraint, use
# the Pyodide package instead of the one on PyPI
if (
req.name in self.builtin_packages
and self.builtin_packages[req.name]["version"] in req.specifier
):
version = self.builtin_packages[req.name]["version"]
transaction["pyodide_packages"].append((req.name, version))
return
if req.marker:
# handle environment markers
# https://www.python.org/dev/peps/pep-0508/#environment-markers
if not req.marker.evaluate(ctx):
return
# Is some version of this package is already installed?
if req.name in transaction["locked"]:
ver = transaction["locked"][req.name]
if ver in req.specifier:
# installed version matches, nothing to do
return
else:
raise ValueError(
f"Requested '{requirement}', "
f"but {req.name}=={ver} is already installed"
)
metadata = await _get_pypi_json(req.name)
wheel, ver = self.find_wheel(metadata, req)
await self.add_wheel(req.name, wheel, ver, req.extras, ctx, transaction)
def _parse_package_lines(package_lines: List[str]) -> Set[Requirement]:
"""Parse a requirement line
ignores commented line
and inline comments
"""
filtered_requirements: Set[Requirement] = set()
for package_line in package_lines:
package_line = package_line.strip()
if not package_line or package_line.startswith("#"):
continue
package_line, *_ = package_line.split("#", maxsplit=1)
requirement = Requirement(package_line.strip())
requirement.name = canonicalize_name(requirement.name)
requirement.specifier.prereleases = True
filtered_requirements.add(requirement)
return filtered_requirements
def fix_package_name(package_or_url: str, package: str) -> str:
try:
package_req = Requirement(package_or_url)
except InvalidRequirement:
# not a valid PEP508 package specification
return package_or_url
if canonicalize_name(package_req.name) != canonicalize_name(package):
logging.warning(
textwrap.fill(
f"{hazard} Name supplied in package specifier was "
f"{package_req.name!r} but package found has name "
f"{package!r}. Using {package!r}.",
subsequent_indent=" ",
))
package_req.name = package
return str(package_req)
def parse_requirement(requirement: str, *,
constraint: bool = False, weak: bool = False) -> Union[Package, str]:
"""Parse a single requirement.
It will handle
- requirements handled by packaging.requirements.Requirement
- command-line options, which are returned as-is.
Anything not handled is returned as a string (with a warning).
"""
try:
req = Requirement(requirement)
except ValueError:
if not requirement.startswith('-'):
warnings.warn('Requirement {} could not be parsed and is not an option'
.format(requirement))
return requirement
req.name = canonicalize_name(req.name)
return Package(req, constraint=constraint, weak=weak)
def _determine_filtered_package_requirements(self) -> List[Requirement]:
"""
Parse the configuration file for [blocklist]packages
Returns
-------
list of packaging.requirements.Requirement
For all PEP440 package specifiers
"""
filtered_requirements: Set[Requirement] = set()
try:
lines = self.blocklist["packages"]
package_lines = lines.split("\n")
except KeyError:
package_lines = []
for package_line in package_lines:
package_line = package_line.strip()
if not package_line or package_line.startswith("#"):
continue
requirement = Requirement(package_line)
requirement.name = canonicalize_name(requirement.name)
requirement.specifier.prereleases = True
filtered_requirements.add(requirement)
return list(filtered_requirements)
async def add_requirement(self, requirement: str | Requirement, ctx,
transaction):
"""Add a requirement to the transaction.
See PEP 508 for a description of the requirements.
https://www.python.org/dev/peps/pep-0508
"""
if isinstance(requirement, Requirement):
req = requirement
elif requirement.endswith(".whl"):
# custom download location
name, wheel, version = _parse_wheel_url(requirement)
name = name.lower()
if not _is_pure_python_wheel(wheel["filename"]):
raise ValueError(
f"'{wheel['filename']}' is not a pure Python 3 wheel")
await self.add_wheel(name, wheel, version, (), ctx, transaction)
return
else:
req = Requirement(requirement)
req.name = req.name.lower()
# If there's a Pyodide package that matches the version constraint, use
# the Pyodide package instead of the one on PyPI
if (req.name in BUILTIN_PACKAGES
and BUILTIN_PACKAGES[req.name]["version"] in req.specifier):
version = BUILTIN_PACKAGES[req.name]["version"]
transaction["pyodide_packages"].append(
PackageMetadata(name=req.name,
version=version,
source="pyodide"))
return
if req.marker:
# handle environment markers
# https://www.python.org/dev/peps/pep-0508/#environment-markers
if not req.marker.evaluate(ctx):
return
# Is some version of this package is already installed?
if req.name in transaction["locked"]:
ver = transaction["locked"][req.name].version
if ver in req.specifier:
# installed version matches, nothing to do
return
else:
raise ValueError(f"Requested '{requirement}', "
f"but {req.name}=={ver} is already installed")
metadata = await _get_pypi_json(req.name)
maybe_wheel, maybe_ver = self.find_wheel(metadata, req)
if maybe_wheel is None or maybe_ver is None:
if transaction["keep_going"]:
transaction["failed"].append(req)
else:
raise ValueError(
f"Couldn't find a pure Python 3 wheel for '{req}'. "
"You can use `micropip.install(..., keep_going=True)` to get a list of all packages with missing wheels."
)
else:
await self.add_wheel(req.name, maybe_wheel, maybe_ver, req.extras,
ctx, transaction)
def run_packages(
specs,
file,
algorithm,
python_versions=None,
verbose=False,
include_prereleases=False,
dry_run=False,
previous_versions=None,
interactive=False,
synchronous=False,
index_url=DEFAULT_INDEX_URL,
):
assert index_url
assert isinstance(specs, list), type(specs)
all_new_lines = []
first_interactive = True
yes_to_all = False
lookup_memory = {}
if not synchronous and len(specs) > 1:
pre_download_packages(lookup_memory,
specs,
verbose=verbose,
index_url=index_url)
for spec in specs:
package, version, restriction = _explode_package_spec(spec)
# It's important to keep a track of what the package was called before
# so that if we have to amend the requirements file, we know what to
# look for before.
previous_name = package
# The 'previous_versions' dict is based on the old names. So figure
# out what the previous version was *before* the new/"correct" name
# is figured out.
previous_version = previous_versions.get(
package) if previous_versions else None
req = Requirement(package)
data = get_package_hashes(
package=req.name,
version=version,
verbose=verbose,
python_versions=python_versions,
algorithm=algorithm,
include_prereleases=include_prereleases,
lookup_memory=lookup_memory,
index_url=index_url,
)
package = data["package"]
# We need to keep this `req` instance for the sake of turning it into a string
# the correct way. But, the name might actually be wrong. Suppose the user
# asked for "Django" but on PyPI it's actually called "django", then we want
# correct that.
# We do that by modifying only the `name` part of the `Requirement` instance.
req.name = package
if previous_versions is None:
# Need to be smart here. It's a little counter-intuitive.
# If no previous_versions was supplied that has an implied the fact;
# the user was explicit about what they want to install.
# The name it was called in the old requirements file doesn't matter.
previous_name = package
new_version_specifier = SpecifierSet("=={}".format(data["version"]))
if previous_version:
# We have some form of previous version and a new version.
# If they' already equal, just skip this one.
if previous_version == new_version_specifier:
continue
if interactive:
try:
response = interactive_upgrade_request(
package,
previous_version,
new_version_specifier,
print_header=first_interactive,
force_yes=yes_to_all,
)
first_interactive = False
if response == "NO":
continue
elif response == "ALL":
# If you ever answer "all" to the update question, we don't want
# stop showing the interactive prompt but we don't need to
# ask any questions any more. This way, you get to see the
# upgrades that are going to happen.
yes_to_all = True
elif response == "QUIT":
return 1
except KeyboardInterrupt:
return 1
maybe_restriction = "" if not restriction else "; {0}".format(
restriction)
new_lines = "{0}=={1}{2} \\\n".format(req, data["version"],
maybe_restriction)
padding = " " * 4
for i, release in enumerate(
sorted(data["hashes"], key=lambda r: r["hash"])):
new_lines += "{0}--hash={1}:{2}".format(padding, algorithm,
release["hash"])
if i != len(data["hashes"]) - 1:
new_lines += " \\"
new_lines += "\n"
all_new_lines.append((package, previous_name, new_lines))
if not all_new_lines:
# This can happen if you use 'interactive' and said no to everything or
# if every single package you listed already has the latest version.
return 0
with open(file) as f:
old_requirements = f.read()
requirements = amend_requirements_content(old_requirements, all_new_lines)
if dry_run:
if verbose:
_verbose("Dry run, not editing ", file)
print("".join(
difflib.unified_diff(
old_requirements.splitlines(True),
requirements.splitlines(True),
fromfile="Old",
tofile="New",
)))
else:
with open(file, "w") as f:
f.write(requirements)
if verbose:
_verbose("Editing", file)
return 0
async def add_requirement_inner(
self,
req: Requirement,
) -> None:
"""Add a requirement to the transaction.
See PEP 508 for a description of the requirements.
https://www.python.org/dev/peps/pep-0508
"""
for e in req.extras:
self.ctx_extras.append({"extra": e})
if self.pre:
req.specifier.prereleases = True
if req.marker:
# handle environment markers
# https://www.python.org/dev/peps/pep-0508/#environment-markers
# For a requirement being installed as part of an optional feature
# via the extra specifier, the evaluation of the marker requires
# the extra key in self.ctx to have the value specified in the
# primary requirement.
# The req.extras attribute is only set for the primary requirement
# and hence has to be available during the evaluation of the
# dependencies. Thus, we use the self.ctx_extras attribute above to
# store all the extra values we come across during the transaction and
# attempt the marker evaluation for all of these values. If any of the
# evaluations return true we include the dependency.
def eval_marker(e: dict[str, str]) -> bool:
self.ctx.update(e)
# need the assertion here to make mypy happy:
# https://github.com/python/mypy/issues/4805
assert req.marker is not None
return req.marker.evaluate(self.ctx)
self.ctx.update({"extra": ""})
# The current package may have been brought into the transaction
# without any of the optional requirement specification, but has
# another marker, such as implementation_name. In this scenario,
# self.ctx_extras is empty and hence the eval_marker() function
# will not be called at all.
if not req.marker.evaluate(self.ctx) and not any(
[eval_marker(e) for e in self.ctx_extras]):
return
# Is some version of this package is already installed?
req.name = canonicalize_name(req.name)
if self.check_version_satisfied(req):
return
# If there's a Pyodide package that matches the version constraint, use
# the Pyodide package instead of the one on PyPI
if req.name in BUILTIN_PACKAGES and req.specifier.contains(
BUILTIN_PACKAGES[req.name]["version"], prereleases=True):
version = BUILTIN_PACKAGES[req.name]["version"]
self.pyodide_packages.append(
PackageMetadata(name=req.name,
version=str(version),
source="pyodide"))
return
metadata = await _get_pypi_json(req.name, self.fetch_kwargs)
try:
wheel = find_wheel(metadata, req)
except ValueError:
self.failed.append(req)
if not self.keep_going:
raise
else:
return
if self.check_version_satisfied(req):
# Maybe while we were downloading pypi_json some other branch
# installed the wheel?
return
await self.add_wheel(wheel, req.extras)
