def test_matches_expected(self):
environment = default_environment()
iver = "{0.major}.{0.minor}.{0.micro}".format(
sys.implementation.version
)
if sys.implementation.version.releaselevel != "final":
iver = "{0}{1}[0]{2}".format(
iver,
sys.implementation.version.releaselevel,
sys.implementation.version.serial,
)
assert environment == {
"implementation_name": sys.implementation.name,
"implementation_version": iver,
"os_name": os.name,
"platform_machine": platform.machine(),
"platform_release": platform.release(),
"platform_system": platform.system(),
"platform_version": platform.version(),
"python_full_version": platform.python_version(),
"platform_python_implementation": platform.python_implementation(),
"python_version": platform.python_version()[:3],
"sys_platform": sys.platform,
}
def test_matches_expected(self):
environment = default_environment()
iver = "{0.major}.{0.minor}.{0.micro}".format(
sys.implementation.version)
if sys.implementation.version.releaselevel != "final":
iver = "{0}{1}[0]{2}".format(
iver,
sys.implementation.version.releaselevel,
sys.implementation.version.serial,
)
assert environment == {
"implementation_name": sys.implementation.name,
"implementation_version": iver,
"os_name": os.name,
"platform_machine": platform.machine(),
"platform_release": platform.release(),
"platform_system": platform.system(),
"platform_version": platform.version(),
"python_full_version": platform.python_version(),
"platform_python_implementation": platform.python_implementation(),
"python_version": platform.python_version()[:3],
"sys_platform": sys.platform,
}
def requires(self):
"""Just the strings (name and spec) for my immediate dependencies. Cheap."""
all_requires = self.metadata.get("requires_dist", [])
if not all_requires:
return []
result = []
if self.env is None:
env_data = default_environment()
else:
env_data = dict(self.env)
for req_str in all_requires:
req = pkg_resources.Requirement.parse(req_str)
req_short, _sep, _marker = str(req).partition(";")
if req.marker is None:
# unconditional dependency
result.append(req_short)
continue
# conditional dependency - must be evaluated in environment context
for extra in [None] + self.extras_requested:
if req.marker.evaluate(dict(env_data, extra=extra)):
self.log.debug("included conditional dep", req=req_str)
result.append(req_short)
break
else:
self.log.debug("dropped conditional dep", req=req_str)
result = sorted(set(result)) # this makes the dep tree deterministic/repeatable
return result
async def gather_requirements(
self,
requirements: str | list[str],
ctx=None,
keep_going: bool = False,
):
ctx = ctx or default_environment()
ctx.setdefault("extra", None)
if isinstance(requirements, str):
requirements = [requirements]
transaction: dict[str, Any] = {
"wheels": [],
"pyodide_packages": [],
"locked": copy.deepcopy(self.installed_packages),
"failed": [],
"keep_going": keep_going,
}
requirement_promises = []
for requirement in requirements:
requirement_promises.append(
self.add_requirement(requirement, ctx, transaction))
await gather(*requirement_promises)
return transaction
def _get_wheel_requirements(metadata, extras_requested):
"""Extract the immediate dependencies from wheel metadata."""
all_requires = metadata.get("requires_dist", [])
if not all_requires:
return []
result = []
env_data = default_environment()
for req_str in all_requires:
req = parse_req(req_str)
req_short, _sep, _marker = str(req).partition(";")
if req.marker is None:
# unconditional dependency
result.append(req_short)
continue
# conditional dependency - must be evaluated in environment context
for extra in [None] + extras_requested:
if req.marker.evaluate(dict(env_data, extra=extra)):
logger.debug("included conditional dep %s", req_str)
result.append(req_short)
break
else:
logger.debug("dropped conditional dep %s", req_str)
result = sorted(
set(result)) # this makes the dep tree deterministic/repeatable
return result
def marker_env(args):
"Get the marker environment"
env = {}
env.update(default_environment())
if args.python_version:
env['python_version'] = args.python_version
env['python_full_version'] = args.python_version
return env
def coverage_init(reg, options):
global DEFAULT_ENVIRONMENT
DEFAULT_ENVIRONMENT.update([
(k.upper(), v)
for k, v in default_environment().items()])
if 'markers' in options:
os.environ.update(DEFAULT_ENVIRONMENT)
def current_platform(self) -> Optional["TargetPlatform"]:
environment_json_string = json.dumps(default_environment())
environment = self._load_default_environment(environment_json_string)
python_version = python_version_from_version_string(
environment["python_version"])
if python_version is None:
return None
else:
return self._target_platform_from_default_environment_string(
environment_json_string, python_version=python_version)
def resolve(requirements, index_urls, python_version, exclude_packages,
transitive, virtualenv):
# type: (List[str], List[str], int, Optional[Set[str]], bool, Optional[str]) -> Dict[str, Any]
"""Resolve given requirements for the given Python version."""
assert python_version in (2, 3), "Unknown Python version"
python_bin = "python3" if python_version == 3 else "python2"
if not virtualenv:
run_command("virtualenv -p " + python_bin + " venv")
python_bin = os.path.join("venv", "bin", python_bin)
run_command("{} -m pip install pipdeptree".format(python_bin))
else:
python_bin = os.path.join(virtualenv, "bin", python_bin)
environment_packages = get_environment_packages(python_bin)
result = {
"tree": [],
"errors": [],
"unparsed": [],
"unresolved": [],
"environment": default_environment(),
"environment_packages": environment_packages,
"platform": sysconfig.get_platform(),
}
all_solvers = []
for index_url in index_urls:
source = Source(index_url)
from .python_solver import PythonReleasesFetcher
all_solvers.append(
PythonSolver(
dependency_parser=PythonDependencyParser(),
releases_fetcher=PythonReleasesFetcher(source=source),
), )
for solver in all_solvers:
solver_result = _do_resolve_index(
python_bin=python_bin,
solver=solver,
all_solvers=all_solvers,
requirements=requirements,
exclude_packages=exclude_packages,
transitive=transitive,
)
result["tree"].extend(solver_result["tree"]) # type: ignore
result["errors"].extend(solver_result["errors"]) # type: ignore
result["unparsed"].extend(solver_result["unparsed"]) # type: ignore
result["unresolved"].extend(
solver_result["unresolved"]) # type: ignore
return result
def test_multidigit_minor_version(self, monkeypatch):
version_info = (3, 10, 0, "final", 0)
monkeypatch.setattr(sys, "version_info", version_info, raising=False)
monkeypatch.setattr(platform, "python_version", lambda: "3.10.0", raising=False)
monkeypatch.setattr(
platform, "python_version_tuple", lambda: ("3", "10", "0"), raising=False
)
environment = default_environment()
assert environment["python_version"] == "3.10"
def get_env_info() -> Dict[str, object]:
"""Public helper to get the same env we pass to the plugin."""
env_info: Dict[str, object] = {}
env_info.update(default_environment())
# Feel free to send PRs that extend this dict:
env_info.update({
'sys_version_info': sys.version_info,
'os_environ': os.environ,
'is_installed': _is_installed,
'package_version': _package_version,
})
return env_info
def test_matches_expected_no_sys_implementation(self):
environment = default_environment()
assert environment == {
"implementation_name": "",
"implementation_version": "0",
"os_name": os.name,
"platform_machine": platform.machine(),
"platform_release": platform.release(),
"platform_system": platform.system(),
"platform_version": platform.version(),
"python_full_version": platform.python_version(),
"platform_python_implementation": platform.python_implementation(),
"python_version": platform.python_version()[:3],
"sys_platform": sys.platform,
}
def test_matches_expected_no_sys_implementation(self):
environment = default_environment()
assert environment == {
"implementation_name": "",
"implementation_version": "0",
"os_name": os.name,
"platform_machine": platform.machine(),
"platform_release": platform.release(),
"platform_system": platform.system(),
"platform_version": platform.version(),
"python_full_version": platform.python_version(),
"platform_python_implementation": platform.python_implementation(),
"python_version": platform.python_version()[:3],
"sys_platform": sys.platform,
}
def main():
# type () -> None
if 'src' not in os.environ:
with open(os.environ['out'], 'wb') as fp:
fp.write(
json.dumps(default_environment(),
indent=4,
ensure_ascii=False,
sort_keys=True).encode('utf-8'))
return
dest = extract_source(os.environ['src'])
change_permissions(dest)
distro = get_distro(dest)
output = {
'metadata': {
'name': u(distro.get_name()),
'version': u(distro.get_version()),
'author': u(distro.get_author()),
'author_email': u(distro.get_author_email()),
'description': u(distro.get_description()),
'license': u(distro.get_license())
},
'requirements': {
'install': [
u(x) for x in req_names(
getattr(distro, 'install_requires', distro.get_requires()))
],
'test': [
u(x)
for x in req_names(getattr(distro, 'tests_require', []) or [])
],
'setup':
[u(x) for x in req_names(getattr(distro, 'setup_requires', []))],
'extras':
req_names_extras(getattr(distro, 'extras_require', {}))
}
}
with open(os.environ['out'], 'wb') as fp:
fp.write(
json.dumps(output, indent=4, ensure_ascii=False,
sort_keys=True).encode('utf-8'))
def __init__(self, req_string, parent=None, index_url=None, env=None, extra_index_url=None):
self.dist_path = None
if req_string.endswith(".whl") and os.path.isfile(req_string):
self.dist_path = req_string
whl = WheelFile(req_string)
whl_name_info = whl.parsed_filename.groupdict()
self.name = canonicalize_name(whl_name_info["name"])
self.specifier = "==" + canonicalize_version(whl_name_info["ver"])
self.req = pkg_resources.Requirement.parse(self.name + self.specifier)
else:
self.req = pkg_resources.Requirement.parse(req_string)
self.name = canonicalize_name(self.req.name)
self.specifier = str(self.req.specifier)
self.extras_requested = sorted(self.req.extras)
log = self.log = logger.bind(dist=str(self.req))
log.info("init johnnydist", parent=parent and str(parent.req))
if parent is not None:
self.index_url = parent.index_url
self.extra_index_url = parent.extra_index_url
self.required_by = [str(parent.req)]
self.env = parent.env
self.env_data = parent.env_data
else:
self.index_url = index_url
self.extra_index_url = extra_index_url
self.required_by = []
self.env = env
if self.env is None:
self.env_data = default_environment()
else:
self.env_data = dict(self.env)
log.debug("target env", **self.env_data)
if self.dist_path is None:
log.debug("fetching best wheel")
with wimpy.working_directory(self.tmp()):
data = pipper.get(
req_string,
index_url=self.index_url,
env=self.env,
extra_index_url=self.extra_index_url,
)
self.dist_path = data["path"]
self.parent = parent
self._recursed = False
def test_matches_expected_deleted_sys_implementation(self, monkeypatch):
monkeypatch.delattr(sys, "implementation")
environment = default_environment()
assert environment == {
"implementation_name": "",
"implementation_version": "0",
"os_name": os.name,
"platform_machine": platform.machine(),
"platform_release": platform.release(),
"platform_system": platform.system(),
"platform_version": platform.version(),
"python_full_version": platform.python_version(),
"platform_python_implementation": platform.python_implementation(),
"python_version": ".".join(platform.python_version_tuple()[:2]),
"sys_platform": sys.platform,
}
async def gather_requirements(self, requirements: Union[str, List[str]], ctx=None):
ctx = ctx or default_environment()
ctx.setdefault("extra", None)
if isinstance(requirements, str):
requirements = [requirements]
transaction: Dict[str, Any] = {
"wheels": [],
"pyodide_packages": [],
"locked": dict(self.installed_packages),
}
requirement_promises = []
for requirement in requirements:
requirement_promises.append(
self.add_requirement(requirement, ctx, transaction)
)
await gather(*requirement_promises)
return transaction
def test_monkeypatch_sys_implementation(self, monkeypatch):
monkeypatch.setattr(
sys, "implementation",
pretend.stub(version=FakeVersionInfo(3, 4, 2, "final", 0),
name="linux"),
raising=False)
environment = default_environment()
assert environment == {
"implementation_name": "linux",
"implementation_version": "3.4.2",
"os_name": os.name,
"platform_machine": platform.machine(),
"platform_release": platform.release(),
"platform_system": platform.system(),
"platform_version": platform.version(),
"python_full_version": platform.python_version(),
"platform_python_implementation": platform.python_implementation(),
"python_version": platform.python_version()[:3],
"sys_platform": sys.platform,
}
def test_monkeypatch_sys_implementation(self, monkeypatch):
monkeypatch.setattr(sys,
"implementation",
pretend.stub(version=FakeVersionInfo(
3, 4, 2, "final", 0),
name="linux"),
raising=False)
environment = default_environment()
assert environment == {
"implementation_name": "linux",
"implementation_version": "3.4.2",
"os_name": os.name,
"platform_machine": platform.machine(),
"platform_release": platform.release(),
"platform_system": platform.system(),
"platform_version": platform.version(),
"python_full_version": platform.python_version(),
"platform_python_implementation": platform.python_implementation(),
"python_version": platform.python_version()[:3],
"sys_platform": sys.platform,
}
def __init__(self,
req_string,
parent=None,
index_url=None,
env=None,
extra_index_url=None):
self.dist_path = None
if req_string.endswith(".whl") and os.path.isfile(req_string):
self.dist_path = req_string
whl = WheelFile(req_string)
whl_name_info = whl.parsed_filename.groupdict()
self.name = canonicalize_name(whl_name_info["name"])
self.specifier = "==" + canonicalize_version(whl_name_info["ver"])
self.req = pkg_resources.Requirement.parse(self.name +
self.specifier)
else:
self.req = pkg_resources.Requirement.parse(req_string)
self.name = canonicalize_name(self.req.name)
self.specifier = str(self.req.specifier)
self.extras_requested = sorted(self.req.extras)
log = self.log = logger.bind(dist=str(self.req))
log.info("init johnnydist", parent=parent and str(parent.req))
if parent is not None:
self.index_url = parent.index_url
self.extra_index_url = parent.extra_index_url
self.required_by = [str(parent.req)]
self.env = parent.env
self.env_data = parent.env_data
else:
self.index_url = index_url
self.extra_index_url = extra_index_url
self.required_by = []
self.env = env
if self.env is None:
self.env_data = default_environment()
else:
self.env_data = dict(self.env)
log.debug("target env", **self.env_data)
if self.dist_path is None:
log.debug("fetching best wheel")
tmpdir = tempfile.mkdtemp()
log.debug("created scratch", tmpdir=tmpdir)
try:
with wimpy.working_directory(tmpdir):
data = pipper.get(
req_string,
index_url=self.index_url,
env=self.env,
extra_index_url=self.extra_index_url,
tmpdir=tmpdir,
)
self.dist_path = data["path"]
# triggers retrieval of any info we need from downloaded dist
self.import_names
self.metadata
finally:
log.debug("removing scratch", tmpdir=tmpdir)
shutil.rmtree(tmpdir)
self.parent = parent
self._recursed = False
import functools
import operator
import os
import tomlkit
from packaging.markers import default_environment
from pdm.models.markers import Marker
MARKER_KEYS = list(default_environment().keys())
def convert_pipfile_requirement(req):
markers = []
if "markers" in req:
markers.append(Marker(req["markers"]))
for key in MARKER_KEYS:
if key in req:
marker = Marker(f"{key}{req[key]}")
markers.append(marker)
del req[key]
if markers:
marker = functools.reduce(operator.and_, markers)
req["marker"] = str(marker).replace('"', "'")
return req
def check_fingerprint(project, filename):
return os.path.basename(filename) == "Pipfile"
def test_that_current_platform_to_environment_dict_equals_default_environment(
current_platform: TargetPlatform):
assert current_platform.environment_dictionary() == default_environment()
async def install(
requirements: str | list[str],
keep_going: bool = False,
deps: bool = True,
credentials: str | None = None,
pre: bool = False,
) -> None:
"""Install the given package and all of its dependencies.
See :ref:`loading packages <loading_packages>` for more information.
This only works for packages that are either pure Python or for packages
with C extensions that are built in Pyodide. If a pure Python package is not
found in the Pyodide repository it will be loaded from PyPI.
When used in web browsers, downloads from PyPI will be cached. When run in
Node.js, packages are currently not cached, and will be re-downloaded each
time ``micropip.install`` is run.
Parameters
----------
requirements : ``str | List[str]``
A requirement or list of requirements to install. Each requirement is a
string, which should be either a package name or URL to a wheel:
- If the requirement ends in ``.whl`` it will be interpreted as a URL.
The file must be a wheel named in compliance with the
`PEP 427 naming convention <https://www.python.org/dev/peps/pep-0427/#file-format>`_.
- If the requirement does not end in ``.whl``, it will interpreted as the
name of a package. A package by this name must either be present in the
Pyodide repository at :any:`indexURL <globalThis.loadPyodide>` or on PyPI
keep_going : ``bool``, default: False
This parameter decides the behavior of the micropip when it encounters a
Python package without a pure Python wheel while doing dependency
resolution:
- If ``False``, an error will be raised on first package with a missing wheel.
- If ``True``, the micropip will keep going after the first error, and report a list
of errors at the end.
deps : ``bool``, default: True
If ``True``, install dependencies specified in METADATA file for
each package. Otherwise do not install dependencies.
credentials : ``Optional[str]``
This parameter specifies the value of ``credentials`` when calling the
`fetch() <https://developer.mozilla.org/en-US/docs/Web/API/fetch>`__ function
which is used to download the package.
When not specified, ``fetch()`` is called without ``credentials``.
pre : ``bool``, default: False
If ``True``, include pre-release and development versions.
By default, micropip only finds stable versions.
Returns
-------
``Future``
A ``Future`` that resolves to ``None`` when all packages have been
downloaded and installed.
"""
importlib.invalidate_caches()
ctx = default_environment()
if isinstance(requirements, str):
requirements = [requirements]
fetch_kwargs = dict()
if credentials:
fetch_kwargs["credentials"] = credentials
# Note: getsitepackages is not available in a virtual environment...
# See https://github.com/pypa/virtualenv/issues/228 (issue is closed but
# problem is not fixed)
from site import getsitepackages
wheel_base = Path(getsitepackages()[0])
transaction = Transaction(
ctx=ctx,
ctx_extras=[],
keep_going=keep_going,
deps=deps,
pre=pre,
fetch_kwargs=fetch_kwargs,
)
await transaction.gather_requirements(requirements)
if transaction.failed:
failed_requirements = ", ".join(
[f"'{req}'" for req in transaction.failed])
raise ValueError(
f"Can't find a pure Python 3 wheel for: {failed_requirements}\n"
f"See: {FAQ_URLS['cant_find_wheel']}\n")
wheel_promises = []
# Install built-in packages
pyodide_packages = transaction.pyodide_packages
if len(pyodide_packages):
# Note: branch never happens in out-of-browser testing because in
# that case BUILTIN_PACKAGES is empty.
wheel_promises.append(
asyncio.ensure_future(
loadPackage(to_js([name
for [name, _, _] in pyodide_packages]))))
# Now install PyPI packages
for wheel in transaction.wheels:
# detect whether the wheel metadata is from PyPI or from custom location
# wheel metadata from PyPI has SHA256 checksum digest.
wheel_promises.append(wheel.install(wheel_base))
await gather(*wheel_promises)
def main(
dependencies,
requirements_file,
install,
editable,
user,
lock,
pipe,
json,
sort,
tree,
tree_ascii,
tree_json,
tree_json_exact,
reversed_tree,
max_depth,
cache_dir,
no_cache_dir,
index_url,
extra_index_url,
pre,
verbose,
):
if verbose == 0:
logger.setLevel(logging.ERROR)
if verbose == 1:
logger.setLevel(logging.WARNING)
if verbose == 2:
logger.setLevel(logging.INFO)
if verbose >= 3:
logger.setLevel(logging.DEBUG)
logger.debug("Environment: {}".format(default_environment()))
logger.debug("Pip version: {}".format(PIP_VERSION))
if (sum((
pipe,
(json or tree),
tree_ascii,
tree_json,
tree_json_exact,
reversed_tree,
)) > 1):
raise click.ClickException(
"Illegal combination of output formats selected")
if tree_ascii or reversed_tree:
tree = True
elif tree_json_exact:
tree_json = True
if max_depth == 0 or max_depth < -1:
raise click.ClickException(
"Illegal --max_depth selected: {}".format(max_depth))
if max_depth == -1:
max_depth = 0
elif max_depth and not (tree or tree_json or reversed_tree):
raise click.ClickException(
"--max-depth has no effect without --tree, --tree-json, --tree-json-exact, or --reversed-tree"
)
if requirements_file:
if dependencies:
raise click.ClickException(
"-r can not be used in conjunction with directly passed requirements"
)
dependencies = []
for path in requirements_file:
dependencies += read_requirements(path)
if editable:
if not install:
raise click.ClickException(
"--editable has no effect without --install")
if not sorted(dependencies)[0].startswith("."):
raise click.ClickException(
"--editable does not accept input '{}'".format(
" ".join(dependencies)))
if user:
if not install:
raise click.ClickException(
"--user has no effect without --install")
for dep in dependencies:
if os.sep in dep:
raise click.ClickException(
"'{}' looks like a path, and is not supported yet by pipgrip".
format(dep))
if no_cache_dir:
cache_dir = tempfile.mkdtemp()
try:
source = PackageSource(
cache_dir=cache_dir,
index_url=index_url,
extra_index_url=extra_index_url,
pre=pre,
)
for root_dependency in dependencies:
source.root_dep(root_dependency)
solver = VersionSolver(source)
solution = solver.solve()
decision_packages = OrderedDict()
for package, version in solution.decisions.items():
if package == Package.root():
continue
decision_packages[package] = version
logger.debug(decision_packages)
tree_root, packages_tree_dict, packages_flat = build_tree(
source, decision_packages)
if lock:
with io.open(os.path.join(os.getcwd(), "pipgrip.lock"),
mode="w",
encoding="utf-8") as fp:
# a lockfile containing `.` will break pip install -r
fp.write("\n".join(
render_lock(packages_flat, include_dot=False, sort=sort)) +
"\n")
if reversed_tree:
raise NotImplementedError()
# TODO tree_root = reverse_tree(tree_root)
if tree:
if json:
output = dumps(
render_json_tree_full(tree_root, max_depth, sort))
else:
output = render_tree(tree_root, max_depth, tree_ascii)
elif tree_json:
output = dumps(render_json_tree(tree_root, max_depth,
tree_json_exact),
sort_keys=sort)
elif pipe:
output = " ".join(
render_lock(packages_flat, include_dot=True, sort=sort))
elif json:
output = dumps(packages_flat, sort_keys=sort)
else:
output = "\n".join(
render_lock(packages_flat, include_dot=True, sort=sort))
click.echo(output)
if install:
install_packages(
# sort to ensure . is added right after --editable
packages=sorted(dependencies),
constraints=render_lock(packages_flat,
include_dot=False,
sort=True),
index_url=index_url,
extra_index_url=extra_index_url,
pre=pre,
cache_dir=cache_dir,
editable=editable,
user=user,
)
except (SolverFailure, click.ClickException, CalledProcessError) as exc:
raise click.ClickException(str(exc))
finally:
if no_cache_dir:
shutil.rmtree(cache_dir)
