def test_impossible_pyspec():
spec = PySpecSet(">=3.6,<3.4")
a = PySpecSet(">=2.7")
assert spec.is_impossible
assert (spec & a).is_impossible
assert spec | a == a
spec_copy = spec.copy()
assert spec_copy.is_impossible
assert str(spec_copy) == "impossible"
def get_dependencies(
self,
candidate: Candidate) -> Tuple[List[Requirement], PySpecSet, str]:
"""Get (dependencies, python_specifier, summary) of the candidate."""
requirements, requires_python, summary = [], "", ""
last_ext_info = None
for getter in self.dependency_generators():
try:
requirements, requires_python, summary = getter(candidate)
except CandidateInfoNotFound:
last_ext_info = sys.exc_info()
continue
break
else:
if last_ext_info is not None:
raise last_ext_info[1].with_traceback(last_ext_info[2])
requirements = [parse_requirement(line) for line in requirements]
if candidate.req.extras:
# HACK: If this candidate has extras, add the original candidate
# (same pinned version, no extras) as its dependency. This ensures
# the same package with different extras (treated as distinct by
# the resolver) have the same version.
self_req = candidate.req.copy()
self_req.extras = None
requirements.append(self_req)
return requirements, PySpecSet(requires_python), summary
def resolve_func(
lines,
requires_python="",
allow_prereleases=None,
strategy="all",
tracked_names=None,
):
repository.environment.python_requires = PySpecSet(requires_python)
if allow_prereleases is not None:
project.tool_settings["allow_prereleases"] = allow_prereleases
requirements = []
for line in lines:
if line.startswith("-e "):
requirements.append(parse_requirement(line[3:], True))
else:
requirements.append(parse_requirement(line))
provider = project.get_provider(strategy, tracked_names)
ui = project.core.ui
with ui.open_spinner("Resolving dependencies") as spin, ui.logging(
"lock"):
reporter = SpinnerReporter(spin, requirements)
resolver = Resolver(provider, reporter)
mapping, *_ = _resolve(resolver, requirements,
repository.environment.python_requires)
return mapping
def test_non_editable_no_override_editable(project, working_set, is_editable):
project.environment.python_requires = PySpecSet(">=3.6")
actions.do_add(
project,
editables=[
"git+https://github.com/test-root/demo.git#egg=demo",
"git+https://github.com/test-root/demo-module.git#egg=demo-module",
],
)
actions.do_add(
project,
packages=["git+https://github.com/test-root/demo.git#egg=demo"],
no_editable=not is_editable,
)
assert working_set["demo-module"].link_file
assert bool(working_set["demo"].link_file) is is_editable
dependencies = project.get_pyproject_dependencies("default")
if is_editable:
assert dependencies == [
"-e git+https://github.com/test-root/demo.git#egg=demo",
"-e git+https://github.com/test-root/demo-module.git#egg=demo-module",
"demo @ git+https://github.com/test-root/demo.git",
]
else:
assert dependencies == [
"demo @ git+https://github.com/test-root/demo.git",
"-e git+https://github.com/test-root/demo-module.git#egg=demo-module",
]
def resolve_requirements(
repository,
lines,
requires_python="",
allow_prereleases=None,
strategy="reuse",
preferred_pins=None,
tracked_names=None,
):
requirements = {}
if isinstance(lines, list):
lines = {"default": lines}
for k, v in lines.items():
for line in v:
req = parse_requirement(line)
requirements.setdefault(k, {})[identify(req)] = req
requires_python = PySpecSet(requires_python)
if not preferred_pins:
provider = BaseProvider(repository, requires_python, allow_prereleases)
else:
provider_class = (ReusePinProvider
if strategy == "reuse" else EagerUpdateProvider)
provider = provider_class(
preferred_pins,
tracked_names or (),
repository,
requires_python,
allow_prereleases,
)
flat_reqs = list(
itertools.chain(*[deps.values() for _, deps in requirements.items()]))
reporter = SimpleReporter(flat_reqs)
resolver = Resolver(provider, reporter)
mapping, *_ = resolve(resolver, requirements, requires_python)
return mapping
def find_matches(
self,
requirement: Requirement,
requires_python: PySpecSet = PySpecSet(),
allow_prereleases: Optional[bool] = None,
allow_all: bool = False,
) -> List[Candidate]:
"""Find matching candidates of a requirement.
:param requirement: the given requirement.
:param requires_python: the Python version constraint.
:param allow_prereleases: whether allow prerelease versions, or let us determine
if not given. If no non-prerelease is available, prereleases will be used.
:param allow_all: whether allow all wheels.
:returns: a list of candidates.
"""
if requirement.is_named:
return self._find_named_matches(
requirement, requires_python, allow_prereleases, allow_all
)
else:
# Fetch metadata so that resolver can know the candidate's name.
can = Candidate(requirement, self.environment)
can.get_metadata()
return [can]
def get_dependencies(
self, candidate: Candidate
) -> tuple[list[Requirement], PySpecSet, str]:
"""Get (dependencies, python_specifier, summary) of the candidate."""
requires_python, summary = "", ""
requirements: list[str] = []
last_ext_info = None
for getter in self.dependency_generators():
try:
requirements, requires_python, summary = getter(candidate)
except CandidateInfoNotFound:
last_ext_info = sys.exc_info()
continue
break
else:
if last_ext_info is not None:
raise last_ext_info[1].with_traceback(last_ext_info[2]) # type: ignore
reqs = [parse_requirement(line) for line in requirements]
if candidate.req.extras:
# XXX: If the requirement has extras, add the original candidate
# (without extras) as its dependency. This ensures the same package with
# different extras resolve to the same version.
self_req = dataclasses.replace(candidate.req, extras=None, marker=None)
reqs.append(self_req)
# Store the metadata on the candidate for caching
candidate.requires_python = requires_python
candidate.summary = summary
return reqs, PySpecSet(requires_python), summary
def test_add_editable_package(project, working_set, is_dev):
# Ensure that correct python version is used.
project.environment.python_requires = PySpecSet(">=3.6")
actions.do_add(project, is_dev, packages=["demo"])
actions.do_add(
project,
is_dev,
editables=["git+https://github.com/test-root/demo.git#egg=demo"],
)
group = (
project.tool_settings["dev-dependencies"]["dev"]
if is_dev
else project.meta["dependencies"]
)
assert "demo" in group[0]
assert "-e git+https://github.com/test-root/demo.git#egg=demo" in group[1]
locked_candidates = project.locked_repository.all_candidates
assert (
locked_candidates["demo"].prepare(project.environment).revision
== "1234567890abcdef"
)
assert locked_candidates["idna"].version == "2.7"
assert "idna" in working_set
actions.do_sync(project, no_editable=True)
assert not working_set["demo"].link_file
def resolve_requirements(
repository,
lines,
requires_python="",
allow_prereleases=None,
strategy="all",
preferred_pins=None,
tracked_names=None,
):
requirements = []
for line in lines:
if line.startswith("-e "):
requirements.append(parse_requirement(line[3:], True))
else:
requirements.append(parse_requirement(line))
requires_python = PySpecSet(requires_python)
if not preferred_pins:
provider = BaseProvider(repository, requires_python, allow_prereleases)
else:
provider_class = (ReusePinProvider
if strategy == "reuse" else EagerUpdateProvider)
provider = provider_class(
preferred_pins,
tracked_names or (),
repository,
requires_python,
allow_prereleases,
)
ui = termui.UI()
with ui.open_spinner("Resolving dependencies") as spin, ui.logging("lock"):
reporter = SpinnerReporter(spin, requirements)
resolver = Resolver(provider, reporter)
mapping, *_ = resolve(resolver, requirements, requires_python)
return mapping
def test_deprecated_section_argument(project, invoke, working_set):
project.environment.python_requires = PySpecSet(">=3.6")
r = invoke(["add", "--section", "optional", "demo"], obj=project)
assert r.exit_code == 0
assert "DEPRECATED" in r.stderr
assert "demo" in working_set
assert "demo" in project.get_dependencies("optional")
def marker(self, value) -> None:
try:
m = self._marker = get_marker(value)
if not m:
self.marker_no_python, self.requires_python = None, PySpecSet()
else:
self.marker_no_python, self.requires_python = m.split_pyspec()
except InvalidMarker as e:
raise RequirementError("Invalid marker: %s" % str(e)) from None
def test_editable_package_override_non_editable(project, working_set):
project.environment.python_requires = PySpecSet(">=3.6")
actions.do_add(
project,
packages=["git+https://github.com/test-root/demo.git#egg=demo"])
actions.do_add(
project,
editables=["git+https://github.com/test-root/demo.git#egg=demo"],
)
assert working_set["demo"].editable
def find_candidates(
self,
requirement: Requirement,
requires_python: PySpecSet = PySpecSet(),
allow_prereleases: Optional[bool] = None,
allow_all: bool = False,
) -> Iterable[Candidate]:
"""Find candidates of the given NamedRequirement. Let it to be implemented in
subclasses.
"""
# `allow_prereleases` is None means leave it to specifier to decide whether to
# include prereleases
if allow_prereleases is None:
allow_prereleases = requirement.allow_prereleases
requires_python = requires_python & requirement.requires_python
cans = list(self._find_candidates(requirement))
sorted_cans = sorted(
(
c
for c in cans
if requirement.specifier.contains(c.version, allow_prereleases)
and (allow_all or requires_python.is_subset(c.requires_python))
),
key=lambda c: (c.version, c.link.is_wheel),
reverse=True,
)
if not sorted_cans and allow_prereleases is None:
# No non-pre-releases is found, force pre-releases now
sorted_cans = sorted(
(
c
for c in cans
if requirement.specifier.contains(c.version, True)
and (allow_all or requires_python.is_subset(c.requires_python))
),
key=lambda c: c.version,
reverse=True,
)
return sorted_cans
def find_candidates(
self,
requirement: Requirement,
requires_python: PySpecSet = PySpecSet(),
allow_prereleases: Optional[bool] = None,
allow_all: bool = False,
) -> List[Candidate]:
if allow_prereleases is None:
allow_prereleases = requirement.allow_prereleases
cans = []
for version, candidate in self._pypi_data.get(requirement.key,
{}).items():
c = Candidate(
requirement,
self.environment,
name=requirement.project_name,
version=version,
)
c.requires_python = candidate.get("requires_python", "")
cans.append(c)
sorted_cans = sorted(
(c for c in cans
if requirement.specifier.contains(c.version, allow_prereleases)),
key=lambda c: c.version,
reverse=True,
)
if not allow_all:
sorted_cans = [
can for can in sorted_cans
if requires_python.is_subset(can.requires_python)
]
if not sorted_cans and allow_prereleases is None:
# No non-pre-releases is found, force pre-releases now
sorted_cans = sorted(
(c for c in cans
if requirement.specifier.contains(c.version, True)),
key=lambda c: c.version,
reverse=True,
)
return sorted_cans
def find_candidates(
self,
requirement: Requirement,
requires_python: PySpecSet = PySpecSet(),
allow_prereleases: Optional[bool] = None,
allow_all: bool = False,
) -> Iterable[Candidate]:
sources = self.get_filtered_sources(requirement)
# `allow_prereleases` is None means leave it to specifier to decide whether to
# include prereleases
if allow_prereleases is None:
allow_prereleases = requirement.allow_prereleases
with self.environment.get_finder(sources) as finder, allow_all_wheels():
cans = [
Candidate.from_installation_candidate(c, requirement, self.environment)
for c in finder.find_all_candidates(requirement.project_name)
]
sorted_cans = sorted(
(
c
for c in cans
if requirement.specifier.contains(c.version, allow_prereleases)
and (allow_all or requires_python.is_subset(c.requires_python))
),
key=lambda c: (c.version, c.link.is_wheel),
reverse=True,
)
if not sorted_cans and allow_prereleases is None:
# No non-pre-releases is found, force pre-releases now
sorted_cans = sorted(
(
c
for c in cans
if requirement.specifier.contains(c.version, True)
and (allow_all or requires_python.is_subset(c.requires_python))
),
key=lambda c: c.version,
reverse=True,
)
return sorted_cans
def _find_named_matches(
self,
requirement: Requirement,
requires_python: PySpecSet = PySpecSet(),
allow_prereleases: Optional[bool] = None,
allow_all: bool = False,
) -> List[Candidate]:
"""Find candidates of the given NamedRequirement. Let it to be implemented in
subclasses.
"""
raise NotImplementedError
def _build_pyspec_from_marker(markers: List[Any]) -> PySpecSet:
def split_version(version: str) -> List[str]:
if "," in version:
return [v.strip() for v in version.split(",")]
return version.split()
groups = [PySpecSet()]
for marker in markers:
if isinstance(marker, list):
# It is a submarker
groups[-1] = groups[-1] & _build_pyspec_from_marker(marker)
elif isinstance(marker, tuple):
key, op, version = [i.value for i in marker]
if key == "python_version":
if op == ">":
int_versions = [int(ver) for ver in version.split(".")]
int_versions[-1] += 1
version = ".".join(str(v) for v in int_versions)
op = ">="
elif op in ("==", "!="):
if len(version.split(".")) < 3:
version += ".*"
elif op in ("in", "not in"):
version = " ".join(v + ".*"
for v in split_version(version))
if op == "in":
pyspec = reduce(operator.or_,
(PySpecSet(f"=={v}")
for v in split_version(version)))
elif op == "not in":
pyspec = reduce(operator.and_,
(PySpecSet(f"!={v}")
for v in split_version(version)))
else:
pyspec = PySpecSet(f"{op}{version}")
groups[-1] = groups[-1] & pyspec
else:
assert marker in ("and", "or")
if marker == "or":
groups.append(PySpecSet())
return reduce(operator.or_, groups)
def __init__(self, **kwargs):
self._marker = None
self.from_section = "default"
self.marker_no_python = None # type: Optional[Marker]
self.requires_python = PySpecSet() # type: PySpecSet
for k, v in kwargs.items():
if k == "specifier":
v = get_specifier(v)
setattr(self, k, v)
if self.name and not self.project_name:
self.project_name = safe_name(self.name)
self.key = self.project_name.lower()
def environment(self) -> Environment:
if self.is_global:
env = GlobalEnvironment(self)
# Rewrite global project's python requires to be
# compatible with the exact version
env.python_requires = PySpecSet(
"==" + get_python_version(self.python_executable, True)[0])
return env
if self.config["use_venv"] and is_venv_python(self.python_executable):
# Only recognize venv created by python -m venv and virtualenv>20
return GlobalEnvironment(self)
return Environment(self)
def get_environment(self) -> Environment:
"""Get the environment selected by this project"""
if self.is_global:
env = GlobalEnvironment(self)
# Rewrite global project's python requires to be
# compatible with the exact version
env.python_requires = PySpecSet(f"=={self.python.version}")
return env
if self.config["use_venv"] and is_venv_python(self.python.executable):
# Only recognize venv created by python -m venv and virtualenv>20
return GlobalEnvironment(self)
return Environment(self)
def test_remove_both_normal_and_editable_packages(project, is_dev):
project.environment.python_requires = PySpecSet(">=3.6")
actions.do_add(project, is_dev, packages=["demo"])
actions.do_add(
project,
is_dev,
editables=["git+https://github.com/test-root/demo.git#egg=demo"],
)
section = "dev-dependencies" if is_dev else "dependencies"
actions.do_remove(project, is_dev, packages=["demo"])
assert not project.meta[section]
assert "demo" not in project.get_locked_candidates(
"dev" if is_dev else "default")
def test_add_package_unconstrained_rewrite_specifier(project):
project.environment.python_requires = PySpecSet(">=3.6")
actions.do_add(project, packages=["django"], no_self=True)
locked_candidates = project.locked_repository.all_candidates
assert locked_candidates["django"].version == "2.2.9"
assert project.meta.dependencies[0] == "django~=2.2"
actions.do_add(
project, packages=["django-toolbar"], no_self=True, unconstrained=True
)
locked_candidates = project.locked_repository.all_candidates
assert locked_candidates["django"].version == "1.11.8"
assert project.meta.dependencies[0] == "django~=1.11"
def test_remove_both_normal_and_editable_packages(project, is_dev):
project.environment.python_requires = PySpecSet(">=3.6")
actions.do_add(project, is_dev, packages=["demo"])
actions.do_add(
project,
is_dev,
editables=["git+https://github.com/test-root/demo.git#egg=demo"],
)
section = (project.tool_settings["dev-dependencies"]["dev"]
if is_dev else project.meta["dependencies"])
actions.do_remove(project, is_dev, packages=["demo"])
assert not section
assert "demo" not in project.locked_repository.all_candidates
def environment(self) -> Environment:
if self.is_global:
env = GlobalEnvironment(self)
# Rewrite global project's python requires to be
# compatible with the exact version
env.python_requires = PySpecSet(
"==" + get_python_version(env.python_executable, True))
return env
if self.config["use_venv"]:
venv_python = get_venv_python(self.root)
if venv_python:
self.project_config["python.path"] = venv_python
return GlobalEnvironment(self)
return Environment(self)
def find_candidates(
self,
requirement: Requirement,
requires_python: PySpecSet = ALLOW_ALL_PYTHON,
allow_prereleases: Optional[bool] = None,
allow_all: bool = False,
) -> Iterable[Candidate]:
for key, info in self.candidate_info.items():
if key[0] != requirement.identify():
continue
if not (requires_python & PySpecSet(info[1])).contains(
str(self.environment.interpreter.version)):
continue
can = self.packages[key]
can.requires_python = info[1]
yield can
def test_add_remote_package_url(project, is_dev):
project.environment.python_requires = PySpecSet(">=3.6")
actions.do_add(
project,
is_dev,
packages=["http://fixtures.test/artifacts/demo-0.0.1-py2.py3-none-any.whl"],
)
group = (
project.tool_settings["dev-dependencies"]["dev"]
if is_dev
else project.meta["dependencies"]
)
assert (
group[0]
== "demo @ http://fixtures.test/artifacts/demo-0.0.1-py2.py3-none-any.whl"
)
def test_add_cached_vcs_requirement(project, mocker):
project.environment.python_requires = PySpecSet(">=3.6")
url = "git+https://github.com/test-root/demo.git@1234567890abcdef#egg=demo"
built_path = FIXTURES / "artifacts/demo-0.0.1-py2.py3-none-any.whl"
wheel_cache = project.make_wheel_cache()
cache_path = Path(wheel_cache.get_path_for_link(Link(url)))
if not cache_path.exists():
cache_path.mkdir(parents=True)
shutil.copy2(built_path, cache_path)
downloader = mocker.patch("pdm.models.pip_shims.unpack_url")
builder = mocker.patch("pdm.builders.WheelBuilder.build")
actions.do_add(project, packages=[url], no_self=True)
lockfile_entry = next(p for p in project.lockfile["package"] if p["name"] == "demo")
assert lockfile_entry["revision"] == "1234567890abcdef"
downloader.assert_not_called()
builder.assert_not_called()
def find_candidates(
self,
requirement: Requirement,
allow_prereleases: bool | None = None,
ignore_requires_python: bool = False,
) -> Iterable[Candidate]:
for key, info in self.candidate_info.items():
if key[0] != requirement.identify():
continue
if not PySpecSet(info[1]).contains(
str(self.environment.interpreter.version), True
):
continue
can = self.packages[key]
can.requires_python = info[1]
can.prepare(self.environment)
can.req = requirement
yield can
def test_add_editable_package(project, working_set, is_dev):
# Ensure that correct python version is used.
project.environment.python_requires = PySpecSet(">=3.6")
actions.do_add(project, is_dev, packages=["demo"])
actions.do_add(
project,
is_dev,
editables=["git+https://github.com/test-root/demo.git#egg=demo"],
)
section = "dev-dependencies" if is_dev else "dependencies"
assert "demo" in project.meta[section][0]
assert ("-e git+https://github.com/test-root/demo.git#egg=demo"
in project.meta[section][1])
locked_candidates = project.get_locked_candidates(
"dev" if is_dev else "default")
assert locked_candidates["demo"].revision == "1234567890abcdef"
assert locked_candidates["idna"].version == "2.7"
assert "idna" in working_set
def get_environment(self) -> Environment:
if self.is_global:
env = GlobalEnvironment(self)
# Rewrite global project's python requires to be
# compatible with the exact version
env.python_requires = PySpecSet(f"=={self.python.version}")
return env
if self.config["python.use_venv"]:
if self.project_config.get("python.path") and not os.getenv(
"PDM_IGNORE_SAVED_PYTHON"):
return (GlobalEnvironment(self) if is_venv_python(
self.python.executable) else Environment(self))
if os.getenv("VIRTUAL_ENV"):
venv = os.getenv("VIRTUAL_ENV")
self.core.ui.echo(
f"Detected inside an active virtualenv {termui.green(venv)}, "
"reuse it.")
# Temporary usage, do not save in .pdm.toml
self._python = PythonInfo.from_path(get_venv_python(
Path(venv)))
return GlobalEnvironment(self)
existing_venv = next((venv for _, venv in iter_venvs(self)), None)
if existing_venv:
self.core.ui.echo(
f"Virtualenv {termui.green(str(existing_venv))} is reused.",
err=True,
)
path = existing_venv
else:
# Create a virtualenv using the selected Python interpreter
self.core.ui.echo(
"python.use_venv is on, creating a virtualenv for this project...",
fg="yellow",
err=True,
)
backend: str = self.config["venv.backend"]
venv_backend = BACKENDS[backend](self, None)
path = venv_backend.create(None, (), False,
self.config["venv.in_project"])
self.core.ui.echo(f"Virtualenv {path} is created successfully")
self.python = PythonInfo.from_path(get_venv_python(path))
return GlobalEnvironment(self)
else:
return Environment(self)
