def test_executor_should_use_cached_link_and_hash(
config, io, pool, mocker, fixture_dir, tmp_dir
):
# Produce a file:/// URI that is a valid link
link_cached = Link(
fixture_dir("distributions")
.joinpath("demo-0.1.0-py2.py3-none-any.whl")
.as_uri()
)
mocker.patch.object(
Chef, "get_cached_archive_for_link", side_effect=lambda _: link_cached
)
env = MockEnv(path=Path(tmp_dir))
executor = Executor(env, pool, config, io)
package = Package("demo", "0.1.0")
package.files = [
{
"file": "demo-0.1.0-py2.py3-none-any.whl",
"hash": "md5:15507846fd4299596661d0197bfb4f90",
}
]
archive = executor._download_link(
Install(package), Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl")
)
assert archive == link_cached
def test_get_cached_archives_for_link(config, mocker):
chef = Chef(
config,
MockEnv(marker_env={
"interpreter_name": "cpython",
"interpreter_version": "3.8.3"
}),
)
distributions = Path(__file__).parent.parent.joinpath(
"fixtures/distributions")
mocker.patch.object(
chef,
"get_cache_directory_for_link",
return_value=distributions,
)
archives = chef.get_cached_archives_for_link(
Link("https://files.python-poetry.org/demo-0.1.0.tar.gz"))
assert archives
assert set(archives) == {
Link(path.as_uri())
for path in distributions.glob("demo-0.1.0*")
}
def test_executor_should_use_cached_link_and_hash(
tmp_venv: VirtualEnv,
pool: Pool,
config: Config,
io: BufferedIO,
mocker: MockerFixture,
fixture_dir: FixtureDirGetter,
):
# Produce a file:/// URI that is a valid link
link_cached = Link(
fixture_dir("distributions").joinpath(
"demo-0.1.0-py2.py3-none-any.whl").as_uri())
mocker.patch(
"poetry.installation.chef.Chef.get_cached_archive_for_link",
return_value=link_cached,
)
package = Package("demo", "0.1.0")
# Set package.files so the executor will attempt to hash the package
package.files = [{
"file":
"demo-0.1.0-py2.py3-none-any.whl",
"hash":
"sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a", # noqa: E501
}]
executor = Executor(tmp_venv, pool, config, io)
archive = executor._download_link(
Install(package),
Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl"),
)
assert archive == link_cached
def test_executor_should_check_every_possible_hash_types(
config, io, pool, mocker, fixture_dir, tmp_dir
):
mocker.patch.object(
Chef, "get_cached_archive_for_link", side_effect=lambda link: link,
)
mocker.patch.object(
Executor,
"_download_archive",
return_value=fixture_dir("distributions").joinpath(
"demo-0.1.0-py2.py3-none-any.whl"
),
)
env = MockEnv(path=Path(tmp_dir))
executor = Executor(env, pool, config, io)
package = Package("demo", "0.1.0")
package.files = [
{
"file": "demo-0.1.0-py2.py3-none-any.whl",
"hash": "md5:15507846fd4299596661d0197bfb4f90",
}
]
archive = executor._download_link(
Install(package), Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl")
)
assert archive == fixture_dir("distributions").joinpath(
"demo-0.1.0-py2.py3-none-any.whl"
)
def _install(self, operation):
package = operation.package
if package.source_type == "directory":
return self._install_directory(operation)
if package.source_type == "git":
return self._install_git(operation)
if package.source_type == "file":
archive = self._prepare_file(operation)
elif package.source_type == "url":
archive = self._download_link(operation, Link(package.source_url))
else:
archive = self._download(operation)
operation_message = self.get_operation_message(operation)
message = " <fg=blue;options=bold>•</> {message}: <info>Installing...</info>".format(
message=operation_message, )
self._write(operation, message)
args = ["install", "--no-deps", str(archive)]
if operation.job_type == "update":
args.insert(2, "-U")
return self.run_pip(*args)
def _install(self, operation: Union[Install, Update]) -> int:
package = operation.package
if package.source_type == "directory":
return self._install_directory(operation)
if package.source_type == "git":
return self._install_git(operation)
if package.source_type == "file":
archive = self._prepare_file(operation)
elif package.source_type == "url":
archive = self._download_link(operation, Link(package.source_url))
else:
archive = self._download(operation)
operation_message = self.get_operation_message(operation)
message = (
" <fg=blue;options=bold>•</> {message}: <info>Installing...</info>".format(
message=operation_message,
)
)
self._write(operation, message)
return pip_install(
str(archive), self._env, upgrade=operation.job_type == "update"
)
def find_links_for_package(self, package: Package) -> list[Link]:
return [
Link(
f"https://foo.bar/files/{escape_name(package.name)}"
f"-{escape_version(package.version.text)}-py2.py3-none-any.whl"
)
]
def test_get_cached_archive_for_link(config: Config, mocker: MockerFixture,
link: str, cached: str):
chef = Chef(
config,
MockEnv(
version_info=(3, 8, 3),
marker_env={
"interpreter_name": "cpython",
"interpreter_version": "3.8.3"
},
supported_tags=[
Tag("cp38", "cp38", "macosx_10_15_x86_64"),
Tag("py3", "none", "any"),
],
),
)
mocker.patch.object(
chef,
"get_cached_archives_for_link",
return_value=[
Path("/cache/demo-0.1.0-py2.py3-none-any"),
Path("/cache/demo-0.1.0.tar.gz"),
Path("/cache/demo-0.1.0-cp38-cp38-macosx_10_15_x86_64.whl"),
Path("/cache/demo-0.1.0-cp37-cp37-macosx_10_15_x86_64.whl"),
],
)
archive = chef.get_cached_archive_for_link(Link(link))
assert Path(cached) == archive
def link_package_data(cls, link: Link) -> Package | None:
name, version_string, version = None, None, None
m = wheel_file_re.match(link.filename) or sdist_file_re.match(link.filename)
if m:
name = canonicalize_name(m.group("name"))
version_string = m.group("ver")
else:
info, ext = link.splitext()
match = cls.VERSION_REGEX.match(info)
if match:
name = match.group(1)
version_string = match.group(2)
if version_string:
try:
version = Version.parse(version_string)
except ValueError:
logger.debug(
"Skipping url (%s) due to invalid version (%s)", link.url, version
)
return None
pkg = None
if name and version:
pkg = Package(name, version, source_url=link.url)
return pkg
def test_pip_install_link(tmp_dir, tmp_venv, fixture_dir):
file_path = Link(
path_to_url(fixture_dir("distributions/demo-0.1.0-py2.py3-none-any.whl"))
)
result = pip_install(file_path, tmp_venv)
assert "Successfully installed demo-0.1.0" in result
def link_source(mocker: MockerFixture) -> LinkSource:
url = "https://example.org"
link_source = LinkSource(url)
mocker.patch(
f"{LinkSource.__module__}.{LinkSource.__qualname__}.links",
new_callable=PropertyMock,
return_value=iter(
[
Link(f"{url}/demo-0.1.0.tar.gz"),
Link(f"{url}/demo-0.1.0_invalid.tar.gz"),
Link(f"{url}/invalid.tar.gz"),
Link(f"{url}/demo-0.1.0-py2.py3-none-any.whl"),
Link(f"{url}/demo-0.1.1.tar.gz"),
]
),
)
return link_source
def find_links_for_package(self, package):
return [
Link(
"https://foo.bar/files/{}-{}-py2.py3-none-any.whl".format(
escape_name(package.name), escape_version(package.version.text)
)
)
]
def get_cached_archives_for_link(self, link): # type: (Link) -> List[Link]
cache_dir = self.get_cache_directory_for_link(link)
archive_types = ["whl", "tar.gz", "tar.bz2", "bz2", "zip"]
links = []
for archive_type in archive_types:
for archive in cache_dir.glob("*.{}".format(archive_type)):
links.append(Link("file://{}".format(str(archive))))
return links
def get_cached_archives_for_link(self, link: Link) -> list[Link]:
cache_dir = self.get_cache_directory_for_link(link)
archive_types = ["whl", "tar.gz", "tar.bz2", "bz2", "zip"]
links = []
for archive_type in archive_types:
for archive in cache_dir.glob(f"*.{archive_type}"):
links.append(Link(archive.as_uri()))
return links
def find_links_for_package(self, package: Package) -> List[Link]:
json_data = self._get("pypi/{}/{}/json".format(package.name, package.version))
if json_data is None:
return []
links = []
for url in json_data["urls"]:
h = "sha256={}".format(url["digests"]["sha256"])
links.append(Link(url["url"] + "#" + h))
return links
def find_links_for_package(self, package: Package) -> List[Link]:
json_data = self._get(f"pypi/{package.name}/{package.version}/json")
if json_data is None:
return []
links = []
for url in json_data["urls"]:
h = f"sha256={url['digests']['sha256']}"
links.append(Link(url["url"] + "#" + h))
return links
def test_get_cached_archive_for_link(config, mocker):
chef = Chef(
config,
MockEnv(
version_info=(3, 8, 3),
marker_env={
"interpreter_name": "cpython",
"interpreter_version": "3.8.3"
},
supported_tags=[
Tag("cp38", "cp38", "macosx_10_15_x86_64"),
Tag("py3", "none", "any"),
],
),
)
cwd = Path.cwd() / ".pypoetrycache"
mocker.patch.object(
chef,
"get_cached_archives_for_link",
return_value=[
Link(f"file:///{cwd}demo-0.1.0-py2.py3-none-any"),
Link(f"file:///{cwd}demo-0.1.0.tar.gz"),
Link(f"file:///{cwd}demo-0.1.0-cp38-cp38-macosx_10_15_x86_64.whl"),
Link(f"file:///{cwd}demo-0.1.0-cp37-cp37-macosx_10_15_x86_64.whl"),
],
)
archive = chef.get_cached_archive_for_link(
Link("https://files.python-poetry.org/demo-0.1.0.tar.gz"))
assert Link(f"file:///{cwd}demo-0.1.0-cp38-cp38-macosx_10_15_x86_64.whl"
) == archive
def links(self) -> Iterator[Link]:
for anchor in self._parsed.findall(".//a"):
if anchor.get("href"):
href = anchor.get("href")
url = self.clean_link(urllib.parse.urljoin(self._url, href))
pyrequire = anchor.get("data-requires-python")
pyrequire = unescape(pyrequire) if pyrequire else None
link = Link(url, self, requires_python=pyrequire)
if link.ext not in self.SUPPORTED_FORMATS:
continue
yield link
def test_get_cache_directory_for_link(config):
chef = Chef(
config,
MockEnv(
marker_env={"interpreter_name": "cpython", "interpreter_version": "3.8.3"}
),
)
directory = chef.get_cache_directory_for_link(
Link("https://files.python-poetry.org/poetry-1.1.0.tar.gz")
)
expected = Path(
"/foo/artifacts/ba/63/13/283a3b3b7f95f05e9e6f84182d276f7bb0951d5b0cc24422b33f7a4648"
)
assert expected == directory
def test_executor_should_not_write_pep610_url_references_for_cached_package(
package: Package,
mocker: MockerFixture,
fixture_dir: FixtureDirGetter,
tmp_venv: VirtualEnv,
pool: Pool,
config: Config,
io: BufferedIO,
):
link_cached = Link(
fixture_dir("distributions").joinpath(
"demo-0.1.0-py2.py3-none-any.whl").as_uri())
mocker.patch("poetry.installation.executor.Executor._download",
return_value=link_cached)
executor = Executor(tmp_venv, pool, config, io)
executor.execute([Install(package)])
verify_installed_distribution(tmp_venv, package)
def link_version(self, link: Link) -> Optional[Version]:
m = wheel_file_re.match(link.filename)
if m:
version = m.group("ver")
else:
info, ext = link.splitext()
match = self.VERSION_REGEX.match(info)
if not match:
return None
version = match.group(2)
try:
version = Version.parse(version)
except ValueError:
return None
return version
def _install(self, operation: Install | Update) -> int:
package = operation.package
if package.source_type == "directory":
return self._install_directory(operation)
if package.source_type == "git":
return self._install_git(operation)
if package.source_type == "file":
archive = self._prepare_file(operation)
elif package.source_type == "url":
archive = self._download_link(operation, Link(package.source_url))
else:
archive = self._download(operation)
operation_message = self.get_operation_message(operation)
message = (f" <fg=blue;options=bold>•</> {operation_message}:"
" <info>Installing...</info>")
self._write(operation, message)
return self.pip_install(archive,
upgrade=operation.job_type == "update")
def test_executor_should_check_every_possible_hash_types_before_failing(
config, io, pool, mocker, fixture_dir, tmp_dir
):
mocker.patch.object(
Chef, "get_cached_archive_for_link", side_effect=lambda link: link,
)
mocker.patch.object(
Executor,
"_download_archive",
return_value=fixture_dir("distributions").joinpath(
"demo-0.1.0-py2.py3-none-any.whl"
),
)
env = MockEnv(path=Path(tmp_dir))
executor = Executor(env, pool, config, io)
package = Package("demo", "0.1.0")
package.files = [
{"file": "demo-0.1.0-py2.py3-none-any.whl", "hash": "md5:123456"},
{"file": "demo-0.1.0-py2.py3-none-any.whl", "hash": "sha256:123456"},
]
expected_message = (
"Invalid hashes "
"("
"md5:15507846fd4299596661d0197bfb4f90, "
"sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a"
") "
"for demo (0.1.0) using archive demo-0.1.0-py2.py3-none-any.whl. "
"Expected one of md5:123456, sha256:123456."
)
with pytest.raises(RuntimeError, match=re.escape(expected_message)):
executor._download_link(
Install(package),
Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl"),
)
def get_cached_archives_for_link(self, link): # type: (Link) -> List[Link]
cache_dir = self.get_cache_directory_for_link(link)
archive_types = ["whl", "tar.gz", "tar.bz2", "bz2", "zip"]
links = []
for archive_type in archive_types:
for archive in cache_dir.glob("*.{}".format(archive_type)):
accept = True
if link.hash:
real_hash = get_file_hash(archive, link.hash_name)
if real_hash != link.hash:
logger = logging.getLogger(__name__)
logger.warning("cache of {} is corrupted".format(
link.filename))
accept = False
if accept:
links.append(Link(archive.as_uri()))
return links
def _get_info_from_urls(self, urls: Dict[str, List[str]]) -> PackageInfo:
# Checking wheels first as they are more likely to hold
# the necessary information
if "bdist_wheel" in urls:
# Check fo a universal wheel
wheels = urls["bdist_wheel"]
universal_wheel = None
universal_python2_wheel = None
universal_python3_wheel = None
platform_specific_wheels = []
for wheel in wheels:
link = Link(wheel)
m = wheel_file_re.match(link.filename)
if not m:
continue
pyver = m.group("pyver")
abi = m.group("abi")
plat = m.group("plat")
if abi == "none" and plat == "any":
# Universal wheel
if pyver == "py2.py3":
# Any Python
universal_wheel = wheel
elif pyver == "py2":
universal_python2_wheel = wheel
else:
universal_python3_wheel = wheel
else:
platform_specific_wheels.append(wheel)
if universal_wheel is not None:
return self._get_info_from_wheel(universal_wheel)
info = None
if universal_python2_wheel and universal_python3_wheel:
info = self._get_info_from_wheel(universal_python2_wheel)
py3_info = self._get_info_from_wheel(universal_python3_wheel)
if py3_info.requires_dist:
if not info.requires_dist:
info.requires_dist = py3_info.requires_dist
return info
py2_requires_dist = set(
dependency_from_pep_508(r).to_pep_508()
for r in info.requires_dist)
py3_requires_dist = set(
dependency_from_pep_508(r).to_pep_508()
for r in py3_info.requires_dist)
base_requires_dist = py2_requires_dist & py3_requires_dist
py2_only_requires_dist = py2_requires_dist - py3_requires_dist
py3_only_requires_dist = py3_requires_dist - py2_requires_dist
# Normalizing requires_dist
requires_dist = list(base_requires_dist)
for requirement in py2_only_requires_dist:
dep = dependency_from_pep_508(requirement)
dep.marker = dep.marker.intersect(
parse_marker("python_version == '2.7'"))
requires_dist.append(dep.to_pep_508())
for requirement in py3_only_requires_dist:
dep = dependency_from_pep_508(requirement)
dep.marker = dep.marker.intersect(
parse_marker("python_version >= '3'"))
requires_dist.append(dep.to_pep_508())
info.requires_dist = sorted(list(set(requires_dist)))
if info:
return info
# Prefer non platform specific wheels
if universal_python3_wheel:
return self._get_info_from_wheel(universal_python3_wheel)
if universal_python2_wheel:
return self._get_info_from_wheel(universal_python2_wheel)
if platform_specific_wheels and "sdist" not in urls:
# Pick the first wheel available and hope for the best
return self._get_info_from_wheel(platform_specific_wheels[0])
return self._get_info_from_sdist(urls["sdist"][0])
def make_url(ext):
checksum = sha256(str(uuid.uuid4()).encode())
return Link("https://files.pythonhosted.org/packages/16/52/dead/"
"demo-1.0.0.{}#sha256={}".format(ext, checksum))
def create_from_pep_508(cls,
name: str,
relative_to: Path | None = None) -> Dependency:
"""
Resolve a PEP-508 requirement string to a `Dependency` instance. If a `relative_to`
path is specified, this is used as the base directory if the identified dependency is
of file or directory type.
"""
from poetry.core.packages.url_dependency import URLDependency
from poetry.core.packages.utils.link import Link
from poetry.core.packages.utils.utils import is_archive_file
from poetry.core.packages.utils.utils import is_installable_dir
from poetry.core.packages.utils.utils import is_url
from poetry.core.packages.utils.utils import path_to_url
from poetry.core.packages.utils.utils import strip_extras
from poetry.core.packages.utils.utils import url_to_path
from poetry.core.packages.vcs_dependency import VCSDependency
from poetry.core.utils.patterns import wheel_file_re
from poetry.core.vcs.git import ParsedUrl
from poetry.core.version.requirements import Requirement
# Removing comments
parts = name.split(" #", 1)
name = parts[0].strip()
if len(parts) > 1:
rest = parts[1]
if " ;" in rest:
name += " ;" + rest.split(" ;", 1)[1]
req = Requirement(name)
name = req.name
link = None
if is_url(name):
link = Link(name)
elif req.url:
link = Link(req.url)
else:
path_str = os.path.normpath(os.path.abspath(name))
p, extras = strip_extras(path_str)
if os.path.isdir(p) and (os.path.sep in name
or name.startswith(".")):
if not is_installable_dir(p):
raise ValueError(
f"Directory {name!r} is not installable. File 'setup.py' "
"not found.")
link = Link(path_to_url(p))
elif is_archive_file(p):
link = Link(path_to_url(p))
# it's a local file, dir, or url
if link:
is_file_uri = link.scheme == "file"
is_relative_uri = is_file_uri and re.search(r"\.\./", link.url)
# Handle relative file URLs
if is_file_uri and is_relative_uri:
path = Path(link.path)
if relative_to:
path = relative_to / path
link = Link(path_to_url(path))
# wheel file
version = None
if link.is_wheel:
m = wheel_file_re.match(link.filename)
if not m:
raise ValueError(f"Invalid wheel name: {link.filename}")
name = m.group("name")
version = m.group("ver")
dep: Dependency | None = None
if link.scheme.startswith("git+"):
url = ParsedUrl.parse(link.url)
dep = VCSDependency(
name,
"git",
url.url,
rev=url.rev,
directory=url.subdirectory,
extras=req.extras,
)
elif link.scheme == "git":
dep = VCSDependency(name,
"git",
link.url_without_fragment,
extras=req.extras)
elif link.scheme in ["http", "https"]:
dep = URLDependency(name, link.url, extras=req.extras)
elif is_file_uri:
# handle RFC 8089 references
path = url_to_path(req.url)
dep = _make_file_or_dir_dep(name=name,
path=path,
base=relative_to,
extras=req.extras)
else:
with suppress(ValueError):
# this is a local path not using the file URI scheme
dep = _make_file_or_dir_dep(
name=name,
path=Path(req.url),
base=relative_to,
extras=req.extras,
)
if dep is None:
dep = Dependency(name, version or "*", extras=req.extras)
if version:
dep._constraint = parse_constraint(version)
else:
constraint: VersionConstraint | str
if req.pretty_constraint:
constraint = req.constraint
else:
constraint = "*"
dep = Dependency(name, constraint, extras=req.extras)
if req.marker:
dep.marker = req.marker
return dep
def test_link_package_data(filename: str, expected: Package | None) -> None:
link = Link(f"https://example.org/{filename}")
assert LinkSource.link_package_data(link) == expected
def test_links_for_version(
version_string: str, filenames: Iterable[str], link_source: LinkSource
) -> None:
version = Version.parse(version_string)
expected = {Link(f"{link_source.url}/{name}") for name in filenames}
assert set(link_source.links_for_version("demo", version)) == expected
