def test_executor_should_use_cached_link_and_hash( config, io, pool, mocker, fixture_dir, tmp_dir ): # Produce a file:/// URI that is a valid link link_cached = Link( fixture_dir("distributions") .joinpath("demo-0.1.0-py2.py3-none-any.whl") .as_uri() ) mocker.patch.object( Chef, "get_cached_archive_for_link", side_effect=lambda _: link_cached ) env = MockEnv(path=Path(tmp_dir)) executor = Executor(env, pool, config, io) package = Package("demo", "0.1.0") package.files = [ { "file": "demo-0.1.0-py2.py3-none-any.whl", "hash": "md5:15507846fd4299596661d0197bfb4f90", } ] archive = executor._download_link( Install(package), Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl") ) assert archive == link_cached
def test_get_cached_archives_for_link(config, mocker): chef = Chef( config, MockEnv(marker_env={ "interpreter_name": "cpython", "interpreter_version": "3.8.3" }), ) distributions = Path(__file__).parent.parent.joinpath( "fixtures/distributions") mocker.patch.object( chef, "get_cache_directory_for_link", return_value=distributions, ) archives = chef.get_cached_archives_for_link( Link("https://files.python-poetry.org/demo-0.1.0.tar.gz")) assert archives assert set(archives) == { Link(path.as_uri()) for path in distributions.glob("demo-0.1.0*") }
def test_executor_should_use_cached_link_and_hash( tmp_venv: VirtualEnv, pool: Pool, config: Config, io: BufferedIO, mocker: MockerFixture, fixture_dir: FixtureDirGetter, ): # Produce a file:/// URI that is a valid link link_cached = Link( fixture_dir("distributions").joinpath( "demo-0.1.0-py2.py3-none-any.whl").as_uri()) mocker.patch( "poetry.installation.chef.Chef.get_cached_archive_for_link", return_value=link_cached, ) package = Package("demo", "0.1.0") # Set package.files so the executor will attempt to hash the package package.files = [{ "file": "demo-0.1.0-py2.py3-none-any.whl", "hash": "sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a", # noqa: E501 }] executor = Executor(tmp_venv, pool, config, io) archive = executor._download_link( Install(package), Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl"), ) assert archive == link_cached
def test_executor_should_check_every_possible_hash_types( config, io, pool, mocker, fixture_dir, tmp_dir ): mocker.patch.object( Chef, "get_cached_archive_for_link", side_effect=lambda link: link, ) mocker.patch.object( Executor, "_download_archive", return_value=fixture_dir("distributions").joinpath( "demo-0.1.0-py2.py3-none-any.whl" ), ) env = MockEnv(path=Path(tmp_dir)) executor = Executor(env, pool, config, io) package = Package("demo", "0.1.0") package.files = [ { "file": "demo-0.1.0-py2.py3-none-any.whl", "hash": "md5:15507846fd4299596661d0197bfb4f90", } ] archive = executor._download_link( Install(package), Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl") ) assert archive == fixture_dir("distributions").joinpath( "demo-0.1.0-py2.py3-none-any.whl" )
def _install(self, operation): package = operation.package if package.source_type == "directory": return self._install_directory(operation) if package.source_type == "git": return self._install_git(operation) if package.source_type == "file": archive = self._prepare_file(operation) elif package.source_type == "url": archive = self._download_link(operation, Link(package.source_url)) else: archive = self._download(operation) operation_message = self.get_operation_message(operation) message = " <fg=blue;options=bold>•</> {message}: <info>Installing...</info>".format( message=operation_message, ) self._write(operation, message) args = ["install", "--no-deps", str(archive)] if operation.job_type == "update": args.insert(2, "-U") return self.run_pip(*args)
def _install(self, operation: Union[Install, Update]) -> int: package = operation.package if package.source_type == "directory": return self._install_directory(operation) if package.source_type == "git": return self._install_git(operation) if package.source_type == "file": archive = self._prepare_file(operation) elif package.source_type == "url": archive = self._download_link(operation, Link(package.source_url)) else: archive = self._download(operation) operation_message = self.get_operation_message(operation) message = ( " <fg=blue;options=bold>•</> {message}: <info>Installing...</info>".format( message=operation_message, ) ) self._write(operation, message) return pip_install( str(archive), self._env, upgrade=operation.job_type == "update" )
def find_links_for_package(self, package: Package) -> list[Link]: return [ Link( f"https://foo.bar/files/{escape_name(package.name)}" f"-{escape_version(package.version.text)}-py2.py3-none-any.whl" ) ]
def test_get_cached_archive_for_link(config: Config, mocker: MockerFixture, link: str, cached: str): chef = Chef( config, MockEnv( version_info=(3, 8, 3), marker_env={ "interpreter_name": "cpython", "interpreter_version": "3.8.3" }, supported_tags=[ Tag("cp38", "cp38", "macosx_10_15_x86_64"), Tag("py3", "none", "any"), ], ), ) mocker.patch.object( chef, "get_cached_archives_for_link", return_value=[ Path("/cache/demo-0.1.0-py2.py3-none-any"), Path("/cache/demo-0.1.0.tar.gz"), Path("/cache/demo-0.1.0-cp38-cp38-macosx_10_15_x86_64.whl"), Path("/cache/demo-0.1.0-cp37-cp37-macosx_10_15_x86_64.whl"), ], ) archive = chef.get_cached_archive_for_link(Link(link)) assert Path(cached) == archive
def link_package_data(cls, link: Link) -> Package | None: name, version_string, version = None, None, None m = wheel_file_re.match(link.filename) or sdist_file_re.match(link.filename) if m: name = canonicalize_name(m.group("name")) version_string = m.group("ver") else: info, ext = link.splitext() match = cls.VERSION_REGEX.match(info) if match: name = match.group(1) version_string = match.group(2) if version_string: try: version = Version.parse(version_string) except ValueError: logger.debug( "Skipping url (%s) due to invalid version (%s)", link.url, version ) return None pkg = None if name and version: pkg = Package(name, version, source_url=link.url) return pkg
def test_pip_install_link(tmp_dir, tmp_venv, fixture_dir): file_path = Link( path_to_url(fixture_dir("distributions/demo-0.1.0-py2.py3-none-any.whl")) ) result = pip_install(file_path, tmp_venv) assert "Successfully installed demo-0.1.0" in result
def link_source(mocker: MockerFixture) -> LinkSource: url = "https://example.org" link_source = LinkSource(url) mocker.patch( f"{LinkSource.__module__}.{LinkSource.__qualname__}.links", new_callable=PropertyMock, return_value=iter( [ Link(f"{url}/demo-0.1.0.tar.gz"), Link(f"{url}/demo-0.1.0_invalid.tar.gz"), Link(f"{url}/invalid.tar.gz"), Link(f"{url}/demo-0.1.0-py2.py3-none-any.whl"), Link(f"{url}/demo-0.1.1.tar.gz"), ] ), ) return link_source
def find_links_for_package(self, package): return [ Link( "https://foo.bar/files/{}-{}-py2.py3-none-any.whl".format( escape_name(package.name), escape_version(package.version.text) ) ) ]
def get_cached_archives_for_link(self, link): # type: (Link) -> List[Link] cache_dir = self.get_cache_directory_for_link(link) archive_types = ["whl", "tar.gz", "tar.bz2", "bz2", "zip"] links = [] for archive_type in archive_types: for archive in cache_dir.glob("*.{}".format(archive_type)): links.append(Link("file://{}".format(str(archive)))) return links
def get_cached_archives_for_link(self, link: Link) -> list[Link]: cache_dir = self.get_cache_directory_for_link(link) archive_types = ["whl", "tar.gz", "tar.bz2", "bz2", "zip"] links = [] for archive_type in archive_types: for archive in cache_dir.glob(f"*.{archive_type}"): links.append(Link(archive.as_uri())) return links
def find_links_for_package(self, package: Package) -> List[Link]: json_data = self._get("pypi/{}/{}/json".format(package.name, package.version)) if json_data is None: return [] links = [] for url in json_data["urls"]: h = "sha256={}".format(url["digests"]["sha256"]) links.append(Link(url["url"] + "#" + h)) return links
def find_links_for_package(self, package: Package) -> List[Link]: json_data = self._get(f"pypi/{package.name}/{package.version}/json") if json_data is None: return [] links = [] for url in json_data["urls"]: h = f"sha256={url['digests']['sha256']}" links.append(Link(url["url"] + "#" + h)) return links
def test_get_cached_archive_for_link(config, mocker): chef = Chef( config, MockEnv( version_info=(3, 8, 3), marker_env={ "interpreter_name": "cpython", "interpreter_version": "3.8.3" }, supported_tags=[ Tag("cp38", "cp38", "macosx_10_15_x86_64"), Tag("py3", "none", "any"), ], ), ) cwd = Path.cwd() / ".pypoetrycache" mocker.patch.object( chef, "get_cached_archives_for_link", return_value=[ Link(f"file:///{cwd}demo-0.1.0-py2.py3-none-any"), Link(f"file:///{cwd}demo-0.1.0.tar.gz"), Link(f"file:///{cwd}demo-0.1.0-cp38-cp38-macosx_10_15_x86_64.whl"), Link(f"file:///{cwd}demo-0.1.0-cp37-cp37-macosx_10_15_x86_64.whl"), ], ) archive = chef.get_cached_archive_for_link( Link("https://files.python-poetry.org/demo-0.1.0.tar.gz")) assert Link(f"file:///{cwd}demo-0.1.0-cp38-cp38-macosx_10_15_x86_64.whl" ) == archive
def links(self) -> Iterator[Link]: for anchor in self._parsed.findall(".//a"): if anchor.get("href"): href = anchor.get("href") url = self.clean_link(urllib.parse.urljoin(self._url, href)) pyrequire = anchor.get("data-requires-python") pyrequire = unescape(pyrequire) if pyrequire else None link = Link(url, self, requires_python=pyrequire) if link.ext not in self.SUPPORTED_FORMATS: continue yield link
def test_get_cache_directory_for_link(config): chef = Chef( config, MockEnv( marker_env={"interpreter_name": "cpython", "interpreter_version": "3.8.3"} ), ) directory = chef.get_cache_directory_for_link( Link("https://files.python-poetry.org/poetry-1.1.0.tar.gz") ) expected = Path( "/foo/artifacts/ba/63/13/283a3b3b7f95f05e9e6f84182d276f7bb0951d5b0cc24422b33f7a4648" ) assert expected == directory
def test_executor_should_not_write_pep610_url_references_for_cached_package( package: Package, mocker: MockerFixture, fixture_dir: FixtureDirGetter, tmp_venv: VirtualEnv, pool: Pool, config: Config, io: BufferedIO, ): link_cached = Link( fixture_dir("distributions").joinpath( "demo-0.1.0-py2.py3-none-any.whl").as_uri()) mocker.patch("poetry.installation.executor.Executor._download", return_value=link_cached) executor = Executor(tmp_venv, pool, config, io) executor.execute([Install(package)]) verify_installed_distribution(tmp_venv, package)
def link_version(self, link: Link) -> Optional[Version]: m = wheel_file_re.match(link.filename) if m: version = m.group("ver") else: info, ext = link.splitext() match = self.VERSION_REGEX.match(info) if not match: return None version = match.group(2) try: version = Version.parse(version) except ValueError: return None return version
def _install(self, operation: Install | Update) -> int: package = operation.package if package.source_type == "directory": return self._install_directory(operation) if package.source_type == "git": return self._install_git(operation) if package.source_type == "file": archive = self._prepare_file(operation) elif package.source_type == "url": archive = self._download_link(operation, Link(package.source_url)) else: archive = self._download(operation) operation_message = self.get_operation_message(operation) message = (f" <fg=blue;options=bold>•</> {operation_message}:" " <info>Installing...</info>") self._write(operation, message) return self.pip_install(archive, upgrade=operation.job_type == "update")
def test_executor_should_check_every_possible_hash_types_before_failing( config, io, pool, mocker, fixture_dir, tmp_dir ): mocker.patch.object( Chef, "get_cached_archive_for_link", side_effect=lambda link: link, ) mocker.patch.object( Executor, "_download_archive", return_value=fixture_dir("distributions").joinpath( "demo-0.1.0-py2.py3-none-any.whl" ), ) env = MockEnv(path=Path(tmp_dir)) executor = Executor(env, pool, config, io) package = Package("demo", "0.1.0") package.files = [ {"file": "demo-0.1.0-py2.py3-none-any.whl", "hash": "md5:123456"}, {"file": "demo-0.1.0-py2.py3-none-any.whl", "hash": "sha256:123456"}, ] expected_message = ( "Invalid hashes " "(" "md5:15507846fd4299596661d0197bfb4f90, " "sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a" ") " "for demo (0.1.0) using archive demo-0.1.0-py2.py3-none-any.whl. " "Expected one of md5:123456, sha256:123456." ) with pytest.raises(RuntimeError, match=re.escape(expected_message)): executor._download_link( Install(package), Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl"), )
def get_cached_archives_for_link(self, link): # type: (Link) -> List[Link] cache_dir = self.get_cache_directory_for_link(link) archive_types = ["whl", "tar.gz", "tar.bz2", "bz2", "zip"] links = [] for archive_type in archive_types: for archive in cache_dir.glob("*.{}".format(archive_type)): accept = True if link.hash: real_hash = get_file_hash(archive, link.hash_name) if real_hash != link.hash: logger = logging.getLogger(__name__) logger.warning("cache of {} is corrupted".format( link.filename)) accept = False if accept: links.append(Link(archive.as_uri())) return links
def _get_info_from_urls(self, urls: Dict[str, List[str]]) -> PackageInfo: # Checking wheels first as they are more likely to hold # the necessary information if "bdist_wheel" in urls: # Check fo a universal wheel wheels = urls["bdist_wheel"] universal_wheel = None universal_python2_wheel = None universal_python3_wheel = None platform_specific_wheels = [] for wheel in wheels: link = Link(wheel) m = wheel_file_re.match(link.filename) if not m: continue pyver = m.group("pyver") abi = m.group("abi") plat = m.group("plat") if abi == "none" and plat == "any": # Universal wheel if pyver == "py2.py3": # Any Python universal_wheel = wheel elif pyver == "py2": universal_python2_wheel = wheel else: universal_python3_wheel = wheel else: platform_specific_wheels.append(wheel) if universal_wheel is not None: return self._get_info_from_wheel(universal_wheel) info = None if universal_python2_wheel and universal_python3_wheel: info = self._get_info_from_wheel(universal_python2_wheel) py3_info = self._get_info_from_wheel(universal_python3_wheel) if py3_info.requires_dist: if not info.requires_dist: info.requires_dist = py3_info.requires_dist return info py2_requires_dist = set( dependency_from_pep_508(r).to_pep_508() for r in info.requires_dist) py3_requires_dist = set( dependency_from_pep_508(r).to_pep_508() for r in py3_info.requires_dist) base_requires_dist = py2_requires_dist & py3_requires_dist py2_only_requires_dist = py2_requires_dist - py3_requires_dist py3_only_requires_dist = py3_requires_dist - py2_requires_dist # Normalizing requires_dist requires_dist = list(base_requires_dist) for requirement in py2_only_requires_dist: dep = dependency_from_pep_508(requirement) dep.marker = dep.marker.intersect( parse_marker("python_version == '2.7'")) requires_dist.append(dep.to_pep_508()) for requirement in py3_only_requires_dist: dep = dependency_from_pep_508(requirement) dep.marker = dep.marker.intersect( parse_marker("python_version >= '3'")) requires_dist.append(dep.to_pep_508()) info.requires_dist = sorted(list(set(requires_dist))) if info: return info # Prefer non platform specific wheels if universal_python3_wheel: return self._get_info_from_wheel(universal_python3_wheel) if universal_python2_wheel: return self._get_info_from_wheel(universal_python2_wheel) if platform_specific_wheels and "sdist" not in urls: # Pick the first wheel available and hope for the best return self._get_info_from_wheel(platform_specific_wheels[0]) return self._get_info_from_sdist(urls["sdist"][0])
def make_url(ext): checksum = sha256(str(uuid.uuid4()).encode()) return Link("https://files.pythonhosted.org/packages/16/52/dead/" "demo-1.0.0.{}#sha256={}".format(ext, checksum))
def create_from_pep_508(cls, name: str, relative_to: Path | None = None) -> Dependency: """ Resolve a PEP-508 requirement string to a `Dependency` instance. If a `relative_to` path is specified, this is used as the base directory if the identified dependency is of file or directory type. """ from poetry.core.packages.url_dependency import URLDependency from poetry.core.packages.utils.link import Link from poetry.core.packages.utils.utils import is_archive_file from poetry.core.packages.utils.utils import is_installable_dir from poetry.core.packages.utils.utils import is_url from poetry.core.packages.utils.utils import path_to_url from poetry.core.packages.utils.utils import strip_extras from poetry.core.packages.utils.utils import url_to_path from poetry.core.packages.vcs_dependency import VCSDependency from poetry.core.utils.patterns import wheel_file_re from poetry.core.vcs.git import ParsedUrl from poetry.core.version.requirements import Requirement # Removing comments parts = name.split(" #", 1) name = parts[0].strip() if len(parts) > 1: rest = parts[1] if " ;" in rest: name += " ;" + rest.split(" ;", 1)[1] req = Requirement(name) name = req.name link = None if is_url(name): link = Link(name) elif req.url: link = Link(req.url) else: path_str = os.path.normpath(os.path.abspath(name)) p, extras = strip_extras(path_str) if os.path.isdir(p) and (os.path.sep in name or name.startswith(".")): if not is_installable_dir(p): raise ValueError( f"Directory {name!r} is not installable. File 'setup.py' " "not found.") link = Link(path_to_url(p)) elif is_archive_file(p): link = Link(path_to_url(p)) # it's a local file, dir, or url if link: is_file_uri = link.scheme == "file" is_relative_uri = is_file_uri and re.search(r"\.\./", link.url) # Handle relative file URLs if is_file_uri and is_relative_uri: path = Path(link.path) if relative_to: path = relative_to / path link = Link(path_to_url(path)) # wheel file version = None if link.is_wheel: m = wheel_file_re.match(link.filename) if not m: raise ValueError(f"Invalid wheel name: {link.filename}") name = m.group("name") version = m.group("ver") dep: Dependency | None = None if link.scheme.startswith("git+"): url = ParsedUrl.parse(link.url) dep = VCSDependency( name, "git", url.url, rev=url.rev, directory=url.subdirectory, extras=req.extras, ) elif link.scheme == "git": dep = VCSDependency(name, "git", link.url_without_fragment, extras=req.extras) elif link.scheme in ["http", "https"]: dep = URLDependency(name, link.url, extras=req.extras) elif is_file_uri: # handle RFC 8089 references path = url_to_path(req.url) dep = _make_file_or_dir_dep(name=name, path=path, base=relative_to, extras=req.extras) else: with suppress(ValueError): # this is a local path not using the file URI scheme dep = _make_file_or_dir_dep( name=name, path=Path(req.url), base=relative_to, extras=req.extras, ) if dep is None: dep = Dependency(name, version or "*", extras=req.extras) if version: dep._constraint = parse_constraint(version) else: constraint: VersionConstraint | str if req.pretty_constraint: constraint = req.constraint else: constraint = "*" dep = Dependency(name, constraint, extras=req.extras) if req.marker: dep.marker = req.marker return dep
def test_link_package_data(filename: str, expected: Package | None) -> None: link = Link(f"https://example.org/{filename}") assert LinkSource.link_package_data(link) == expected
def test_links_for_version( version_string: str, filenames: Iterable[str], link_source: LinkSource ) -> None: version = Version.parse(version_string) expected = {Link(f"{link_source.url}/{name}") for name in filenames} assert set(link_source.links_for_version("demo", version)) == expected