def search(self, query: str) -> List[Package]: results = [] search = {"q": query} response = requests.session().get(self._base_url + "search", params=search) content = parse(response.content, namespaceHTMLElements=False) for result in content.findall(".//*[@class='package-snippet']"): name = result.find("h3/*[@class='package-snippet__name']").text version = result.find( "h3/*[@class='package-snippet__version']").text if not name or not version: continue description = result.find( "p[@class='package-snippet__description']").text if not description: description = "" try: result = Package(name, version, description) result.description = to_str(description.strip()) results.append(result) except InvalidVersion: self._log( f'Unable to parse version "{version}" for the {name} package, skipping', level="debug", ) return results
def create_package_from_pep610(cls, distribution: metadata.Distribution) -> Package: path = Path(str(distribution._path)) # type: ignore[attr-defined] source_type = None source_url = None source_reference = None source_resolved_reference = None develop = False url_reference = json.loads( path.joinpath("direct_url.json").read_text(encoding="utf-8") ) if "archive_info" in url_reference: # File or URL distribution if url_reference["url"].startswith("file:"): # File distribution source_type = "file" source_url = url_to_path(url_reference["url"]).as_posix() else: # URL distribution source_type = "url" source_url = url_reference["url"] elif "dir_info" in url_reference: # Directory distribution source_type = "directory" source_url = url_to_path(url_reference["url"]).as_posix() develop = url_reference["dir_info"].get("editable", False) elif "vcs_info" in url_reference: # VCS distribution source_type = url_reference["vcs_info"]["vcs"] source_url = url_reference["url"] source_resolved_reference = url_reference["vcs_info"]["commit_id"] source_reference = url_reference["vcs_info"].get( "requested_revision", source_resolved_reference ) package = Package( distribution.metadata["name"], distribution.metadata["version"], source_type=source_type, source_url=source_url, source_reference=source_reference, source_resolved_reference=source_resolved_reference, develop=develop, ) package.description = distribution.metadata.get( # type: ignore[attr-defined] "summary", "", ) return package
def search(self, query: str) -> list[Package]: results = [] search = {"q": query} response = requests.session().get(self._base_url + "search", params=search) content = parse(response.content, namespaceHTMLElements=False) for result in content.findall(".//*[@class='package-snippet']"): name_element = result.find("h3/*[@class='package-snippet__name']") version_element = result.find("h3/*[@class='package-snippet__version']") if ( name_element is None or version_element is None or not name_element.text or not version_element.text ): continue name = name_element.text version = version_element.text description_element = result.find( "p[@class='package-snippet__description']" ) description = ( description_element.text if description_element is not None and description_element.text else "" ) try: package = Package(name, version) package.description = to_str(description.strip()) results.append(package) except InvalidVersion: self._log( f'Unable to parse version "{version}" for the {name} package,' " skipping", level="debug", ) return results
def locked_repository( self, with_dev_reqs: bool = False) -> poetry.repositories.Repository: """ Searches and returns a repository of locked packages. """ from poetry.factory import Factory if not self.is_locked(): return poetry.repositories.Repository() lock_data = self.lock_data packages = poetry.repositories.Repository() if with_dev_reqs: locked_packages = lock_data["package"] else: locked_packages = [ p for p in lock_data["package"] if p["category"] == "main" ] if not locked_packages: return packages for info in locked_packages: source = info.get("source", {}) source_type = source.get("type") url = source.get("url") if source_type in ["directory", "file"]: url = self._lock.path.parent.joinpath(url).resolve().as_posix() package = Package( info["name"], info["version"], info["version"], source_type=source_type, source_url=url, source_reference=source.get("reference"), source_resolved_reference=source.get("resolved_reference"), ) package.description = info.get("description", "") package.category = info.get("category", "main") package.groups = info.get("groups", ["default"]) package.optional = info["optional"] if "hashes" in lock_data["metadata"]: # Old lock so we create dummy files from the hashes package.files = [{ "name": h, "hash": h } for h in lock_data["metadata"]["hashes"][info["name"]]] else: package.files = lock_data["metadata"]["files"][info["name"]] package.python_versions = info["python-versions"] extras = info.get("extras", {}) if extras: for name, deps in extras.items(): package.extras[name] = [] for dep in deps: try: dependency = Dependency.create_from_pep_508(dep) except InvalidRequirement: # handle lock files with invalid PEP 508 m = re.match( r"^(.+?)(?:\[(.+?)])?(?:\s+\((.+)\))?$", dep) dep_name = m.group(1) extras = m.group(2) or "" constraint = m.group(3) or "*" dependency = Dependency(dep_name, constraint, extras=extras.split(",")) package.extras[name].append(dependency) if "marker" in info: package.marker = parse_marker(info["marker"]) else: # Compatibility for old locks if "requirements" in info: dep = Dependency("foo", "0.0.0") for name, value in info["requirements"].items(): if name == "python": dep.python_versions = value elif name == "platform": dep.platform = value split_dep = dep.to_pep_508(False).split(";") if len(split_dep) > 1: package.marker = parse_marker(split_dep[1].strip()) for dep_name, constraint in info.get("dependencies", {}).items(): root_dir = self._lock.path.parent if package.source_type == "directory": # root dir should be the source of the package relative to the lock path root_dir = Path(package.source_url) if isinstance(constraint, list): for c in constraint: package.add_dependency( Factory.create_dependency(dep_name, c, root_dir=root_dir)) continue package.add_dependency( Factory.create_dependency(dep_name, constraint, root_dir=root_dir)) if "develop" in info: package.develop = info["develop"] packages.add_package(package) return packages
def locked_repository(self, with_dev_reqs=False ): # type: (bool) -> poetry.repositories.Repository """ Searches and returns a repository of locked packages. """ if not self.is_locked(): return poetry.repositories.Repository() lock_data = self.lock_data packages = poetry.repositories.Repository() if with_dev_reqs: locked_packages = lock_data["package"] else: locked_packages = [ p for p in lock_data["package"] if p["category"] == "main" ] if not locked_packages: return packages for info in locked_packages: package = Package(info["name"], info["version"], info["version"]) package.description = info.get("description", "") package.category = info["category"] package.optional = info["optional"] if "hashes" in lock_data["metadata"]: # Old lock so we create dummy files from the hashes package.files = [{ "name": h, "hash": h } for h in lock_data["metadata"]["hashes"][info["name"]]] else: package.files = lock_data["metadata"]["files"][info["name"]] package.python_versions = info["python-versions"] extras = info.get("extras", {}) if extras: for name, deps in extras.items(): package.extras[name] = [] for dep in deps: m = re.match(r"^(.+?)(?:\s+\((.+)\))?$", dep) dep_name = m.group(1) constraint = m.group(2) or "*" package.extras[name].append( Dependency(dep_name, constraint)) if "marker" in info: package.marker = parse_marker(info["marker"]) else: # Compatibility for old locks if "requirements" in info: dep = Dependency("foo", "0.0.0") for name, value in info["requirements"].items(): if name == "python": dep.python_versions = value elif name == "platform": dep.platform = value split_dep = dep.to_pep_508(False).split(";") if len(split_dep) > 1: package.marker = parse_marker(split_dep[1].strip()) for dep_name, constraint in info.get("dependencies", {}).items(): if isinstance(constraint, list): for c in constraint: package.add_dependency(dep_name, c) continue package.add_dependency(dep_name, constraint) if "develop" in info: package.develop = info["develop"] if "source" in info: package.source_type = info["source"].get("type", "") package.source_url = info["source"]["url"] package.source_reference = info["source"]["reference"] packages.add_package(package) return packages
def create_package_from_distribution(cls, distribution: "Distribution", env: "Env") -> Package: # We first check for a direct_url.json file to determine # the type of package. path = Path(str(distribution._path)) if (path.name.endswith(".dist-info") and path.joinpath("direct_url.json").exists()): return cls.create_package_from_pep610(distribution) is_standard_package = env.is_path_relative_to_lib(path) source_type = None source_url = None source_reference = None source_resolved_reference = None if is_standard_package: if path.name.endswith(".dist-info"): paths = cls.get_package_paths( env=env, name=distribution.metadata["name"]) if paths: is_editable_package = False for src in paths: if cls.is_vcs_package(src, env): ( source_type, source_url, source_reference, ) = cls.get_package_vcs_properties_from_path(src) break if not (is_editable_package or env.is_path_relative_to_lib(src)): is_editable_package = True else: # TODO: handle multiple source directories? if is_editable_package: source_type = "directory" source_url = paths.pop().as_posix() else: if cls.is_vcs_package(path, env): ( source_type, source_url, source_reference, ) = cls.get_package_vcs_properties_from_path( env.path / "src" / canonicalize_name(distribution.metadata["name"])) else: # If not, it's a path dependency source_type = "directory" source_url = str(path.parent) package = Package( distribution.metadata["name"], distribution.metadata["version"], source_type=source_type, source_url=source_url, source_reference=source_reference, source_resolved_reference=source_resolved_reference, ) package.description = distribution.metadata.get("summary", "") return package
def to_package( self, name: Optional[str] = None, extras: Optional[List[str]] = None, root_dir: Optional[Path] = None, ) -> Package: """ Create a new `poetry.core.packages.package.Package` instance using metadata from this instance. :param name: Name to use for the package, if not specified name from this instance is used. :param extras: Extras to activate for this package. :param root_dir: Optional root directory to use for the package. If set, dependency strings will be parsed relative to this directory. """ name = name or self.name if not self.version: # The version could not be determined, so we raise an error since it is mandatory. raise RuntimeError( "Unable to retrieve the package version for {}".format(name)) package = Package( name=name, version=self.version, source_type=self._source_type, source_url=self._source_url, source_reference=self._source_reference, ) package.description = self.summary package.root_dir = root_dir package.python_versions = self.requires_python or "*" package.files = self.files if root_dir or (self._source_type in {"directory"} and self._source_url): # this is a local poetry project, this means we can extract "richer" requirement information # eg: development requirements etc. poetry_package = self._get_poetry_package( path=root_dir or self._source_url) if poetry_package: package.extras = poetry_package.extras package.requires = poetry_package.requires return package seen_requirements = set() for req in self.requires_dist or []: try: # Attempt to parse the PEP-508 requirement string dependency = Dependency.create_from_pep_508( req, relative_to=root_dir) except InvalidMarker: # Invalid marker, We strip the markers hoping for the best req = req.split(";")[0] dependency = Dependency.create_from_pep_508( req, relative_to=root_dir) except ValueError: # Likely unable to parse constraint so we skip it self._log( "Invalid constraint ({}) found in {}-{} dependencies, " "skipping".format(req, package.name, package.version), level="warning", ) continue if dependency.in_extras: # this dependency is required by an extra package for extra in dependency.in_extras: if extra not in package.extras: # this is the first time we encounter this extra for this package package.extras[extra] = [] package.extras[extra].append(dependency) req = dependency.to_pep_508(with_extras=True) if req not in seen_requirements: package.requires.append(dependency) seen_requirements.add(req) return package
def load(cls, env: Env) -> "InstalledRepository": """ Load installed packages. """ repo = cls() seen = set() for entry in reversed(env.sys_path): for distribution in sorted( metadata.distributions(path=[entry]), key=lambda d: str(d._path), ): if not distribution.metadata: continue name = distribution.metadata["name"] path = Path(str(distribution._path)) version = distribution.metadata["version"] package = Package(name, version, version) package.description = distribution.metadata.get("summary", "") if package.name in seen: continue try: path.relative_to(_VENDORS) except ValueError: pass else: continue seen.add(package.name) repo.add_package(package) is_standard_package = env.is_path_relative_to_lib(path) if is_standard_package: if path.name.endswith(".dist-info"): paths = cls.get_package_paths(env=env, name=package.pretty_name) if paths: is_editable_package = False for src in paths: if cls.is_vcs_package(src, env): cls.set_package_vcs_properties( package, env) break if not (is_editable_package or env.is_path_relative_to_lib(src)): is_editable_package = True else: # TODO: handle multiple source directories? if is_editable_package: package._source_type = "directory" package._source_url = paths.pop().as_posix( ) continue if cls.is_vcs_package(path, env): cls.set_package_vcs_properties(package, env) else: # If not, it's a path dependency package._source_type = "directory" package._source_url = str(path.parent) return repo
def to_package( self, name: str | None = None, extras: list[str] | None = None, root_dir: Path | None = None, ) -> Package: """ Create a new `poetry.core.packages.package.Package` instance using metadata from this instance. :param name: Name to use for the package, if not specified name from this instance is used. :param extras: Extras to activate for this package. :param root_dir: Optional root directory to use for the package. If set, dependency strings will be parsed relative to this directory. """ name = name or self.name if not name: raise RuntimeError("Unable to create package with no name") if not self.version: # The version could not be determined, so we raise an error since it is # mandatory. raise RuntimeError( f"Unable to retrieve the package version for {name}") package = Package( name=name, version=self.version, source_type=self._source_type, source_url=self._source_url, source_reference=self._source_reference, ) if self.summary is not None: package.description = self.summary package.root_dir = root_dir package.python_versions = self.requires_python or "*" package.files = self.files # If this is a local poetry project, we can extract "richer" requirement # information, eg: development requirements etc. if root_dir is not None: path = root_dir elif self._source_type == "directory" and self._source_url is not None: path = Path(self._source_url) else: path = None if path is not None: poetry_package = self._get_poetry_package(path=path) if poetry_package: package.extras = poetry_package.extras for dependency in poetry_package.requires: package.add_dependency(dependency) return package seen_requirements = set() for req in self.requires_dist or []: try: # Attempt to parse the PEP-508 requirement string dependency = Dependency.create_from_pep_508( req, relative_to=root_dir) except InvalidMarker: # Invalid marker, We strip the markers hoping for the best req = req.split(";")[0] dependency = Dependency.create_from_pep_508( req, relative_to=root_dir) except ValueError: # Likely unable to parse constraint so we skip it logger.debug( f"Invalid constraint ({req}) found in" f" {package.name}-{package.version} dependencies, skipping", ) continue if dependency.in_extras: # this dependency is required by an extra package for extra in dependency.in_extras: if extra not in package.extras: # this is the first time we encounter this extra for this # package package.extras[extra] = [] package.extras[extra].append(dependency) req = dependency.to_pep_508(with_extras=True) if req not in seen_requirements: package.add_dependency(dependency) seen_requirements.add(req) return package