def search(self, query: str) -> List[Package]:
results = []
search = {"q": query}
response = requests.session().get(self._base_url + "search",
params=search)
content = parse(response.content, namespaceHTMLElements=False)
for result in content.findall(".//*[@class='package-snippet']"):
name = result.find("h3/*[@class='package-snippet__name']").text
version = result.find(
"h3/*[@class='package-snippet__version']").text
if not name or not version:
continue
description = result.find(
"p[@class='package-snippet__description']").text
if not description:
description = ""
try:
result = Package(name, version, description)
result.description = to_str(description.strip())
results.append(result)
except InvalidVersion:
self._log(
f'Unable to parse version "{version}" for the {name} package, skipping',
level="debug",
)
return results
def create_package_from_pep610(cls, distribution: metadata.Distribution) -> Package:
path = Path(str(distribution._path)) # type: ignore[attr-defined]
source_type = None
source_url = None
source_reference = None
source_resolved_reference = None
develop = False
url_reference = json.loads(
path.joinpath("direct_url.json").read_text(encoding="utf-8")
)
if "archive_info" in url_reference:
# File or URL distribution
if url_reference["url"].startswith("file:"):
# File distribution
source_type = "file"
source_url = url_to_path(url_reference["url"]).as_posix()
else:
# URL distribution
source_type = "url"
source_url = url_reference["url"]
elif "dir_info" in url_reference:
# Directory distribution
source_type = "directory"
source_url = url_to_path(url_reference["url"]).as_posix()
develop = url_reference["dir_info"].get("editable", False)
elif "vcs_info" in url_reference:
# VCS distribution
source_type = url_reference["vcs_info"]["vcs"]
source_url = url_reference["url"]
source_resolved_reference = url_reference["vcs_info"]["commit_id"]
source_reference = url_reference["vcs_info"].get(
"requested_revision", source_resolved_reference
)
package = Package(
distribution.metadata["name"],
distribution.metadata["version"],
source_type=source_type,
source_url=source_url,
source_reference=source_reference,
source_resolved_reference=source_resolved_reference,
develop=develop,
)
package.description = distribution.metadata.get( # type: ignore[attr-defined]
"summary",
"",
)
return package
def search(self, query: str) -> list[Package]:
results = []
search = {"q": query}
response = requests.session().get(self._base_url + "search", params=search)
content = parse(response.content, namespaceHTMLElements=False)
for result in content.findall(".//*[@class='package-snippet']"):
name_element = result.find("h3/*[@class='package-snippet__name']")
version_element = result.find("h3/*[@class='package-snippet__version']")
if (
name_element is None
or version_element is None
or not name_element.text
or not version_element.text
):
continue
name = name_element.text
version = version_element.text
description_element = result.find(
"p[@class='package-snippet__description']"
)
description = (
description_element.text
if description_element is not None and description_element.text
else ""
)
try:
package = Package(name, version)
package.description = to_str(description.strip())
results.append(package)
except InvalidVersion:
self._log(
f'Unable to parse version "{version}" for the {name} package,'
" skipping",
level="debug",
)
return results
def locked_repository(
self,
with_dev_reqs: bool = False) -> poetry.repositories.Repository:
"""
Searches and returns a repository of locked packages.
"""
from poetry.factory import Factory
if not self.is_locked():
return poetry.repositories.Repository()
lock_data = self.lock_data
packages = poetry.repositories.Repository()
if with_dev_reqs:
locked_packages = lock_data["package"]
else:
locked_packages = [
p for p in lock_data["package"] if p["category"] == "main"
]
if not locked_packages:
return packages
for info in locked_packages:
source = info.get("source", {})
source_type = source.get("type")
url = source.get("url")
if source_type in ["directory", "file"]:
url = self._lock.path.parent.joinpath(url).resolve().as_posix()
package = Package(
info["name"],
info["version"],
info["version"],
source_type=source_type,
source_url=url,
source_reference=source.get("reference"),
source_resolved_reference=source.get("resolved_reference"),
)
package.description = info.get("description", "")
package.category = info.get("category", "main")
package.groups = info.get("groups", ["default"])
package.optional = info["optional"]
if "hashes" in lock_data["metadata"]:
# Old lock so we create dummy files from the hashes
package.files = [{
"name": h,
"hash": h
} for h in lock_data["metadata"]["hashes"][info["name"]]]
else:
package.files = lock_data["metadata"]["files"][info["name"]]
package.python_versions = info["python-versions"]
extras = info.get("extras", {})
if extras:
for name, deps in extras.items():
package.extras[name] = []
for dep in deps:
try:
dependency = Dependency.create_from_pep_508(dep)
except InvalidRequirement:
# handle lock files with invalid PEP 508
m = re.match(
r"^(.+?)(?:\[(.+?)])?(?:\s+\((.+)\))?$", dep)
dep_name = m.group(1)
extras = m.group(2) or ""
constraint = m.group(3) or "*"
dependency = Dependency(dep_name,
constraint,
extras=extras.split(","))
package.extras[name].append(dependency)
if "marker" in info:
package.marker = parse_marker(info["marker"])
else:
# Compatibility for old locks
if "requirements" in info:
dep = Dependency("foo", "0.0.0")
for name, value in info["requirements"].items():
if name == "python":
dep.python_versions = value
elif name == "platform":
dep.platform = value
split_dep = dep.to_pep_508(False).split(";")
if len(split_dep) > 1:
package.marker = parse_marker(split_dep[1].strip())
for dep_name, constraint in info.get("dependencies", {}).items():
root_dir = self._lock.path.parent
if package.source_type == "directory":
# root dir should be the source of the package relative to the lock path
root_dir = Path(package.source_url)
if isinstance(constraint, list):
for c in constraint:
package.add_dependency(
Factory.create_dependency(dep_name,
c,
root_dir=root_dir))
continue
package.add_dependency(
Factory.create_dependency(dep_name,
constraint,
root_dir=root_dir))
if "develop" in info:
package.develop = info["develop"]
packages.add_package(package)
return packages
def locked_repository(self,
with_dev_reqs=False
): # type: (bool) -> poetry.repositories.Repository
"""
Searches and returns a repository of locked packages.
"""
if not self.is_locked():
return poetry.repositories.Repository()
lock_data = self.lock_data
packages = poetry.repositories.Repository()
if with_dev_reqs:
locked_packages = lock_data["package"]
else:
locked_packages = [
p for p in lock_data["package"] if p["category"] == "main"
]
if not locked_packages:
return packages
for info in locked_packages:
package = Package(info["name"], info["version"], info["version"])
package.description = info.get("description", "")
package.category = info["category"]
package.optional = info["optional"]
if "hashes" in lock_data["metadata"]:
# Old lock so we create dummy files from the hashes
package.files = [{
"name": h,
"hash": h
} for h in lock_data["metadata"]["hashes"][info["name"]]]
else:
package.files = lock_data["metadata"]["files"][info["name"]]
package.python_versions = info["python-versions"]
extras = info.get("extras", {})
if extras:
for name, deps in extras.items():
package.extras[name] = []
for dep in deps:
m = re.match(r"^(.+?)(?:\s+\((.+)\))?$", dep)
dep_name = m.group(1)
constraint = m.group(2) or "*"
package.extras[name].append(
Dependency(dep_name, constraint))
if "marker" in info:
package.marker = parse_marker(info["marker"])
else:
# Compatibility for old locks
if "requirements" in info:
dep = Dependency("foo", "0.0.0")
for name, value in info["requirements"].items():
if name == "python":
dep.python_versions = value
elif name == "platform":
dep.platform = value
split_dep = dep.to_pep_508(False).split(";")
if len(split_dep) > 1:
package.marker = parse_marker(split_dep[1].strip())
for dep_name, constraint in info.get("dependencies", {}).items():
if isinstance(constraint, list):
for c in constraint:
package.add_dependency(dep_name, c)
continue
package.add_dependency(dep_name, constraint)
if "develop" in info:
package.develop = info["develop"]
if "source" in info:
package.source_type = info["source"].get("type", "")
package.source_url = info["source"]["url"]
package.source_reference = info["source"]["reference"]
packages.add_package(package)
return packages
def create_package_from_distribution(cls, distribution: "Distribution",
env: "Env") -> Package:
# We first check for a direct_url.json file to determine
# the type of package.
path = Path(str(distribution._path))
if (path.name.endswith(".dist-info")
and path.joinpath("direct_url.json").exists()):
return cls.create_package_from_pep610(distribution)
is_standard_package = env.is_path_relative_to_lib(path)
source_type = None
source_url = None
source_reference = None
source_resolved_reference = None
if is_standard_package:
if path.name.endswith(".dist-info"):
paths = cls.get_package_paths(
env=env, name=distribution.metadata["name"])
if paths:
is_editable_package = False
for src in paths:
if cls.is_vcs_package(src, env):
(
source_type,
source_url,
source_reference,
) = cls.get_package_vcs_properties_from_path(src)
break
if not (is_editable_package
or env.is_path_relative_to_lib(src)):
is_editable_package = True
else:
# TODO: handle multiple source directories?
if is_editable_package:
source_type = "directory"
source_url = paths.pop().as_posix()
else:
if cls.is_vcs_package(path, env):
(
source_type,
source_url,
source_reference,
) = cls.get_package_vcs_properties_from_path(
env.path / "src" /
canonicalize_name(distribution.metadata["name"]))
else:
# If not, it's a path dependency
source_type = "directory"
source_url = str(path.parent)
package = Package(
distribution.metadata["name"],
distribution.metadata["version"],
source_type=source_type,
source_url=source_url,
source_reference=source_reference,
source_resolved_reference=source_resolved_reference,
)
package.description = distribution.metadata.get("summary", "")
return package
def to_package(
self,
name: Optional[str] = None,
extras: Optional[List[str]] = None,
root_dir: Optional[Path] = None,
) -> Package:
"""
Create a new `poetry.core.packages.package.Package` instance using metadata from this instance.
:param name: Name to use for the package, if not specified name from this instance is used.
:param extras: Extras to activate for this package.
:param root_dir: Optional root directory to use for the package. If set, dependency strings
will be parsed relative to this directory.
"""
name = name or self.name
if not self.version:
# The version could not be determined, so we raise an error since it is mandatory.
raise RuntimeError(
"Unable to retrieve the package version for {}".format(name))
package = Package(
name=name,
version=self.version,
source_type=self._source_type,
source_url=self._source_url,
source_reference=self._source_reference,
)
package.description = self.summary
package.root_dir = root_dir
package.python_versions = self.requires_python or "*"
package.files = self.files
if root_dir or (self._source_type in {"directory"}
and self._source_url):
# this is a local poetry project, this means we can extract "richer" requirement information
# eg: development requirements etc.
poetry_package = self._get_poetry_package(
path=root_dir or self._source_url)
if poetry_package:
package.extras = poetry_package.extras
package.requires = poetry_package.requires
return package
seen_requirements = set()
for req in self.requires_dist or []:
try:
# Attempt to parse the PEP-508 requirement string
dependency = Dependency.create_from_pep_508(
req, relative_to=root_dir)
except InvalidMarker:
# Invalid marker, We strip the markers hoping for the best
req = req.split(";")[0]
dependency = Dependency.create_from_pep_508(
req, relative_to=root_dir)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="warning",
)
continue
if dependency.in_extras:
# this dependency is required by an extra package
for extra in dependency.in_extras:
if extra not in package.extras:
# this is the first time we encounter this extra for this package
package.extras[extra] = []
package.extras[extra].append(dependency)
req = dependency.to_pep_508(with_extras=True)
if req not in seen_requirements:
package.requires.append(dependency)
seen_requirements.add(req)
return package
def load(cls, env: Env) -> "InstalledRepository":
"""
Load installed packages.
"""
repo = cls()
seen = set()
for entry in reversed(env.sys_path):
for distribution in sorted(
metadata.distributions(path=[entry]),
key=lambda d: str(d._path),
):
if not distribution.metadata:
continue
name = distribution.metadata["name"]
path = Path(str(distribution._path))
version = distribution.metadata["version"]
package = Package(name, version, version)
package.description = distribution.metadata.get("summary", "")
if package.name in seen:
continue
try:
path.relative_to(_VENDORS)
except ValueError:
pass
else:
continue
seen.add(package.name)
repo.add_package(package)
is_standard_package = env.is_path_relative_to_lib(path)
if is_standard_package:
if path.name.endswith(".dist-info"):
paths = cls.get_package_paths(env=env,
name=package.pretty_name)
if paths:
is_editable_package = False
for src in paths:
if cls.is_vcs_package(src, env):
cls.set_package_vcs_properties(
package, env)
break
if not (is_editable_package
or env.is_path_relative_to_lib(src)):
is_editable_package = True
else:
# TODO: handle multiple source directories?
if is_editable_package:
package._source_type = "directory"
package._source_url = paths.pop().as_posix(
)
continue
if cls.is_vcs_package(path, env):
cls.set_package_vcs_properties(package, env)
else:
# If not, it's a path dependency
package._source_type = "directory"
package._source_url = str(path.parent)
return repo
def to_package(
self,
name: str | None = None,
extras: list[str] | None = None,
root_dir: Path | None = None,
) -> Package:
"""
Create a new `poetry.core.packages.package.Package` instance using metadata from
this instance.
:param name: Name to use for the package, if not specified name from this
instance is used.
:param extras: Extras to activate for this package.
:param root_dir: Optional root directory to use for the package. If set,
dependency strings will be parsed relative to this directory.
"""
name = name or self.name
if not name:
raise RuntimeError("Unable to create package with no name")
if not self.version:
# The version could not be determined, so we raise an error since it is
# mandatory.
raise RuntimeError(
f"Unable to retrieve the package version for {name}")
package = Package(
name=name,
version=self.version,
source_type=self._source_type,
source_url=self._source_url,
source_reference=self._source_reference,
)
if self.summary is not None:
package.description = self.summary
package.root_dir = root_dir
package.python_versions = self.requires_python or "*"
package.files = self.files
# If this is a local poetry project, we can extract "richer" requirement
# information, eg: development requirements etc.
if root_dir is not None:
path = root_dir
elif self._source_type == "directory" and self._source_url is not None:
path = Path(self._source_url)
else:
path = None
if path is not None:
poetry_package = self._get_poetry_package(path=path)
if poetry_package:
package.extras = poetry_package.extras
for dependency in poetry_package.requires:
package.add_dependency(dependency)
return package
seen_requirements = set()
for req in self.requires_dist or []:
try:
# Attempt to parse the PEP-508 requirement string
dependency = Dependency.create_from_pep_508(
req, relative_to=root_dir)
except InvalidMarker:
# Invalid marker, We strip the markers hoping for the best
req = req.split(";")[0]
dependency = Dependency.create_from_pep_508(
req, relative_to=root_dir)
except ValueError:
# Likely unable to parse constraint so we skip it
logger.debug(
f"Invalid constraint ({req}) found in"
f" {package.name}-{package.version} dependencies, skipping",
)
continue
if dependency.in_extras:
# this dependency is required by an extra package
for extra in dependency.in_extras:
if extra not in package.extras:
# this is the first time we encounter this extra for this
# package
package.extras[extra] = []
package.extras[extra].append(dependency)
req = dependency.to_pep_508(with_extras=True)
if req not in seen_requirements:
package.add_dependency(dependency)
seen_requirements.add(req)
return package