def test_store_get_the_correct_store(self):
cache = CacheManager({
'default': 'dict',
'stores': {
'dict': {
'driver': 'dict'
},
'file': {
'driver': 'file',
'path': os.path.join(tempfile.gettempdir(), 'cachy')
}
}
})
self.assertIsInstance(cache.store().get_store(), DictStore)
self.assertIsInstance(cache.store('dict').get_store(), DictStore)
self.assertIsInstance(cache.store('file').get_store(), FileStore)
def test_store_get_the_correct_store(self):
cache = CacheManager({
'default': 'dict',
'stores': {
'dict': {
'driver': 'dict'
},
'file': {
'driver': 'file',
'path': os.path.join(tempfile.gettempdir(), 'cachy')
}
}
})
self.assertIsInstance(cache.store().get_store(), DictStore)
self.assertIsInstance(cache.store('dict').get_store(), DictStore)
self.assertIsInstance(cache.store('file').get_store(), FileStore)
def test_set_default_driver_changes_driver(self):
cache = CacheManager({
'default': 'dict',
'stores': {
'dict': {
'driver': 'dict'
},
'file': {
'driver': 'file',
'path': os.path.join(tempfile.gettempdir(), 'cachy')
}
}
})
self.assertIsInstance(cache.store().get_store(), DictStore)
cache.set_default_driver('file')
self.assertIsInstance(cache.store().get_store(), FileStore)
def test_set_default_driver_changes_driver(self):
cache = CacheManager({
'default': 'dict',
'stores': {
'dict': {
'driver': 'dict'
},
'file': {
'driver': 'file',
'path': os.path.join(tempfile.gettempdir(), 'cachy')
}
}
})
self.assertIsInstance(cache.store().get_store(), DictStore)
cache.set_default_driver('file')
self.assertIsInstance(cache.store().get_store(), FileStore)
def test_extend_accepts_a_callable_returning_a_repository(self):
cache = CacheManager({
'default': 'my-driver',
'stores': {
'my-driver': {
'driver': 'my-driver'
}
}
})
cache.extend('my-driver', lambda config: Repository(CustomStore()))
self.assertIsInstance(cache.store().get_store(), CustomStore)
def test_extend_accepts_a_callable_returning_a_repository(self):
cache = CacheManager({
'default': 'my-driver',
'stores': {
'my-driver': {
'driver': 'my-driver'
}
}
})
cache.extend('my-driver', lambda config: Repository(CustomStore()))
self.assertIsInstance(cache.store().get_store(), CustomStore)
def test_extend_accepts_a_store_class(self):
cache = CacheManager({
'default': 'my-driver',
'stores': {
'my-driver': {
'driver': 'my-driver'
}
}
})
cache.extend('my-driver', CustomStore)
self.assertIsInstance(cache.store().get_store(), CustomStore)
def test_extend_accepts_a_store_class(self):
cache = CacheManager({
'default': 'my-driver',
'stores': {
'my-driver': {
'driver': 'my-driver'
}
}
})
cache.extend('my-driver', CustomStore)
self.assertIsInstance(cache.store().get_store(), CustomStore)
class PyPiRepository(RemoteRepository):
CACHE_VERSION = parse_constraint("1.0.0")
def __init__(
self,
url: str = "https://pypi.org/",
disable_cache: bool = False,
fallback: bool = True,
) -> None:
super(PyPiRepository, self).__init__(url.rstrip("/") + "/simple/")
self._base_url = url
self._disable_cache = disable_cache
self._fallback = fallback
release_cache_dir = REPOSITORY_CACHE_DIR / "pypi"
self._cache = CacheManager({
"default": "releases",
"serializer": "json",
"stores": {
"releases": {
"driver": "file",
"path": str(release_cache_dir)
},
"packages": {
"driver": "dict"
},
},
})
self._cache_control_cache = FileCache(str(release_cache_dir / "_http"))
inner_session = requests.Session()
retries = Retry(total=5,
backoff_factor=1,
status_forcelist=[502, 503, 504])
inner_session.mount(self._base_url, HTTPAdapter(max_retries=retries))
self._session = CacheControl(inner_session,
cache=self._cache_control_cache)
self._name = "PyPI"
@property
def session(self) -> CacheControl:
return self._session
def find_packages(self, dependency: Dependency) -> List[Package]:
"""
Find packages on the remote server.
"""
constraint = dependency.constraint
if constraint is None:
constraint = "*"
if not isinstance(constraint, VersionConstraint):
constraint = parse_constraint(constraint)
allow_prereleases = dependency.allows_prereleases()
if isinstance(constraint, VersionRange):
if (constraint.max is not None and constraint.max.is_prerelease()
or constraint.min is not None
and constraint.min.is_prerelease()):
allow_prereleases = True
try:
info = self.get_package_info(dependency.name)
except PackageNotFound:
self._log(
"No packages found for {} {}".format(dependency.name,
str(constraint)),
level="debug",
)
return []
packages = []
ignored_pre_release_packages = []
for version, release in info["releases"].items():
if not release:
# Bad release
self._log(
"No release information found for {}-{}, skipping".format(
dependency.name, version),
level="debug",
)
continue
try:
package = Package(info["info"]["name"], version)
except ParseVersionError:
self._log(
'Unable to parse version "{}" for the {} package, skipping'
.format(version, dependency.name),
level="debug",
)
continue
if package.is_prerelease() and not allow_prereleases:
if constraint.is_any():
# we need this when all versions of the package are pre-releases
ignored_pre_release_packages.append(package)
continue
if not constraint or (constraint
and constraint.allows(package.version)):
packages.append(package)
self._log(
"{} packages found for {} {}".format(len(packages),
dependency.name,
str(constraint)),
level="debug",
)
return packages or ignored_pre_release_packages
def package(
self,
name: str,
version: str,
extras: (Union[list, None]) = None,
) -> Package:
return self.get_release_info(name, version).to_package(name=name,
extras=extras)
def search(self, query: str) -> List[Package]:
results = []
search = {"q": query}
response = requests.session().get(self._base_url + "search",
params=search)
content = parse(response.content, namespaceHTMLElements=False)
for result in content.findall(".//*[@class='package-snippet']"):
name = result.find("h3/*[@class='package-snippet__name']").text
version = result.find(
"h3/*[@class='package-snippet__version']").text
if not name or not version:
continue
description = result.find(
"p[@class='package-snippet__description']").text
if not description:
description = ""
try:
result = Package(name, version, description)
result.description = to_str(description.strip())
results.append(result)
except ParseVersionError:
self._log(
'Unable to parse version "{}" for the {} package, skipping'
.format(version, name),
level="debug",
)
return results
def get_package_info(self, name: str) -> dict:
"""
Return the package information given its name.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_package_info(name)
return self._cache.store("packages").remember_forever(
name, lambda: self._get_package_info(name))
def _get_package_info(self, name: str) -> dict:
data = self._get("pypi/{}/json".format(name))
if data is None:
raise PackageNotFound("Package [{}] not found.".format(name))
return data
def get_release_info(self, name: str, version: str) -> PackageInfo:
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return PackageInfo.load(self._get_release_info(name, version))
cached = self._cache.remember_forever(
"{}:{}".format(name, version),
lambda: self._get_release_info(name, version))
cache_version = cached.get("_cache_version", "0.0.0")
if parse_constraint(cache_version) != self.CACHE_VERSION:
# The cache must be updated
self._log(
"The cache for {} {} is outdated. Refreshing.".format(
name, version),
level="debug",
)
cached = self._get_release_info(name, version)
self._cache.forever("{}:{}".format(name, version), cached)
return PackageInfo.load(cached)
def find_links_for_package(self, package: Package) -> List[Link]:
json_data = self._get("pypi/{}/{}/json".format(package.name,
package.version))
if json_data is None:
return []
links = []
for url in json_data["urls"]:
h = "sha256={}".format(url["digests"]["sha256"])
links.append(Link(url["url"] + "#" + h))
return links
def _get_release_info(self, name: str, version: str) -> dict:
self._log("Getting info for {} ({}) from PyPI".format(name, version),
"debug")
json_data = self._get("pypi/{}/{}/json".format(name, version))
if json_data is None:
raise PackageNotFound("Package [{}] not found.".format(name))
info = json_data["info"]
data = PackageInfo(
name=info["name"],
version=info["version"],
summary=info["summary"],
platform=info["platform"],
requires_dist=info["requires_dist"],
requires_python=info["requires_python"],
files=info.get("files", []),
cache_version=str(self.CACHE_VERSION),
)
try:
version_info = json_data["releases"][version]
except KeyError:
version_info = []
for file_info in version_info:
data.files.append({
"file":
file_info["filename"],
"hash":
"sha256:" + file_info["digests"]["sha256"],
})
if self._fallback and data.requires_dist is None:
self._log("No dependencies found, downloading archives",
level="debug")
# No dependencies set (along with other information)
# This might be due to actually no dependencies
# or badly set metadata when uploading
# So, we need to make sure there is actually no
# dependencies by introspecting packages
urls = defaultdict(list)
for url in json_data["urls"]:
# Only get sdist and wheels if they exist
dist_type = url["packagetype"]
if dist_type not in ["sdist", "bdist_wheel"]:
continue
urls[dist_type].append(url["url"])
if not urls:
return data.asdict()
info = self._get_info_from_urls(urls)
data.requires_dist = info.requires_dist
if not data.requires_python:
data.requires_python = info.requires_python
return data.asdict()
def _get(self, endpoint: str) -> Union[dict, None]:
try:
json_response = self.session.get(self._base_url + endpoint)
except requests.exceptions.TooManyRedirects:
# Cache control redirect loop.
# We try to remove the cache and try again
self._cache_control_cache.delete(self._base_url + endpoint)
json_response = self.session.get(self._base_url + endpoint)
if json_response.status_code == 404:
return None
json_data = json_response.json()
return json_data
def _get_info_from_urls(self, urls: Dict[str, List[str]]) -> PackageInfo:
# Checking wheels first as they are more likely to hold
# the necessary information
if "bdist_wheel" in urls:
# Check fo a universal wheel
wheels = urls["bdist_wheel"]
universal_wheel = None
universal_python2_wheel = None
universal_python3_wheel = None
platform_specific_wheels = []
for wheel in wheels:
link = Link(wheel)
m = wheel_file_re.match(link.filename)
if not m:
continue
pyver = m.group("pyver")
abi = m.group("abi")
plat = m.group("plat")
if abi == "none" and plat == "any":
# Universal wheel
if pyver == "py2.py3":
# Any Python
universal_wheel = wheel
elif pyver == "py2":
universal_python2_wheel = wheel
else:
universal_python3_wheel = wheel
else:
platform_specific_wheels.append(wheel)
if universal_wheel is not None:
return self._get_info_from_wheel(universal_wheel)
info = None
if universal_python2_wheel and universal_python3_wheel:
info = self._get_info_from_wheel(universal_python2_wheel)
py3_info = self._get_info_from_wheel(universal_python3_wheel)
if py3_info.requires_dist:
if not info.requires_dist:
info.requires_dist = py3_info.requires_dist
return info
py2_requires_dist = set(
dependency_from_pep_508(r).to_pep_508()
for r in info.requires_dist)
py3_requires_dist = set(
dependency_from_pep_508(r).to_pep_508()
for r in py3_info.requires_dist)
base_requires_dist = py2_requires_dist & py3_requires_dist
py2_only_requires_dist = py2_requires_dist - py3_requires_dist
py3_only_requires_dist = py3_requires_dist - py2_requires_dist
# Normalizing requires_dist
requires_dist = list(base_requires_dist)
for requirement in py2_only_requires_dist:
dep = dependency_from_pep_508(requirement)
dep.marker = dep.marker.intersect(
parse_marker("python_version == '2.7'"))
requires_dist.append(dep.to_pep_508())
for requirement in py3_only_requires_dist:
dep = dependency_from_pep_508(requirement)
dep.marker = dep.marker.intersect(
parse_marker("python_version >= '3'"))
requires_dist.append(dep.to_pep_508())
info.requires_dist = sorted(list(set(requires_dist)))
if info:
return info
# Prefer non platform specific wheels
if universal_python3_wheel:
return self._get_info_from_wheel(universal_python3_wheel)
if universal_python2_wheel:
return self._get_info_from_wheel(universal_python2_wheel)
if platform_specific_wheels and "sdist" not in urls:
# Pick the first wheel available and hope for the best
return self._get_info_from_wheel(platform_specific_wheels[0])
return self._get_info_from_sdist(urls["sdist"][0])
def _get_info_from_wheel(self, url: str) -> PackageInfo:
self._log(
"Downloading wheel: {}".format(
urllib.parse.urlparse(url).path.rsplit("/")[-1]),
level="debug",
)
filename = os.path.basename(
urllib.parse.urlparse(url).path.rsplit("/")[-1])
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
return PackageInfo.from_wheel(filepath)
def _get_info_from_sdist(self, url: str) -> PackageInfo:
self._log(
"Downloading sdist: {}".format(
urllib.parse.urlparse(url).path.rsplit("/")[-1]),
level="debug",
)
filename = os.path.basename(urllib.parse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
return PackageInfo.from_sdist(filepath)
def _download(self, url: str, dest: str) -> None:
return download_file(url, dest, session=self.session)
def _log(self, msg: str, level: str = "info") -> None:
getattr(logger, level)("<debug>{}:</debug> {}".format(self._name, msg))
class LegacyRepository(PyPiRepository):
def __init__(
self, name, url, auth=None, disable_cache=False
): # type: (str, str, Optional[Auth], bool) -> None
if name == "pypi":
raise ValueError("The name [pypi] is reserved for repositories")
self._packages = []
self._name = name
self._url = url.rstrip("/")
self._cache_dir = Path(CACHE_DIR) / "cache" / "repositories" / name
self._cache = CacheManager(
{
"default": "releases",
"serializer": "json",
"stores": {
"releases": {"driver": "file", "path": str(self._cache_dir)},
"packages": {"driver": "dict"},
"matches": {"driver": "dict"},
},
}
)
self._session = CacheControl(
requests.session(), cache=FileCache(str(self._cache_dir / "_http"))
)
url_parts = urlparse.urlparse(self._url)
if not url_parts.username and auth:
self._session.auth = auth
self._disable_cache = disable_cache
@property
def name(self):
return self._name
def find_packages(
self, name, constraint=None, extras=None, allow_prereleases=False
):
packages = []
if constraint is None:
constraint = "*"
if not isinstance(constraint, VersionConstraint):
constraint = parse_constraint(constraint)
if isinstance(constraint, VersionRange):
if (
constraint.max is not None
and constraint.max.is_prerelease()
or constraint.min is not None
and constraint.min.is_prerelease()
):
allow_prereleases = True
key = name
if not constraint.is_any():
key = "{}:{}".format(key, str(constraint))
if self._cache.store("matches").has(key):
versions = self._cache.store("matches").get(key)
else:
page = self._get("/{}/".format(canonicalize_name(name).replace(".", "-")))
if page is None:
return []
versions = []
for version in page.versions:
if version.is_prerelease() and not allow_prereleases:
continue
if constraint.allows(version):
versions.append(version)
self._cache.store("matches").put(key, versions, 5)
for version in versions:
package = Package(name, version)
package.source_type = "legacy"
package.source_url = self._url
if extras is not None:
package.requires_extras = extras
packages.append(package)
self._log(
"{} packages found for {} {}".format(len(packages), name, str(constraint)),
level="debug",
)
return packages
def package(
self, name, version, extras=None
): # type: (...) -> poetry.packages.Package
"""
Retrieve the release information.
This is a heavy task which takes time.
We have to download a package to get the dependencies.
We also need to download every file matching this release
to get the various hashes.
Note that, this will be cached so the subsequent operations
should be much faster.
"""
try:
index = self._packages.index(
poetry.packages.Package(name, version, version)
)
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = poetry.packages.Package(name, version, version)
if release_info["requires_python"]:
package.python_versions = release_info["requires_python"]
package.source_type = "legacy"
package.source_url = self._url
package.source_reference = self.name
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.in_extras:
for extra in dependency.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get("summary", "")
# Adding hashes information
package.hashes = release_info["digests"]
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
def _get_release_info(self, name, version): # type: (str, str) -> dict
page = self._get("/{}/".format(canonicalize_name(name).replace(".", "-")))
if page is None:
raise PackageNotFound('No package named "{}"'.format(name))
data = {
"name": name,
"version": version,
"summary": "",
"requires_dist": [],
"requires_python": None,
"digests": [],
}
links = list(page.links_for_version(Version.parse(version)))
if not links:
raise PackageNotFound(
'No valid distribution links found for package: "{}" version: "{}"'.format(
name, version
)
)
urls = {}
hashes = []
default_link = links[0]
for link in links:
if link.is_wheel:
m = wheel_file_re.match(default_link.filename)
python = m.group("pyver")
platform = m.group("plat")
if python == "py2.py3" and platform == "any":
urls["bdist_wheel"] = default_link.url
elif link.filename.endswith(".tar.gz"):
urls["sdist"] = link.url
elif (
link.filename.endswith((".zip", ".bz2", ".xz", ".Z", ".tar"))
and "sdist" not in urls
):
urls["sdist"] = link.url
hash = link.hash
if link.hash_name == "sha256":
hashes.append(hash)
data["digests"] = hashes
if not urls:
if default_link.is_wheel:
urls["bdist_wheel"] = default_link.url
elif default_link.filename.endswith(".tar.gz"):
urls["sdist"] = default_link.url
elif (
default_link.filename.endswith((".zip", ".bz2")) and "sdist" not in urls
):
urls["sdist"] = default_link.url
else:
return data
info = self._get_info_from_urls(urls)
data["summary"] = info["summary"]
data["requires_dist"] = info["requires_dist"]
data["requires_python"] = info["requires_python"]
return data
def _download(self, url, dest): # type: (str, str) -> None
r = self._session.get(url, stream=True)
with open(dest, "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
def _get(self, endpoint): # type: (str) -> Union[Page, None]
url = self._url + endpoint
response = self._session.get(url)
if response.status_code == 404:
return
return Page(url, response.content, response.headers)
class PyPiRepository(Repository):
def __init__(self,
url="https://pypi.org/",
disable_cache=False,
fallback=True):
self._name = "PyPI"
self._url = url
self._disable_cache = disable_cache
self._fallback = fallback
release_cache_dir = Path(CACHE_DIR) / "cache" / "repositories" / "pypi"
self._cache = CacheManager({
"default": "releases",
"serializer": "json",
"stores": {
"releases": {
"driver": "file",
"path": str(release_cache_dir)
},
"packages": {
"driver": "dict"
},
},
})
self._session = CacheControl(session(),
cache=FileCache(
str(release_cache_dir / "_http")))
super(PyPiRepository, self).__init__()
def find_packages(
self,
name, # type: str
constraint=None, # type: Union[VersionConstraint, str, None]
extras=None, # type: Union[list, None]
allow_prereleases=False, # type: bool
): # type: (...) -> List[Package]
"""
Find packages on the remote server.
"""
if constraint is None:
constraint = "*"
if not isinstance(constraint, VersionConstraint):
constraint = parse_constraint(constraint)
info = self.get_package_info(name)
packages = []
for version, release in info["releases"].items():
if not release:
# Bad release
self._log(
"No release information found for {}-{}, skipping".format(
name, version),
level="debug",
)
continue
package = Package(name, version)
if (package.is_prerelease() and not allow_prereleases
and not constraint.allows(package.version)):
continue
if not constraint or (constraint
and constraint.allows(package.version)):
if extras is not None:
package.requires_extras = extras
packages.append(package)
self._log(
"{} packages found for {} {}".format(len(packages), name,
str(constraint)),
level="debug",
)
return packages
def package(
self,
name, # type: str
version, # type: str
extras=None, # type: (Union[list, None])
): # type: (...) -> Union[Package, None]
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.extras:
for extra in dependency.extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get("summary", "")
if release_info["requires_python"]:
package.python_versions = release_info["requires_python"]
if release_info["platform"]:
package.platform = release_info["platform"]
# Adding hashes information
package.hashes = release_info["digests"]
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
return package
def search(self, query, mode=0):
results = []
search = {"name": query}
if mode == self.SEARCH_FULLTEXT:
search["summary"] = query
client = ServerProxy("https://pypi.python.org/pypi")
hits = client.search(search, "or")
for hit in hits:
result = Package(hit["name"], hit["version"], hit["version"])
result.description = to_str(hit["summary"])
results.append(result)
return results
def get_package_info(self, name): # type: (str) -> dict
"""
Return the package information given its name.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_package_info(name)
return self._cache.store("packages").remember_forever(
name, lambda: self._get_package_info(name))
def _get_package_info(self, name): # type: (str) -> dict
data = self._get("pypi/{}/json".format(name))
if data is None:
raise ValueError("Package [{}] not found.".format(name))
return data
def get_release_info(self, name, version): # type: (str, str) -> dict
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_release_info(name, version)
return self._cache.remember_forever(
"{}:{}".format(name, version),
lambda: self._get_release_info(name, version))
def _get_release_info(self, name, version): # type: (str, str) -> dict
self._log("Getting info for {} ({}) from PyPI".format(name, version),
"debug")
json_data = self._get("pypi/{}/{}/json".format(name, version))
if json_data is None:
raise ValueError("Package [{}] not found.".format(name))
info = json_data["info"]
data = {
"name": info["name"],
"version": info["version"],
"summary": info["summary"],
"platform": info["platform"],
"requires_dist": info["requires_dist"],
"requires_python": info["requires_python"],
"digests": [],
"_fallback": False,
}
try:
version_info = json_data["releases"][version]
except KeyError:
version_info = []
for file_info in version_info:
data["digests"].append(file_info["digests"]["sha256"])
if self._fallback and data["requires_dist"] is None:
self._log("No dependencies found, downloading archives",
level="debug")
# No dependencies set (along with other information)
# This might be due to actually no dependencies
# or badly set metadata when uploading
# So, we need to make sure there is actually no
# dependencies by introspecting packages
urls = {}
for url in json_data["urls"]:
# Only get sdist and universal wheels if they exist
dist_type = url["packagetype"]
if dist_type not in ["sdist", "bdist_wheel"]:
continue
if dist_type == "sdist" and "sdist" not in urls:
urls[url["packagetype"]] = url["url"]
continue
if "bdist_wheel" in urls:
continue
# If bdist_wheel, check if it's universal
filename = url["filename"]
if not re.search("-py2\.py3-none-any.whl", filename):
continue
urls[dist_type] = url["url"]
if "sdist" in url and "bdist_wheel" not in urls:
# If can't found a universal wheel
# but we found an sdist, inspect the sdist first
info = self._get_info_from_urls(urls)
if info["requires_dist"]:
data["requires_dist"] = info["requires_dist"]
if not data["requires_python"]:
data["requires_python"] = info["requires_python"]
return data
else:
del urls["sdist"]
if not urls:
# If we don't have urls, we try to take the first one
# we find and go from there
if not json_data["urls"]:
return data
for url in json_data["urls"]:
# Only get sdist and universal wheels if they exist
dist_type = url["packagetype"]
if dist_type != "bdist_wheel":
continue
urls[url["packagetype"]] = url["url"]
break
if not urls:
return data
info = self._get_info_from_urls(urls)
data["requires_dist"] = info["requires_dist"]
if not data["requires_python"]:
data["requires_python"] = info["requires_python"]
return data
def _get(self, endpoint): # type: (str) -> Union[dict, None]
json_response = self._session.get(self._url + endpoint)
if json_response.status_code == 404:
return None
json_data = json_response.json()
return json_data
def _get_info_from_urls(
self, urls
): # type: (Dict[str, str]) -> Dict[str, Union[str, List, None]]
if "bdist_wheel" in urls:
self._log(
"Downloading wheel: {}".format(
urls["bdist_wheel"].split("/")[-1]),
level="debug",
)
return self._get_info_from_wheel(urls["bdist_wheel"])
self._log("Downloading sdist: {}".format(urls["sdist"].split("/")[-1]),
level="debug")
return self._get_info_from_sdist(urls["sdist"])
def _get_info_from_wheel(
self, url): # type: (str) -> Dict[str, Union[str, List, None]]
info = {"summary": "", "requires_python": None, "requires_dist": None}
filename = os.path.basename(urlparse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = os.path.join(temp_dir, filename)
self._download(url, filepath)
try:
meta = pkginfo.Wheel(filepath)
except ValueError:
# Unable to determine dependencies
# Assume none
return info
if meta.summary:
info["summary"] = meta.summary or ""
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = meta.requires_dist
return info
def _get_info_from_sdist(
self, url): # type: (str) -> Dict[str, Union[str, List, None]]
info = {"summary": "", "requires_python": None, "requires_dist": None}
filename = os.path.basename(urlparse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
try:
meta = pkginfo.SDist(str(filepath))
if meta.summary:
info["summary"] = meta.summary
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
return info
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
# Still not dependencies found
# So, we unpack and introspect
suffix = filepath.suffix
gz = None
if suffix == ".zip":
tar = zipfile.ZipFile(str(filepath))
else:
if suffix == ".bz2":
gz = BZ2File(str(filepath))
else:
gz = GzipFile(str(filepath))
suffix = ".tar.gz"
tar = tarfile.TarFile(str(filepath), fileobj=gz)
try:
tar.extractall(os.path.join(temp_dir, "unpacked"))
finally:
if gz:
gz.close()
tar.close()
unpacked = Path(temp_dir) / "unpacked"
sdist_dir = unpacked / Path(filename).name.rstrip(suffix)
# Checking for .egg-info at root
eggs = list(sdist_dir.glob("*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Searching for .egg-info in sub directories
eggs = list(sdist_dir.glob("**/*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Still nothing, assume no dependencies
# We could probably get them by executing
# python setup.py egg-info but I don't feel
# confortable executing a file just for the sake
# of getting dependencies.
return info
def _inspect_sdist_with_setup(self, sdist_dir):
info = {"requires_python": None, "requires_dist": None}
setup = sdist_dir / "setup.py"
if not setup.exists():
return info
venv = Venv.create(NullIO())
current_dir = os.getcwd()
os.chdir(sdist_dir.as_posix())
try:
venv.run("python", "setup.py", "egg_info")
egg_info = list(sdist_dir.glob("**/*.egg-info"))[0]
meta = pkginfo.UnpackedSDist(str(egg_info))
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
else:
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
info["requires_dist"] = parse_requires(f.read())
except Exception:
pass
os.chdir(current_dir)
return info
def _download(self, url, dest): # type: (str, str) -> None
r = get(url, stream=True)
with open(dest, "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
def _log(self, msg, level="info"):
getattr(logger,
level)("<comment>{}:</comment> {}".format(self._name, msg))
class LegacyRepository(PyPiRepository):
def __init__(self, name, url):
if name == 'pypi':
raise ValueError('The name [pypi] is reserved for repositories')
self._packages = []
self._name = name
self._url = url
command = get_pip_command()
opts, _ = command.parse_args([])
self._session = command._build_session(opts)
self._repository = PyPIRepository(opts, self._session)
self._cache_dir = Path(CACHE_DIR) / 'cache' / 'repositories' / name
self._cache = CacheManager({
'default': 'releases',
'serializer': 'json',
'stores': {
'releases': {
'driver': 'file',
'path': str(self._cache_dir)
},
'packages': {
'driver': 'dict'
},
'matches': {
'driver': 'dict'
}
}
})
@property
def name(self):
return self._name
def find_packages(self,
name,
constraint=None,
extras=None,
allow_prereleases=False):
packages = []
if constraint is not None and not isinstance(constraint,
BaseConstraint):
version_parser = VersionParser()
constraint = version_parser.parse_constraints(constraint)
key = name
if constraint:
key = '{}:{}'.format(key, str(constraint))
if self._cache.store('matches').has(key):
versions = self._cache.store('matches').get(key)
else:
candidates = [
str(c.version)
for c in self._repository.find_all_candidates(name)
]
versions = []
for version in candidates:
if version in versions:
continue
if (not constraint
or (constraint
and constraint.matches(Constraint('=', version)))):
versions.append(version)
self._cache.store('matches').put(key, versions, 5)
for version in versions:
packages.append(Package(name, version, extras=extras))
return packages
def package(self,
name,
version,
extras=None): # type: (...) -> poetry.packages.Package
"""
Retrieve the release information.
This is a heavy task which takes time.
We have to download a package to get the dependencies.
We also need to download every file matching this release
to get the various hashes.
Note that, this will be cached so the subsequent operations
should be much faster.
"""
try:
index = self._packages.index(
poetry.packages.Package(name, version, version))
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = poetry.packages.Package(name, version, version)
for req in release_info['requires_dist']:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(';')[0]
dependency = dependency_from_pep_508(req)
if dependency.extras:
for extra in dependency.extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get('summary', '')
# Adding hashes information
package.hashes = release_info['digests']
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
def get_release_info(self, name, version): # type: (str, str) -> dict
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
return self._cache.store('releases').remember_forever(
'{}:{}'.format(name, version),
lambda: self._get_release_info(name, version))
def _get_release_info(self, name, version): # type: (str, str) -> dict
ireq = InstallRequirement.from_line('{}=={}'.format(name, version))
resolver = Resolver([ireq],
self._repository,
cache=DependencyCache(self._cache_dir.as_posix()))
try:
requirements = list(resolver._iter_dependencies(ireq))
except (InstallationError, RequirementParseError):
# setup.py egg-info error most likely
# So we assume no dependencies
requirements = []
requires = []
for dep in requirements:
constraint = str(dep.req.specifier)
require = dep.name
if constraint:
require += ' ({})'.format(constraint)
requires.append(require)
try:
hashes = resolver.resolve_hashes([ireq])[ireq]
except IndexError:
# Sometimes pip-tools fails when getting indices
hashes = []
hashes = [h.split(':')[1] for h in hashes]
data = {
'name': name,
'version': version,
'summary': '',
'requires_dist': requires,
'digests': hashes
}
resolver.repository.freshen_build_caches()
return data
class LegacyRepository(PyPiRepository):
def __init__(
self,
name: str,
url: str,
config: Optional[Config] = None,
disable_cache: bool = False,
cert: Optional[Path] = None,
client_cert: Optional[Path] = None,
) -> None:
if name == "pypi":
raise ValueError("The name [pypi] is reserved for repositories")
self._packages = []
self._name = name
self._url = url.rstrip("/")
self._client_cert = client_cert
self._cert = cert
self._cache_dir = REPOSITORY_CACHE_DIR / name
self._cache = CacheManager({
"default": "releases",
"serializer": "json",
"stores": {
"releases": {
"driver": "file",
"path": str(self._cache_dir)
},
"packages": {
"driver": "dict"
},
"matches": {
"driver": "dict"
},
},
})
self._authenticator = Authenticator(
config=config or Config(use_environment=True))
self._session = CacheControl(self._authenticator.session,
cache=FileCache(
str(self._cache_dir / "_http")))
username, password = self._authenticator.get_credentials_for_url(
self._url)
if username is not None and password is not None:
self._authenticator.session.auth = requests.auth.HTTPBasicAuth(
username, password)
if self._cert:
self._authenticator.session.verify = str(self._cert)
if self._client_cert:
self._authenticator.session.cert = str(self._client_cert)
self._disable_cache = disable_cache
@property
def cert(self) -> Optional[Path]:
return self._cert
@property
def client_cert(self) -> Optional[Path]:
return self._client_cert
@property
def authenticated_url(self) -> str:
if not self._session.auth:
return self.url
parsed = urllib.parse.urlparse(self.url)
return "{scheme}://{username}:{password}@{netloc}{path}".format(
scheme=parsed.scheme,
username=quote(self._session.auth.username, safe=""),
password=quote(self._session.auth.password, safe=""),
netloc=parsed.netloc,
path=parsed.path,
)
def find_packages(self, dependency: "Dependency") -> List[Package]:
packages = []
constraint = dependency.constraint
if constraint is None:
constraint = "*"
if not isinstance(constraint, VersionConstraint):
constraint = parse_constraint(constraint)
allow_prereleases = dependency.allows_prereleases()
if isinstance(constraint, VersionRange):
if (constraint.max is not None and constraint.max.is_unstable()
or constraint.min is not None
and constraint.min.is_unstable()):
allow_prereleases = True
key = dependency.name
if not constraint.is_any():
key = "{}:{}".format(key, str(constraint))
ignored_pre_release_versions = []
if self._cache.store("matches").has(key):
versions = self._cache.store("matches").get(key)
else:
page = self._get_page("/{}/".format(
dependency.name.replace(".", "-")))
if page is None:
return []
versions = []
for version in page.versions:
if version.is_unstable() and not allow_prereleases:
if constraint.is_any():
# we need this when all versions of the package are pre-releases
ignored_pre_release_versions.append(version)
continue
if constraint.allows(version):
versions.append(version)
self._cache.store("matches").put(key, versions, 5)
for package_versions in (versions, ignored_pre_release_versions):
for version in package_versions:
package = Package(
dependency.name,
version,
source_type="legacy",
source_reference=self.name,
source_url=self._url,
)
packages.append(package)
self._log(
"{} packages found for {} {}".format(len(packages),
dependency.name,
str(constraint)),
level="debug",
)
if packages or not constraint.is_any():
# we have matching packages, or constraint is not (*)
break
return packages
def package(self,
name: str,
version: str,
extras: Optional[List[str]] = None) -> Package:
"""
Retrieve the release information.
This is a heavy task which takes time.
We have to download a package to get the dependencies.
We also need to download every file matching this release
to get the various hashes.
Note that this will be cached so the subsequent operations
should be much faster.
"""
try:
index = self._packages.index(Package(name, version, version))
return self._packages[index]
except ValueError:
package = super().package(name, version, extras)
package._source_type = "legacy"
package._source_url = self._url
package._source_reference = self.name
return package
def find_links_for_package(self, package: Package) -> List[Link]:
page = self._get_page("/{}/".format(package.name.replace(".", "-")))
if page is None:
return []
return list(page.links_for_version(package.version))
def _get_release_info(self, name: str, version: str) -> dict:
page = self._get_page("/{}/".format(
canonicalize_name(name).replace(".", "-")))
if page is None:
raise PackageNotFound(f'No package named "{name}"')
data = PackageInfo(
name=name,
version=version,
summary="",
platform=None,
requires_dist=[],
requires_python=None,
files=[],
cache_version=str(self.CACHE_VERSION),
)
links = list(page.links_for_version(Version.parse(version)))
if not links:
raise PackageNotFound(
'No valid distribution links found for package: "{}" version: "{}"'
.format(name, version))
urls = defaultdict(list)
files = []
for link in links:
if link.is_wheel:
urls["bdist_wheel"].append(link.url)
elif link.filename.endswith(
(".tar.gz", ".zip", ".bz2", ".xz", ".Z", ".tar")):
urls["sdist"].append(link.url)
file_hash = f"{link.hash_name}:{link.hash}" if link.hash else None
if not link.hash or (link.hash_name
not in ("sha256", "sha384", "sha512")
and hasattr(hashlib, link.hash_name)):
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / link.filename
self._download(link.url, str(filepath))
known_hash = (getattr(hashlib, link.hash_name)()
if link.hash_name else None)
required_hash = hashlib.sha256()
chunksize = 4096
with filepath.open("rb") as f:
while True:
chunk = f.read(chunksize)
if not chunk:
break
if known_hash:
known_hash.update(chunk)
required_hash.update(chunk)
if not known_hash or known_hash.hexdigest() == link.hash:
file_hash = "{}:{}".format(required_hash.name,
required_hash.hexdigest())
files.append({"file": link.filename, "hash": file_hash})
data.files = files
info = self._get_info_from_urls(urls)
data.summary = info.summary
data.requires_dist = info.requires_dist
data.requires_python = info.requires_python
return data.asdict()
def _get_page(self, endpoint: str) -> Optional[Page]:
url = self._url + endpoint
try:
response = self.session.get(url)
if response.status_code in (401, 403):
self._log(
f"Authorization error accessing {url}",
level="warning",
)
return None
if response.status_code == 404:
return None
response.raise_for_status()
except requests.HTTPError as e:
raise RepositoryError(e)
if response.url != url:
self._log(
"Response URL {response_url} differs from request URL {url}".
format(response_url=response.url, url=url),
level="debug",
)
return Page(response.url, response.content, response.headers)
def _download(self, url, dest): # type: (str, str) -> None
return download_file(url, dest, session=self.session)
class LegacyRepository(PyPiRepository):
def __init__(self, name, url):
if name == "pypi":
raise ValueError("The name [pypi] is reserved for repositories")
self._packages = []
self._name = name
self._url = url.rstrip("/")
self._cache_dir = Path(CACHE_DIR) / "cache" / "repositories" / name
self._cache = CacheManager({
"default": "releases",
"serializer": "json",
"stores": {
"releases": {
"driver": "file",
"path": str(self._cache_dir)
},
"packages": {
"driver": "dict"
},
"matches": {
"driver": "dict"
},
},
})
self._session = CacheControl(requests.session(),
cache=FileCache(
str(self._cache_dir / "_http")))
@property
def name(self):
return self._name
def find_packages(self,
name,
constraint=None,
extras=None,
allow_prereleases=False):
packages = []
if constraint is not None and not isinstance(constraint,
VersionConstraint):
constraint = parse_constraint(constraint)
key = name
if constraint:
key = "{}:{}".format(key, str(constraint))
if self._cache.store("matches").has(key):
versions = self._cache.store("matches").get(key)
else:
page = self._get("/{}".format(
canonicalize_name(name).replace(".", "-")))
if page is None:
raise ValueError('No package named "{}"'.format(name))
versions = []
for version in page.versions:
if not constraint or (constraint
and constraint.allows(version)):
versions.append(version)
self._cache.store("matches").put(key, versions, 5)
for version in versions:
package = Package(name, version)
if extras is not None:
package.requires_extras = extras
packages.append(package)
return packages
def package(self,
name,
version,
extras=None): # type: (...) -> poetry.packages.Package
"""
Retrieve the release information.
This is a heavy task which takes time.
We have to download a package to get the dependencies.
We also need to download every file matching this release
to get the various hashes.
Note that, this will be cached so the subsequent operations
should be much faster.
"""
try:
index = self._packages.index(
poetry.packages.Package(name, version, version))
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = poetry.packages.Package(name, version, version)
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
if dependency.extras:
for extra in dependency.extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get("summary", "")
# Adding hashes information
package.hashes = release_info["digests"]
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
def get_release_info(self, name, version): # type: (str, str) -> dict
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
return self._cache.store("releases").remember_forever(
"{}:{}".format(name, version),
lambda: self._get_release_info(name, version))
def _get_release_info(self, name, version): # type: (str, str) -> dict
page = self._get("/{}".format(
canonicalize_name(name).replace(".", "-")))
if page is None:
raise ValueError('No package named "{}"'.format(name))
data = {
"name": name,
"version": version,
"summary": "",
"requires_dist": [],
"requires_python": [],
"digests": [],
}
links = list(page.links_for_version(Version.parse(version)))
urls = {}
hashes = []
default_link = links[0]
for link in links:
if link.is_wheel:
urls["bdist_wheel"] = link.url
elif link.filename.endswith(".tar.gz"):
urls["sdist"] = link.url
elif link.filename.endswith(
(".zip", ".bz2")) and "sdist" not in urls:
urls["sdist"] = link.url
hash = link.hash
if link.hash_name == "sha256":
hashes.append(hash)
data["digests"] = hashes
if not urls:
if default_link.is_wheel:
m = wheel_file_re.match(default_link.filename)
python = m.group("pyver")
platform = m.group("plat")
if python == "py2.py3" and platform == "any":
urls["bdist_wheel"] = default_link.url
elif default_link.filename.endswith(".tar.gz"):
urls["sdist"] = default_link.url
elif (default_link.filename.endswith((".zip", ".bz2"))
and "sdist" not in urls):
urls["sdist"] = default_link.url
else:
return data
info = self._get_info_from_urls(urls)
data["summary"] = info["summary"]
data["requires_dist"] = info["requires_dist"]
data["requires_python"] = info["requires_python"]
return data
def _get(self, endpoint): # type: (str) -> Union[Page, None]
url = self._url + endpoint
response = self._session.get(url)
if response.status_code == 404:
return
return Page(url, response.content, response.headers)
class LegacyRepository(PyPiRepository):
def __init__(
self, name, url, auth=None, disable_cache=False, cert=None, client_cert=None
): # type: (str, str, Optional[Auth], bool, Optional[Path], Optional[Path]) -> None
if name == "pypi":
raise ValueError("The name [pypi] is reserved for repositories")
self._packages = []
self._name = name
self._url = url.rstrip("/")
self._auth = auth
self._client_cert = client_cert
self._cert = cert
self._cache_dir = REPOSITORY_CACHE_DIR / name
self._cache = CacheManager(
{
"default": "releases",
"serializer": "json",
"stores": {
"releases": {"driver": "file", "path": str(self._cache_dir)},
"packages": {"driver": "dict"},
"matches": {"driver": "dict"},
},
}
)
self._session = CacheControl(
requests.session(), cache=FileCache(str(self._cache_dir / "_http"))
)
url_parts = urlparse.urlparse(self._url)
if not url_parts.username and self._auth:
self._session.auth = self._auth
if self._cert:
self._session.verify = str(self._cert)
if self._client_cert:
self._session.cert = str(self._client_cert)
self._disable_cache = disable_cache
@property
def cert(self): # type: () -> Optional[Path]
return self._cert
@property
def client_cert(self): # type: () -> Optional[Path]
return self._client_cert
@property
def authenticated_url(self): # type: () -> str
if not self._auth:
return self.url
parsed = urlparse.urlparse(self.url)
return "{scheme}://{username}:{password}@{netloc}{path}".format(
scheme=parsed.scheme,
username=quote(self._auth.auth.username, safe=""),
password=quote(self._auth.auth.password, safe=""),
netloc=parsed.netloc,
path=parsed.path,
)
def find_packages(self, dependency):
packages = []
constraint = dependency.constraint
if constraint is None:
constraint = "*"
if not isinstance(constraint, VersionConstraint):
constraint = parse_constraint(constraint)
allow_prereleases = dependency.allows_prereleases()
if isinstance(constraint, VersionRange):
if (
constraint.max is not None
and constraint.max.is_prerelease()
or constraint.min is not None
and constraint.min.is_prerelease()
):
allow_prereleases = True
key = dependency.name
if not constraint.is_any():
key = "{}:{}".format(key, str(constraint))
ignored_pre_release_versions = []
if self._cache.store("matches").has(key):
versions = self._cache.store("matches").get(key)
else:
page = self._get("/{}/".format(dependency.name.replace(".", "-")))
if page is None:
return []
versions = []
for version in page.versions:
if version.is_prerelease() and not allow_prereleases:
if constraint.is_any():
# we need this when all versions of the package are pre-releases
ignored_pre_release_versions.append(version)
continue
if constraint.allows(version):
versions.append(version)
self._cache.store("matches").put(key, versions, 5)
for package_versions in (versions, ignored_pre_release_versions):
for version in package_versions:
package = Package(
dependency.name,
version,
source_type="legacy",
source_reference=self.name,
source_url=self._url,
)
packages.append(package)
self._log(
"{} packages found for {} {}".format(
len(packages), dependency.name, str(constraint)
),
level="debug",
)
if packages or not constraint.is_any():
# we have matching packages, or constraint is not (*)
break
return packages
def package(self, name, version, extras=None): # type: (...) -> Package
"""
Retrieve the release information.
This is a heavy task which takes time.
We have to download a package to get the dependencies.
We also need to download every file matching this release
to get the various hashes.
Note that this will be cached so the subsequent operations
should be much faster.
"""
try:
index = self._packages.index(Package(name, version, version))
return self._packages[index]
except ValueError:
package = super(LegacyRepository, self).package(name, version, extras)
package._source_type = "legacy"
package._source_url = self._url
package._source_reference = self.name
return package
def find_links_for_package(self, package):
page = self._get("/{}/".format(package.name.replace(".", "-")))
if page is None:
return []
return list(page.links_for_version(package.version))
def _get_release_info(self, name, version): # type: (str, str) -> dict
page = self._get("/{}/".format(canonicalize_name(name).replace(".", "-")))
if page is None:
raise PackageNotFound('No package named "{}"'.format(name))
data = PackageInfo(
name=name,
version=version,
summary="",
platform=None,
requires_dist=[],
requires_python=None,
files=[],
cache_version=str(self.CACHE_VERSION),
)
links = list(page.links_for_version(Version.parse(version)))
if not links:
raise PackageNotFound(
'No valid distribution links found for package: "{}" version: "{}"'.format(
name, version
)
)
urls = defaultdict(list)
files = []
for link in links:
if link.is_wheel:
urls["bdist_wheel"].append(link.url)
elif link.filename.endswith(
(".tar.gz", ".zip", ".bz2", ".xz", ".Z", ".tar")
):
urls["sdist"].append(link.url)
h = link.hash
if h:
h = link.hash_name + ":" + link.hash
files.append({"file": link.filename, "hash": h})
data.files = files
info = self._get_info_from_urls(urls)
data.summary = info.summary
data.requires_dist = info.requires_dist
data.requires_python = info.requires_python
return data.asdict()
def _get(self, endpoint): # type: (str) -> Union[Page, None]
url = self._url + endpoint
try:
response = self.session.get(url)
if response.status_code == 404:
return
response.raise_for_status()
except requests.HTTPError as e:
raise RepositoryError(e)
return Page(url, response.content, response.headers)
class PyPiRepository(Repository):
def __init__(self, url='https://pypi.org/', disable_cache=False):
self._url = url
self._disable_cache = disable_cache
self._cache = CacheManager({
'default': 'releases',
'serializer': 'json',
'stores': {
'releases': {
'driver': 'file',
'path': Path(CACHE_DIR) / 'cache' / 'repositories' / 'pypi'
},
'packages': {
'driver': 'dict'
}
}
})
super().__init__()
def find_packages(self,
name: str,
constraint: Union[Constraint, str, None] = None,
extras: Union[list, None] = None) -> List[Package]:
"""
Find packages on the remote server.
"""
packages = []
if constraint is not None and not isinstance(constraint,
BaseConstraint):
version_parser = VersionParser()
constraint = version_parser.parse_constraints(constraint)
info = self.get_package_info(name)
versions = []
for version, release in info['releases'].items():
if (not constraint or
(constraint and constraint.matches(Constraint('=', version)))):
versions.append(version)
for version in versions:
packages.append(self.package(name, version, extras=extras))
return packages
def package(self,
name: str,
version: str,
extras: Union[list, None] = None) -> Package:
try:
index = self._packages.index(Package(name, version, version))
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
for req in release_info['requires_dist']:
req = InstallRequirement.from_line(req)
name = req.name
version = str(req.req.specifier)
dependency = Dependency(name, version, optional=req.markers)
is_extra = False
if req.markers:
# Setting extra dependencies and requirements
requirements = self._convert_markers(req.markers._markers)
if 'python_version' in requirements:
ors = []
for or_ in requirements['python_version']:
ands = []
for op, version in or_:
ands.append(f'{op}{version}')
ors.append(' '.join(ands))
dependency.python_versions = ' || '.join(ors)
if 'sys_platform' in requirements:
ors = []
for or_ in requirements['sys_platform']:
ands = []
for op, platform in or_:
ands.append(f'{op}{platform}')
ors.append(' '.join(ands))
dependency.platform = ' || '.join(ors)
if 'extra' in requirements:
is_extra = True
for _extras in requirements['extra']:
for _, extra in _extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not is_extra:
package.requires.append(dependency)
# Adding description
package.description = release_info.get('summary', '')
# Adding hashes information
package.hashes = release_info['digests']
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
def search(self, query, mode=0):
results = []
search = {'name': query}
if mode == self.SEARCH_FULLTEXT:
search['summary'] = query
client = ServerProxy(self._url)
hits = client.search(search, 'or')
for hit in hits:
results.append({
'name': hit['name'],
'description': hit['summary'],
'version': hit['version']
})
return results
def get_package_info(self, name: str) -> dict:
"""
Return the package information given its name.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_package_info(name)
return self._cache.store('packages').remember_forever(
f'{name}', lambda: self._get_package_info(name))
def _get_package_info(self, name: str) -> dict:
data = self._get(self._url + f'pypi/{name}/json')
if data is None:
raise ValueError(f'Package [{name}] not found.')
return data
def get_release_info(self, name: str, version: str) -> dict:
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_release_info(name, version)
return self._cache.remember_forever(
f'{name}:{version}', lambda: self._get_release_info(name, version))
def _get_release_info(self, name: str, version: str) -> dict:
json_data = self._get(self._url + f'pypi/{name}/{version}/json')
if json_data is None:
raise ValueError(f'Package [{name}] not found.')
info = json_data['info']
data = {
'name': info['name'],
'version': info['version'],
'summary': info['summary'],
'platform': info['platform'],
'requires_dist': info['requires_dist'],
'requires_python': info['requires_python'],
'digests': []
}
for file_info in json_data['releases'][version]:
data['digests'].append(file_info['digests']['sha256'])
return data
def _get(self, url: str) -> Union[dict, None]:
json_response = get(url)
if json_response.status_code == 404:
return None
json_data = json_response.json()
return json_data
def _group_markers(self, markers):
groups = [[]]
for marker in markers:
assert isinstance(marker, (list, tuple, str))
if isinstance(marker, list):
groups[-1].append(self._group_markers(marker))
elif isinstance(marker, tuple):
lhs, op, rhs = marker
groups[-1].append((lhs.value, op, rhs.value))
else:
assert marker in ["and", "or"]
if marker == "or":
groups.append([])
return groups
def _convert_markers(self, markers):
groups = self._group_markers(markers)[0]
requirements = {}
def _group(_groups, or_=False):
nonlocal requirements
for group in _groups:
if isinstance(group, tuple):
variable, op, value = group
group_name = str(variable)
if group_name not in requirements:
requirements[group_name] = [[]]
elif or_:
requirements[group_name].append([])
requirements[group_name][-1].append((str(op), str(value)))
else:
_group(group, or_=True)
_group(groups)
return requirements
class PyPiRepository(Repository):
def __init__(self, url='https://pypi.org/', disable_cache=False):
self._url = url
self._disable_cache = disable_cache
release_cache_dir = Path(CACHE_DIR) / 'cache' / 'repositories' / 'pypi'
self._cache = CacheManager({
'default': 'releases',
'serializer': 'json',
'stores': {
'releases': {
'driver': 'file',
'path': str(release_cache_dir)
},
'packages': {
'driver': 'dict'
}
}
})
self._session = CacheControl(session(),
cache=FileCache(
str(release_cache_dir / '_packages')))
super().__init__()
def find_packages(self,
name: str,
constraint: Union[Constraint, str, None] = None,
extras: Union[list, None] = None) -> List[Package]:
"""
Find packages on the remote server.
"""
packages = []
if constraint is not None and not isinstance(constraint,
BaseConstraint):
version_parser = VersionParser()
constraint = version_parser.parse_constraints(constraint)
info = self.get_package_info(name)
versions = []
for version, release in info['releases'].items():
if (not constraint or
(constraint and constraint.matches(Constraint('=', version)))):
versions.append(version)
for version in versions:
packages.append(self.package(name, version, extras=extras))
return packages
def package(self,
name: str,
version: str,
extras: Union[list, None] = None) -> Package:
try:
index = self._packages.index(Package(name, version, version))
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
requires_dist = release_info['requires_dist'] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(';')[0]
dependency = dependency_from_pep_508(req)
if dependency.extras:
for extra in dependency.extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get('summary', '')
# Adding hashes information
package.hashes = release_info['digests']
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
def search(self, query, mode=0):
results = []
search = {'name': query}
if mode == self.SEARCH_FULLTEXT:
search['summary'] = query
client = ServerProxy(self._url)
hits = client.search(search, 'or')
for hit in hits:
results.append({
'name': hit['name'],
'description': hit['summary'],
'version': hit['version']
})
return results
def get_package_info(self, name: str) -> dict:
"""
Return the package information given its name.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_package_info(name)
return self._cache.store('packages').remember_forever(
name, lambda: self._get_package_info(name))
def _get_package_info(self, name: str) -> dict:
data = self._get('pypi/{}/json'.format(name))
if data is None:
raise ValueError('Package [{}] not found.'.format(name))
return data
def get_release_info(self, name: str, version: str) -> dict:
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_release_info(name, version)
return self._cache.remember_forever(
'{}:{}'.format(name, version),
lambda: self._get_release_info(name, version))
def _get_release_info(self, name: str, version: str) -> dict:
json_data = self._get('pypi/{}/{}/json'.format(name, version))
if json_data is None:
raise ValueError('Package [{}] not found.'.format(name))
info = json_data['info']
data = {
'name': info['name'],
'version': info['version'],
'summary': info['summary'],
'platform': info['platform'],
'requires_dist': info['requires_dist'],
'requires_python': info['requires_python'],
'digests': []
}
for file_info in json_data['releases'][version]:
data['digests'].append(file_info['digests']['sha256'])
return data
def _get(self, endpoint: str) -> Union[dict, None]:
json_response = self._session.get(self._url + endpoint)
if json_response.status_code == 404:
return None
json_data = json_response.json()
return json_data
class LegacyRepository(PyPiRepository):
def __init__(self, name, url, disable_cache=False):
if name == "pypi":
raise ValueError("The name [pypi] is reserved for repositories")
self._packages = []
self._name = name
self._url = url.rstrip("/")
self._cache_dir = Path(CACHE_DIR) / "cache" / "repositories" / name
self._cache = CacheManager(
{
"default": "releases",
"serializer": "json",
"stores": {
"releases": {"driver": "file", "path": str(self._cache_dir)},
"packages": {"driver": "dict"},
"matches": {"driver": "dict"},
},
}
)
self._session = CacheControl(
requests.session(), cache=FileCache(str(self._cache_dir / "_http"))
)
url_parts = urlparse.urlparse(self._url)
if not url_parts.username:
self._session.auth = get_http_basic_auth(self.name)
self._disable_cache = disable_cache
@property
def name(self):
return self._name
def find_packages(
self, name, constraint=None, extras=None, allow_prereleases=False
):
packages = []
if constraint is not None and not isinstance(constraint, VersionConstraint):
constraint = parse_constraint(constraint)
key = name
if constraint:
key = "{}:{}".format(key, str(constraint))
if self._cache.store("matches").has(key):
versions = self._cache.store("matches").get(key)
else:
page = self._get("/{}/".format(canonicalize_name(name).replace(".", "-")))
if page is None:
return []
versions = []
for version in page.versions:
if not constraint or (constraint and constraint.allows(version)):
versions.append(version)
self._cache.store("matches").put(key, versions, 5)
for version in versions:
package = Package(name, version)
package.source_type = "legacy"
package.source_url = self._url
if extras is not None:
package.requires_extras = extras
packages.append(package)
self._log(
"{} packages found for {} {}".format(len(packages), name, str(constraint)),
level="debug",
)
return packages
def package(
self, name, version, extras=None
): # type: (...) -> poetry.packages.Package
"""
Retrieve the release information.
This is a heavy task which takes time.
We have to download a package to get the dependencies.
We also need to download every file matching this release
to get the various hashes.
Note that, this will be cached so the subsequent operations
should be much faster.
"""
try:
index = self._packages.index(
poetry.packages.Package(name, version, version)
)
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = poetry.packages.Package(name, version, version)
package.source_type = "legacy"
package.source_url = self._url
package.source_reference = self.name
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
if dependency.extras:
for extra in dependency.extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get("summary", "")
# Adding hashes information
package.hashes = release_info["digests"]
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
def _get_release_info(self, name, version): # type: (str, str) -> dict
page = self._get("/{}/".format(canonicalize_name(name).replace(".", "-")))
if page is None:
raise ValueError('No package named "{}"'.format(name))
data = {
"name": name,
"version": version,
"summary": "",
"requires_dist": [],
"requires_python": [],
"digests": [],
}
links = list(page.links_for_version(Version.parse(version)))
urls = {}
hashes = []
default_link = links[0]
for link in links:
if link.is_wheel:
urls["bdist_wheel"] = link.url
elif link.filename.endswith(".tar.gz"):
urls["sdist"] = link.url
elif link.filename.endswith((".zip", ".bz2")) and "sdist" not in urls:
urls["sdist"] = link.url
hash = link.hash
if link.hash_name == "sha256":
hashes.append(hash)
data["digests"] = hashes
if not urls:
if default_link.is_wheel:
m = wheel_file_re.match(default_link.filename)
python = m.group("pyver")
platform = m.group("plat")
if python == "py2.py3" and platform == "any":
urls["bdist_wheel"] = default_link.url
elif default_link.filename.endswith(".tar.gz"):
urls["sdist"] = default_link.url
elif (
default_link.filename.endswith((".zip", ".bz2")) and "sdist" not in urls
):
urls["sdist"] = default_link.url
else:
return data
info = self._get_info_from_urls(urls)
data["summary"] = info["summary"]
data["requires_dist"] = info["requires_dist"]
data["requires_python"] = info["requires_python"]
return data
def _download(self, url, dest): # type: (str, str) -> None
r = self._session.get(url, stream=True)
with open(dest, "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
def _get(self, endpoint): # type: (str) -> Union[Page, None]
url = self._url + endpoint
response = self._session.get(url)
if response.status_code == 404:
return
return Page(url, response.content, response.headers)
class LegacyRepository(PyPiRepository):
def __init__(self, name, url):
if name == 'pypi':
raise ValueError('The name [pypi] is reserved for repositories')
self._name = name
self._url = url
command = get_pip_command()
opts, _ = command.parse_args([])
self._session = command._build_session(opts)
self._repository = PyPIRepository(opts, self._session)
self._cache_dir = Path(CACHE_DIR) / 'cache' / 'repositories' / name
self._cache = CacheManager({
'default': 'releases',
'serializer': 'json',
'stores': {
'releases': {
'driver': 'file',
'path': Path(CACHE_DIR) / 'cache' / 'repositories' / name
},
'packages': {
'driver': 'dict'
},
'matches': {
'driver': 'dict'
}
}
})
def find_packages(self, name, constraint=None, extras=None):
packages = []
if constraint is not None and not isinstance(constraint,
BaseConstraint):
version_parser = VersionParser()
constraint = version_parser.parse_constraints(constraint)
key = name
if constraint:
key = f'{key}:{str(constraint)}'
if self._cache.store('matches').has(key):
versions = self._cache.store('matches').get(key)
else:
candidates = [str(c.version) for c in self._repository.find_all_candidates(name)]
versions = []
for version in candidates:
if version in versions:
continue
if (
not constraint
or (constraint and constraint.matches(Constraint('=', version)))
):
versions.append(version)
self._cache.store('matches').put(key, versions, 5)
for version in versions:
packages.append(self.package(name, version, extras=extras))
return packages
def package(self, name, version, extras=None) -> 'poetry.packages.Package':
"""
Retrieve the release information.
This is a heavy task which takes time.
We have to download a package to get the dependencies.
We also need to download every file matching this release
to get the various hashes.
Note that, this will be cached so the subsequent operations
should be much faster.
"""
try:
index = self._packages.index(
poetry.packages.Package(name, version, version)
)
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = poetry.packages.Package(name, version, version)
for req in release_info['requires_dist']:
req = InstallRequirement.from_line(req)
name = req.name
version = str(req.req.specifier)
dependency = Dependency(
name,
version,
optional=req.markers
)
is_extra = False
if req.markers:
# Setting extra dependencies and requirements
requirements = self._convert_markers(
req.markers._markers
)
if 'python_version' in requirements:
ors = []
for or_ in requirements['python_version']:
ands = []
for op, version in or_:
ands.append(f'{op}{version}')
ors.append(' '.join(ands))
dependency.python_versions = ' || '.join(ors)
if 'sys_platform' in requirements:
ors = []
for or_ in requirements['sys_platform']:
ands = []
for op, platform in or_:
ands.append(f'{op}{platform}')
ors.append(' '.join(ands))
dependency.platform = ' || '.join(ors)
if 'extra' in requirements:
is_extra = True
for _extras in requirements['extra']:
for _, extra in _extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not is_extra:
package.requires.append(dependency)
# Adding description
package.description = release_info.get('summary', '')
# Adding hashes information
package.hashes = release_info['digests']
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
def get_release_info(self, name: str, version: str) -> dict:
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
return self._cache.store('releases').remember_forever(
f'{name}:{version}',
lambda: self._get_release_info(name, version)
)
def _get_release_info(self, name: str, version: str) -> dict:
ireq = InstallRequirement.from_line(f'{name}=={version}')
resolver = Resolver(
[ireq], self._repository,
cache=DependencyCache(self._cache_dir.as_posix())
)
try:
requirements = list(resolver._iter_dependencies(ireq))
except InstallationError as e:
# setup.py egg-info error most likely
# So we assume no dependencies
requirements = []
requires = []
for dep in requirements:
constraint = str(dep.req.specifier)
require = f'{dep.name}'
if constraint:
require += f' ({constraint})'
requires.append(require)
hashes = resolver.resolve_hashes([ireq])[ireq]
hashes = [h.split(':')[1] for h in hashes]
data = {
'name': name,
'version': version,
'summary': '',
'requires_dist': requires,
'digests': hashes
}
resolver.repository.freshen_build_caches()
return data
class PyPiRepository(Repository):
CACHE_VERSION = parse_constraint("1.0.0b2")
def __init__(self,
url="https://pypi.org/",
disable_cache=False,
fallback=True):
self._url = url
self._disable_cache = disable_cache
self._fallback = fallback
release_cache_dir = Path(CACHE_DIR) / "cache" / "repositories" / "pypi"
self._cache = CacheManager({
"default": "releases",
"serializer": "json",
"stores": {
"releases": {
"driver": "file",
"path": str(release_cache_dir)
},
"packages": {
"driver": "dict"
},
},
})
self._session = CacheControl(session(),
cache=FileCache(
str(release_cache_dir / "_http")))
self._inspector = Inspector()
super(PyPiRepository, self).__init__()
self._name = "PyPI"
@property
def url(self): # type: () -> str
return self._url
@property
def authenticated_url(self): # type: () -> str
return self._url
def find_packages(
self,
name, # type: str
constraint=None, # type: Union[VersionConstraint, str, None]
extras=None, # type: Union[list, None]
allow_prereleases=False, # type: bool
): # type: (...) -> List[Package]
"""
Find packages on the remote server.
"""
if constraint is None:
constraint = "*"
if not isinstance(constraint, VersionConstraint):
constraint = parse_constraint(constraint)
if isinstance(constraint, VersionRange):
if (constraint.max is not None and constraint.max.is_prerelease()
or constraint.min is not None
and constraint.min.is_prerelease()):
allow_prereleases = True
try:
info = self.get_package_info(name)
except PackageNotFound:
self._log(
"No packages found for {} {}".format(name, str(constraint)),
level="debug",
)
return []
packages = []
for version, release in info["releases"].items():
if not release:
# Bad release
self._log(
"No release information found for {}-{}, skipping".format(
name, version),
level="debug",
)
continue
try:
package = Package(name, version)
except ParseVersionError:
self._log(
'Unable to parse version "{}" for the {} package, skipping'
.format(version, name),
level="debug",
)
continue
if package.is_prerelease() and not allow_prereleases:
continue
if not constraint or (constraint
and constraint.allows(package.version)):
if extras is not None:
package.requires_extras = extras
packages.append(package)
self._log(
"{} packages found for {} {}".format(len(packages), name,
str(constraint)),
level="debug",
)
return packages
def package(
self,
name, # type: str
version, # type: str
extras=None, # type: (Union[list, None])
): # type: (...) -> Union[Package, None]
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.in_extras:
for extra in dependency.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get("summary", "")
if release_info["requires_python"]:
package.python_versions = release_info["requires_python"]
if release_info["platform"]:
package.platform = release_info["platform"]
# Adding hashes information
package.files = release_info["files"]
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
return package
def search(self, query):
results = []
search = {"q": query}
response = session().get(self._url + "search", params=search)
content = parse(response.content, namespaceHTMLElements=False)
for result in content.findall(".//*[@class='package-snippet']"):
name = result.find("h3/*[@class='package-snippet__name']").text
version = result.find(
"h3/*[@class='package-snippet__version']").text
if not name or not version:
continue
description = result.find(
"p[@class='package-snippet__description']").text
if not description:
description = ""
try:
result = Package(name, version, description)
result.description = to_str(description.strip())
results.append(result)
except ParseVersionError:
self._log(
'Unable to parse version "{}" for the {} package, skipping'
.format(version, name),
level="debug",
)
return results
def get_package_info(self, name): # type: (str) -> dict
"""
Return the package information given its name.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_package_info(name)
return self._cache.store("packages").remember_forever(
name, lambda: self._get_package_info(name))
def _get_package_info(self, name): # type: (str) -> dict
data = self._get("pypi/{}/json".format(name))
if data is None:
raise PackageNotFound("Package [{}] not found.".format(name))
return data
def get_release_info(self, name, version): # type: (str, str) -> dict
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_release_info(name, version)
cached = self._cache.remember_forever(
"{}:{}".format(name, version),
lambda: self._get_release_info(name, version))
cache_version = cached.get("_cache_version", "0.0.0")
if parse_constraint(cache_version) != self.CACHE_VERSION:
# The cache must be updated
self._log(
"The cache for {} {} is outdated. Refreshing.".format(
name, version),
level="debug",
)
cached = self._get_release_info(name, version)
self._cache.forever("{}:{}".format(name, version), cached)
return cached
def _get_release_info(self, name, version): # type: (str, str) -> dict
self._log("Getting info for {} ({}) from PyPI".format(name, version),
"debug")
json_data = self._get("pypi/{}/{}/json".format(name, version))
if json_data is None:
raise PackageNotFound("Package [{}] not found.".format(name))
info = json_data["info"]
data = {
"name": info["name"],
"version": info["version"],
"summary": info["summary"],
"platform": info["platform"],
"requires_dist": info["requires_dist"],
"requires_python": info["requires_python"],
"files": [],
"_cache_version": str(self.CACHE_VERSION),
}
try:
version_info = json_data["releases"][version]
except KeyError:
version_info = []
for file_info in version_info:
data["files"].append({
"file":
file_info["filename"],
"hash":
"sha256:" + file_info["digests"]["sha256"],
})
if self._fallback and data["requires_dist"] is None:
self._log("No dependencies found, downloading archives",
level="debug")
# No dependencies set (along with other information)
# This might be due to actually no dependencies
# or badly set metadata when uploading
# So, we need to make sure there is actually no
# dependencies by introspecting packages
urls = defaultdict(list)
for url in json_data["urls"]:
# Only get sdist and wheels if they exist
dist_type = url["packagetype"]
if dist_type not in ["sdist", "bdist_wheel"]:
continue
urls[dist_type].append(url["url"])
if not urls:
return data
info = self._get_info_from_urls(urls)
data["requires_dist"] = info["requires_dist"]
if not data["requires_python"]:
data["requires_python"] = info["requires_python"]
return data
def _get(self, endpoint): # type: (str) -> Union[dict, None]
json_response = self._session.get(self._url + endpoint)
if json_response.status_code == 404:
return None
json_data = json_response.json()
return json_data
def _get_info_from_urls(
self, urls
): # type: (Dict[str, List[str]]) -> Dict[str, Union[str, List, None]]
# Checking wheels first as they are more likely to hold
# the necessary information
if "bdist_wheel" in urls:
# Check fo a universal wheel
wheels = urls["bdist_wheel"]
universal_wheel = None
universal_python2_wheel = None
universal_python3_wheel = None
platform_specific_wheels = []
for wheel in wheels:
link = Link(wheel)
m = wheel_file_re.match(link.filename)
if not m:
continue
pyver = m.group("pyver")
abi = m.group("abi")
plat = m.group("plat")
if abi == "none" and plat == "any":
# Universal wheel
if pyver == "py2.py3":
# Any Python
universal_wheel = wheel
elif pyver == "py2":
universal_python2_wheel = wheel
else:
universal_python3_wheel = wheel
else:
platform_specific_wheels.append(wheel)
if universal_wheel is not None:
return self._get_info_from_wheel(universal_wheel)
info = {}
if universal_python2_wheel and universal_python3_wheel:
info = self._get_info_from_wheel(universal_python2_wheel)
py3_info = self._get_info_from_wheel(universal_python3_wheel)
if py3_info["requires_dist"]:
if not info["requires_dist"]:
info["requires_dist"] = py3_info["requires_dist"]
return info
py2_requires_dist = set(
dependency_from_pep_508(r).to_pep_508()
for r in info["requires_dist"])
py3_requires_dist = set(
dependency_from_pep_508(r).to_pep_508()
for r in py3_info["requires_dist"])
base_requires_dist = py2_requires_dist & py3_requires_dist
py2_only_requires_dist = py2_requires_dist - py3_requires_dist
py3_only_requires_dist = py3_requires_dist - py2_requires_dist
# Normalizing requires_dist
requires_dist = list(base_requires_dist)
for requirement in py2_only_requires_dist:
dep = dependency_from_pep_508(requirement)
dep.marker = dep.marker.intersect(
parse_marker("python_version == '2.7'"))
requires_dist.append(dep.to_pep_508())
for requirement in py3_only_requires_dist:
dep = dependency_from_pep_508(requirement)
dep.marker = dep.marker.intersect(
parse_marker("python_version >= '3'"))
requires_dist.append(dep.to_pep_508())
info["requires_dist"] = sorted(list(set(requires_dist)))
if info:
return info
# Prefer non platform specific wheels
if universal_python3_wheel:
return self._get_info_from_wheel(universal_python3_wheel)
if universal_python2_wheel:
return self._get_info_from_wheel(universal_python2_wheel)
if platform_specific_wheels and "sdist" not in urls:
# Pick the first wheel available and hope for the best
return self._get_info_from_wheel(platform_specific_wheels[0])
return self._get_info_from_sdist(urls["sdist"][0])
def _get_info_from_wheel(
self, url): # type: (str) -> Dict[str, Union[str, List, None]]
self._log(
"Downloading wheel: {}".format(
urlparse.urlparse(url).path.rsplit("/")[-1]),
level="debug",
)
filename = os.path.basename(
urlparse.urlparse(url).path.rsplit("/")[-1])
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
return self._inspector.inspect_wheel(filepath)
def _get_info_from_sdist(
self, url): # type: (str) -> Dict[str, Union[str, List, None]]
self._log(
"Downloading sdist: {}".format(
urlparse.urlparse(url).path.rsplit("/")[-1]),
level="debug",
)
filename = os.path.basename(urlparse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
return self._inspector.inspect_sdist(filepath)
def _download(self, url, dest): # type: (str, str) -> None
r = get(url, stream=True)
r.raise_for_status()
with open(dest, "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
def _log(self, msg, level="info"):
getattr(logger,
level)("<comment>{}:</comment> {}".format(self._name, msg))
class LegacyRepository(PyPiRepository):
def __init__(self, name, url):
if name == 'pypi':
raise ValueError('The name [pypi] is reserved for repositories')
self._packages = []
self._name = name
self._url = url.rstrip('/')
self._cache_dir = Path(CACHE_DIR) / 'cache' / 'repositories' / name
self._cache = CacheManager({
'default': 'releases',
'serializer': 'json',
'stores': {
'releases': {
'driver': 'file',
'path': str(self._cache_dir)
},
'packages': {
'driver': 'dict'
},
'matches': {
'driver': 'dict'
}
}
})
self._session = CacheControl(requests.session(),
cache=FileCache(
str(self._cache_dir / '_http')))
@property
def name(self):
return self._name
def find_packages(self,
name,
constraint=None,
extras=None,
allow_prereleases=False):
packages = []
if constraint is not None and not isinstance(constraint,
VersionConstraint):
constraint = parse_constraint(constraint)
key = name
if constraint:
key = '{}:{}'.format(key, str(constraint))
if self._cache.store('matches').has(key):
versions = self._cache.store('matches').get(key)
else:
page = self._get('/{}'.format(
canonicalize_name(name).replace('.', '-')))
if page is None:
raise ValueError('No package named "{}"'.format(name))
versions = []
for version in page.versions:
if (not constraint
or (constraint and constraint.allows(version))):
versions.append(version)
self._cache.store('matches').put(key, versions, 5)
for version in versions:
package = Package(name, version)
if extras is not None:
package.requires_extras = extras
packages.append(package)
return packages
def package(self,
name,
version,
extras=None): # type: (...) -> poetry.packages.Package
"""
Retrieve the release information.
This is a heavy task which takes time.
We have to download a package to get the dependencies.
We also need to download every file matching this release
to get the various hashes.
Note that, this will be cached so the subsequent operations
should be much faster.
"""
try:
index = self._packages.index(
poetry.packages.Package(name, version, version))
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = poetry.packages.Package(name, version, version)
requires_dist = release_info['requires_dist'] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(';')[0]
dependency = dependency_from_pep_508(req)
if dependency.extras:
for extra in dependency.extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get('summary', '')
# Adding hashes information
package.hashes = release_info['digests']
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
def get_release_info(self, name, version): # type: (str, str) -> dict
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
return self._cache.store('releases').remember_forever(
'{}:{}'.format(name, version),
lambda: self._get_release_info(name, version))
def _get_release_info(self, name, version): # type: (str, str) -> dict
page = self._get('/{}'.format(
canonicalize_name(name).replace('.', '-')))
if page is None:
raise ValueError('No package named "{}"'.format(name))
data = {
'name': name,
'version': version,
'summary': '',
'requires_dist': [],
'requires_python': [],
'digests': []
}
links = list(page.links_for_version(Version.parse(version)))
urls = {}
hashes = []
default_link = links[0]
for link in links:
if link.is_wheel:
urls['bdist_wheel'] = link.url
elif link.filename.endswith('.tar.gz'):
urls['sdist'] = link.url
elif link.filename.endswith(
('.zip', '.bz2')) and 'sdist' not in urls:
urls['sdist'] = link.url
hash = link.hash
if link.hash_name == 'sha256':
hashes.append(hash)
data['digests'] = hashes
if not urls:
if default_link.is_wheel:
m = wheel_file_re.match(default_link.filename)
python = m.group('pyver')
platform = m.group('plat')
if python == 'py2.py3' and platform == 'any':
urls['bdist_wheel'] = default_link.url
elif default_link.filename.endswith('.tar.gz'):
urls['sdist'] = default_link.url
elif default_link.filename.endswith(
('.zip', '.bz2')) and 'sdist' not in urls:
urls['sdist'] = default_link.url
else:
return data
info = self._get_info_from_urls(urls)
data['summary'] = info['summary']
data['requires_dist'] = info['requires_dist']
data['requires_python'] = info['requires_python']
return data
def _get(self, endpoint): # type: (str) -> Union[Page, None]
url = self._url + endpoint
response = self._session.get(url)
if response.status_code == 404:
return
return Page(url, response.content, response.headers)
class PyPiRepository(Repository):
def __init__(self,
url='https://pypi.org/',
disable_cache=False,
fallback=True):
self._name = 'PyPI'
self._url = url
self._disable_cache = disable_cache
self._fallback = fallback
release_cache_dir = Path(CACHE_DIR) / 'cache' / 'repositories' / 'pypi'
self._cache = CacheManager({
'default': 'releases',
'serializer': 'json',
'stores': {
'releases': {
'driver': 'file',
'path': str(release_cache_dir)
},
'packages': {
'driver': 'dict'
}
}
})
self._session = CacheControl(
session(),
cache=FileCache(str(release_cache_dir / '_http'))
)
super(PyPiRepository, self).__init__()
def find_packages(self,
name, # type: str
constraint=None, # type: Union[Constraint, str, None]
extras=None, # type: Union[list, None]
allow_prereleases=False # type: bool
): # type: (...) -> List[Package]
"""
Find packages on the remote server.
"""
if constraint is not None and not isinstance(constraint, BaseConstraint):
version_parser = VersionParser()
constraint = version_parser.parse_constraints(constraint)
info = self.get_package_info(name)
packages = []
for version, release in info['releases'].items():
if not release:
# Bad release
self._log(
'No release information found for {}-{}, skipping'.format(
name, version
),
level='debug'
)
continue
package = Package(name, version)
if package.is_prerelease() and not allow_prereleases:
continue
if (
not constraint
or (constraint and constraint.matches(Constraint('=', version)))
):
if extras is not None:
package.requires_extras = extras
packages.append(package)
self._log(
'{} packages found for {} {}'.format(
len(packages), name, str(constraint)
),
level='debug'
)
return packages
def package(self,
name, # type: str
version, # type: str
extras=None # type: (Union[list, None])
): # type: (...) -> Union[Package, None]
try:
index = self._packages.index(Package(name, version, version))
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
if (
self._fallback
and release_info['requires_dist'] is None
and not release_info['requires_python']
and '_fallback' not in release_info
):
# Force cache update
self._log(
'No dependencies found, downloading archives',
level='debug'
)
self._cache.forget('{}:{}'.format(name, version))
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
requires_dist = release_info['requires_dist'] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(';')[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
'Invalid constraint ({}) found in {}-{} dependencies, '
'skipping'.format(
req, package.name, package.version
),
level='debug'
)
continue
if dependency.extras:
for extra in dependency.extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get('summary', '')
if release_info['requires_python']:
package.python_versions = release_info['requires_python']
if release_info['platform']:
package.platform = release_info['platform']
# Adding hashes information
package.hashes = release_info['digests']
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
def search(self, query, mode=0):
results = []
search = {
'name': query
}
if mode == self.SEARCH_FULLTEXT:
search['summary'] = query
client = ServerProxy('https://pypi.python.org/pypi')
hits = client.search(search, 'or')
for hit in hits:
result = Package(hit['name'], hit['version'], hit['version'])
result.description = to_str(hit['summary'])
results.append(result)
return results
def get_package_info(self, name): # type: (str) -> dict
"""
Return the package information given its name.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_package_info(name)
return self._cache.store('packages').remember_forever(
name,
lambda: self._get_package_info(name)
)
def _get_package_info(self, name): # type: (str) -> dict
data = self._get('pypi/{}/json'.format(name))
if data is None:
raise ValueError('Package [{}] not found.'.format(name))
return data
def get_release_info(self, name, version): # type: (str, str) -> dict
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_release_info(name, version)
return self._cache.remember_forever(
'{}:{}'.format(name, version),
lambda: self._get_release_info(name, version)
)
def _get_release_info(self, name, version): # type: (str, str) -> dict
json_data = self._get('pypi/{}/{}/json'.format(name, version))
if json_data is None:
raise ValueError('Package [{}] not found.'.format(name))
info = json_data['info']
data = {
'name': info['name'],
'version': info['version'],
'summary': info['summary'],
'platform': info['platform'],
'requires_dist': info['requires_dist'],
'requires_python': info['requires_python'],
'digests': [],
'_fallback': False
}
try:
version_info = json_data['releases'][version]
except KeyError:
version_info = []
for file_info in version_info:
data['digests'].append(file_info['digests']['sha256'])
if (
self._fallback
and data['requires_dist'] is None
and not data['requires_python']
):
# No dependencies set (along with other information)
# This might be due to actually no dependencies
# or badly set metadata when uploading
# So, we need to make sure there is actually no
# dependencies by introspecting packages
data['_fallback'] = True
urls = {}
for url in json_data['urls']:
# Only get sdist and universal wheels
dist_type = url['packagetype']
if dist_type not in ['sdist', 'bdist_wheel']:
continue
if dist_type == 'sdist' and 'dist' not in urls:
urls[url['packagetype']] = url['url']
continue
if 'bdist_wheel' in urls:
continue
# If bdist_wheel, check if it's universal
python_version = url['python_version']
if python_version not in ['py2.py3', 'py3', 'py2']:
continue
parts = urlparse.urlparse(url['url'])
filename = os.path.basename(parts.path)
if '-none-any' not in filename:
continue
if not urls:
return data
requires_dist = self._get_requires_dist_from_urls(urls)
data['requires_dist'] = requires_dist
return data
def _get(self, endpoint): # type: (str) -> Union[dict, None]
json_response = self._session.get(self._url + endpoint)
if json_response.status_code == 404:
return None
json_data = json_response.json()
return json_data
def _get_requires_dist_from_urls(self, urls
): # type: (dict) -> Union[list, None]
if 'bdist_wheel' in urls:
return self._get_requires_dist_from_wheel(urls['bdist_wheek'])
return self._get_requires_dist_from_sdist(urls['sdist'])
def _get_requires_dist_from_wheel(self, url
): # type: (str) -> Union[list, None]
filename = os.path.basename(urlparse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = os.path.join(temp_dir, filename)
self._download(url, filepath)
try:
meta = pkginfo.Wheel(filepath)
except ValueError:
# Unable to determine dependencies
# Assume none
return
if meta.requires_dist:
return meta.requires_dist
def _get_requires_dist_from_sdist(self, url
): # type: (str) -> Union[list, None]
filename = os.path.basename(urlparse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
try:
meta = pkginfo.SDist(str(filepath))
if meta.requires_dist:
return meta.requires_dist
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
# Still not dependencies found
# So, we unpack and introspect
suffix = filepath.suffix
gz = None
if suffix == '.zip':
tar = zipfile.ZipFile(str(filepath))
else:
if suffix == '.bz2':
gz = BZ2File(str(filepath))
else:
gz = GzipFile(str(filepath))
tar = tarfile.TarFile(str(filepath), fileobj=gz)
try:
tar.extractall(os.path.join(temp_dir, 'unpacked'))
finally:
if gz:
gz.close()
tar.close()
unpacked = Path(temp_dir) / 'unpacked'
sdist_dir = unpacked / Path(filename).name.rstrip('.tar.gz')
# Checking for .egg-info
eggs = list(sdist_dir.glob('*.egg-info'))
if eggs:
egg_info = eggs[0]
requires = egg_info / 'requires.txt'
if requires.exists():
with requires.open() as f:
return parse_requires(f.read())
return
# Still nothing, assume no dependencies
# We could probably get them by executing
# python setup.py egg-info but I don't feel
# confortable executing a file just for the sake
# of getting dependencies.
return
def _download(self, url, dest): # type: (str, str) -> None
r = get(url, stream=True)
with open(dest, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
def _log(self, msg, level='info'):
getattr(logger, level)('{}: {}'.format(self._name, msg))
class PyPiRepository(Repository):
def __init__(self,
url='https://pypi.org/',
disable_cache=False,
fallback=False):
self._url = url
self._disable_cache = disable_cache
self._fallback = fallback
release_cache_dir = Path(CACHE_DIR) / 'cache' / 'repositories' / 'pypi'
self._cache = CacheManager({
'default': 'releases',
'serializer': 'json',
'stores': {
'releases': {
'driver': 'file',
'path': str(release_cache_dir)
},
'packages': {
'driver': 'dict'
}
}
})
self._session = CacheControl(session(),
cache=FileCache(
str(release_cache_dir / '_http')))
super(PyPiRepository, self).__init__()
def find_packages(
self,
name, # type: str
constraint=None, # type: Union[Constraint, str, None]
extras=None # type: Union[list, None]
): # type: (...) -> List[Package]
"""
Find packages on the remote server.
"""
packages = []
if constraint is not None and not isinstance(constraint,
BaseConstraint):
version_parser = VersionParser()
constraint = version_parser.parse_constraints(constraint)
info = self.get_package_info(name)
versions = []
for version, release in info['releases'].items():
if (not constraint or
(constraint and constraint.matches(Constraint('=', version)))):
versions.append(version)
for version in versions:
packages.append(Package(name, version, version))
return packages
def package(
self,
name, # type: str
version, # type: str
extras=None # type: (Union[list, None])
): # type: (...) -> Union[Package, None]
try:
index = self._packages.index(Package(name, version, version))
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
if (self._fallback and release_info['requires_dist'] is None
and not release_info['requires_python']
and not release_info['platform']):
# No dependencies set (along with other information)
# This might be due to actually no dependencies
# or badly set metadata when uploading
# So, we return None so that the fallback repository
# can pick up more accurate info
return
package = Package(name, version, version)
requires_dist = release_info['requires_dist'] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(';')[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
continue
if dependency.extras:
for extra in dependency.extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get('summary', '')
# Adding hashes information
package.hashes = release_info['digests']
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
def search(self, query, mode=0):
results = []
search = {'name': query}
if mode == self.SEARCH_FULLTEXT:
search['summary'] = query
client = ServerProxy('https://pypi.python.org/pypi')
hits = client.search(search, 'or')
for hit in hits:
result = Package(hit['name'], hit['version'], hit['version'])
result.description = hit['summary']
results.append(result)
return results
def get_package_info(self, name): # type: (str) -> dict
"""
Return the package information given its name.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_package_info(name)
return self._cache.store('packages').remember_forever(
name, lambda: self._get_package_info(name))
def _get_package_info(self, name): # type: (str) -> dict
data = self._get('pypi/{}/json'.format(name))
if data is None:
raise ValueError('Package [{}] not found.'.format(name))
return data
def get_release_info(self, name, version): # type: (str, str) -> dict
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_release_info(name, version)
return self._cache.remember_forever(
'{}:{}'.format(name, version),
lambda: self._get_release_info(name, version))
def _get_release_info(self, name, version): # type: (str, str) -> dict
json_data = self._get('pypi/{}/{}/json'.format(name, version))
if json_data is None:
raise ValueError('Package [{}] not found.'.format(name))
info = json_data['info']
data = {
'name': info['name'],
'version': info['version'],
'summary': info['summary'],
'platform': info['platform'],
'requires_dist': info['requires_dist'],
'requires_python': info['requires_python'],
'digests': []
}
try:
version_info = json_data['releases'][version]
except KeyError:
version_info = []
for file_info in version_info:
data['digests'].append(file_info['digests']['sha256'])
return data
def _get(self, endpoint): # type: (str) -> Union[dict, None]
json_response = self._session.get(self._url + endpoint)
if json_response.status_code == 404:
return None
json_data = json_response.json()
return json_data
class PyPiRepository(Repository):
CACHE_VERSION = parse_constraint("0.12.0")
def __init__(self,
url="https://pypi.org/",
disable_cache=False,
fallback=True):
self._url = url
self._disable_cache = disable_cache
self._fallback = fallback
release_cache_dir = Path(CACHE_DIR) / "cache" / "repositories" / "pypi"
self._cache = CacheManager({
"default": "releases",
"serializer": "json",
"stores": {
"releases": {
"driver": "file",
"path": str(release_cache_dir)
},
"packages": {
"driver": "dict"
},
},
})
self._session = CacheControl(session(),
cache=FileCache(
str(release_cache_dir / "_http")))
super(PyPiRepository, self).__init__()
self._name = "PyPI"
@property
def url(self): # type: () -> str
return self._url
@property
def authenticated_url(self): # type: () -> str
return self._url
def find_packages(
self,
name, # type: str
constraint=None, # type: Union[VersionConstraint, str, None]
extras=None, # type: Union[list, None]
allow_prereleases=False, # type: bool
): # type: (...) -> List[Package]
"""
Find packages on the remote server.
"""
if constraint is None:
constraint = "*"
if not isinstance(constraint, VersionConstraint):
constraint = parse_constraint(constraint)
if isinstance(constraint, VersionRange):
if (constraint.max is not None and constraint.max.is_prerelease()
or constraint.min is not None
and constraint.min.is_prerelease()):
allow_prereleases = True
try:
info = self.get_package_info(name)
except PackageNotFound:
self._log(
"No packages found for {} {}".format(name, str(constraint)),
level="debug",
)
return []
packages = []
for version, release in info["releases"].items():
if not release:
# Bad release
self._log(
"No release information found for {}-{}, skipping".format(
name, version),
level="debug",
)
continue
try:
package = Package(name, version)
except ParseVersionError:
self._log(
'Unable to parse version "{}" for the {} package, skipping'
.format(version, name),
level="debug",
)
continue
if package.is_prerelease() and not allow_prereleases:
continue
if not constraint or (constraint
and constraint.allows(package.version)):
if extras is not None:
package.requires_extras = extras
packages.append(package)
self._log(
"{} packages found for {} {}".format(len(packages), name,
str(constraint)),
level="debug",
)
return packages
def package(
self,
name, # type: str
version, # type: str
extras=None, # type: (Union[list, None])
): # type: (...) -> Union[Package, None]
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.in_extras:
for extra in dependency.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get("summary", "")
if release_info["requires_python"]:
package.python_versions = release_info["requires_python"]
if release_info["platform"]:
package.platform = release_info["platform"]
# Adding hashes information
package.hashes = release_info["digests"]
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
return package
def search(self, query, mode=0):
results = []
search = {"name": query}
if mode == self.SEARCH_FULLTEXT:
search["summary"] = query
client = ServerProxy("https://pypi.python.org/pypi")
hits = client.search(search, "or")
for hit in hits:
result = Package(hit["name"], hit["version"], hit["version"])
result.description = to_str(hit["summary"])
results.append(result)
return results
def get_package_info(self, name): # type: (str) -> dict
"""
Return the package information given its name.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_package_info(name)
return self._cache.store("packages").remember_forever(
name, lambda: self._get_package_info(name))
def _get_package_info(self, name): # type: (str) -> dict
data = self._get("pypi/{}/json".format(name))
if data is None:
raise PackageNotFound("Package [{}] not found.".format(name))
return data
def get_release_info(self, name, version): # type: (str, str) -> dict
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_release_info(name, version)
cached = self._cache.remember_forever(
"{}:{}".format(name, version),
lambda: self._get_release_info(name, version))
cache_version = cached.get("_cache_version", "0.0.0")
if parse_constraint(cache_version) != self.CACHE_VERSION:
# The cache must be updated
self._log(
"The cache for {} {} is outdated. Refreshing.".format(
name, version),
level="debug",
)
cached = self._get_release_info(name, version)
self._cache.forever("{}:{}".format(name, version), cached)
return cached
def _get_release_info(self, name, version): # type: (str, str) -> dict
self._log("Getting info for {} ({}) from PyPI".format(name, version),
"debug")
json_data = self._get("pypi/{}/{}/json".format(name, version))
if json_data is None:
raise PackageNotFound("Package [{}] not found.".format(name))
info = json_data["info"]
data = {
"name": info["name"],
"version": info["version"],
"summary": info["summary"],
"platform": info["platform"],
"requires_dist": info["requires_dist"],
"requires_python": info["requires_python"],
"digests": [],
"_cache_version": str(self.CACHE_VERSION),
}
try:
version_info = json_data["releases"][version]
except KeyError:
version_info = []
for file_info in version_info:
data["digests"].append(file_info["digests"]["sha256"])
if self._fallback and data["requires_dist"] is None:
self._log("No dependencies found, downloading archives",
level="debug")
# No dependencies set (along with other information)
# This might be due to actually no dependencies
# or badly set metadata when uploading
# So, we need to make sure there is actually no
# dependencies by introspecting packages
urls = defaultdict(list)
for url in json_data["urls"]:
# Only get sdist and wheels if they exist
dist_type = url["packagetype"]
if dist_type not in ["sdist", "bdist_wheel"]:
continue
urls[dist_type].append(url["url"])
if not urls:
return data
info = self._get_info_from_urls(urls)
data["requires_dist"] = info["requires_dist"]
if not data["requires_python"]:
data["requires_python"] = info["requires_python"]
return data
def _get(self, endpoint): # type: (str) -> Union[dict, None]
json_response = self._session.get(self._url + endpoint)
if json_response.status_code == 404:
return None
json_data = json_response.json()
return json_data
def _get_info_from_urls(
self, urls
): # type: (Dict[str, List[str]]) -> Dict[str, Union[str, List, None]]
# Checking wheels first as they are more likely to hold
# the necessary information
if "bdist_wheel" in urls:
# Check fo a universal wheel
wheels = urls["bdist_wheel"]
universal_wheel = None
universal_python2_wheel = None
universal_python3_wheel = None
platform_specific_wheels = []
for wheel in wheels:
link = Link(wheel)
m = wheel_file_re.match(link.filename)
if not m:
continue
pyver = m.group("pyver")
abi = m.group("abi")
plat = m.group("plat")
if abi == "none" and plat == "any":
# Universal wheel
if pyver == "py2.py3":
# Any Python
universal_wheel = wheel
elif pyver == "py2":
universal_python2_wheel = wheel
else:
universal_python3_wheel = wheel
else:
platform_specific_wheels.append(wheel)
if universal_wheel is not None:
return self._get_info_from_wheel(universal_wheel)
info = {}
if universal_python2_wheel and universal_python3_wheel:
info = self._get_info_from_wheel(universal_python2_wheel)
py3_info = self._get_info_from_wheel(universal_python3_wheel)
if py3_info["requires_dist"]:
if not info["requires_dist"]:
info["requires_dist"] = py3_info["requires_dist"]
return info
py2_requires_dist = set(
dependency_from_pep_508(r).to_pep_508()
for r in info["requires_dist"])
py3_requires_dist = set(
dependency_from_pep_508(r).to_pep_508()
for r in py3_info["requires_dist"])
base_requires_dist = py2_requires_dist & py3_requires_dist
py2_only_requires_dist = py2_requires_dist - py3_requires_dist
py3_only_requires_dist = py3_requires_dist - py2_requires_dist
# Normalizing requires_dist
requires_dist = list(base_requires_dist)
for requirement in py2_only_requires_dist:
dep = dependency_from_pep_508(requirement)
dep.marker = dep.marker.intersect(
parse_marker("python_version == '2.7'"))
requires_dist.append(dep.to_pep_508())
for requirement in py3_only_requires_dist:
dep = dependency_from_pep_508(requirement)
dep.marker = dep.marker.intersect(
parse_marker("python_version >= '3'"))
requires_dist.append(dep.to_pep_508())
info["requires_dist"] = sorted(list(set(requires_dist)))
if info:
return info
# Prefer non platform specific wheels
if universal_python3_wheel:
return self._get_info_from_wheel(universal_python3_wheel)
if universal_python2_wheel:
return self._get_info_from_wheel(universal_python2_wheel)
if platform_specific_wheels and "sdist" not in urls:
# Pick the first wheel available and hope for the best
return self._get_info_from_wheel(platform_specific_wheels[0])
return self._get_info_from_sdist(urls["sdist"][0])
def _get_info_from_wheel(
self, url): # type: (str) -> Dict[str, Union[str, List, None]]
self._log(
"Downloading wheel: {}".format(
urlparse.urlparse(url).path.rsplit("/")[-1]),
level="debug",
)
info = {"summary": "", "requires_python": None, "requires_dist": None}
filename = os.path.basename(
urlparse.urlparse(url).path.rsplit("/")[-1])
with temporary_directory() as temp_dir:
filepath = os.path.join(temp_dir, filename)
self._download(url, filepath)
try:
meta = pkginfo.Wheel(filepath)
except ValueError:
# Unable to determine dependencies
# Assume none
return info
if meta.summary:
info["summary"] = meta.summary or ""
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = meta.requires_dist
return info
def _get_info_from_sdist(
self, url): # type: (str) -> Dict[str, Union[str, List, None]]
self._log(
"Downloading sdist: {}".format(
urlparse.urlparse(url).path.rsplit("/")[-1]),
level="debug",
)
info = {"summary": "", "requires_python": None, "requires_dist": None}
filename = os.path.basename(urlparse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
try:
meta = pkginfo.SDist(str(filepath))
if meta.summary:
info["summary"] = meta.summary
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
return info
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
# Still not dependencies found
# So, we unpack and introspect
suffix = filepath.suffix
gz = None
if suffix == ".zip":
tar = zipfile.ZipFile(str(filepath))
else:
if suffix == ".bz2":
gz = BZ2File(str(filepath))
suffixes = filepath.suffixes
if len(suffixes) > 1 and suffixes[-2] == ".tar":
suffix = ".tar.bz2"
else:
gz = GzipFile(str(filepath))
suffix = ".tar.gz"
tar = tarfile.TarFile(str(filepath), fileobj=gz)
try:
tar.extractall(os.path.join(temp_dir, "unpacked"))
finally:
if gz:
gz.close()
tar.close()
unpacked = Path(temp_dir) / "unpacked"
sdist_dir = unpacked / Path(filename).name.rstrip(suffix)
# Checking for .egg-info at root
eggs = list(sdist_dir.glob("*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Searching for .egg-info in sub directories
eggs = list(sdist_dir.glob("**/*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Still nothing, try reading (without executing it)
# the setup.py file.
try:
setup_info = self._inspect_sdist_with_setup(sdist_dir)
for key, value in info.items():
if value:
continue
info[key] = setup_info[key]
return info
except Exception as e:
self._log(
"An error occurred when reading setup.py or setup.cfg: {}".
format(str(e)),
"warning",
)
return info
def _inspect_sdist_with_setup(self, sdist_dir):
info = {"requires_python": None, "requires_dist": None}
result = SetupReader.read_from_directory(sdist_dir)
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
info["requires_dist"] = parse_requires(requires)
info["requires_python"] = result["python_requires"]
return info
def _download(self, url, dest): # type: (str, str) -> None
r = get(url, stream=True)
r.raise_for_status()
with open(dest, "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
def _log(self, msg, level="info"):
getattr(logger,
level)("<comment>{}:</comment> {}".format(self._name, msg))
class PyPiRepository(Repository):
def __init__(self, url="https://pypi.org/", disable_cache=False, fallback=True):
self._name = "PyPI"
self._url = url
self._disable_cache = disable_cache
self._fallback = fallback
release_cache_dir = Path(CACHE_DIR) / "cache" / "repositories" / "pypi"
self._cache = CacheManager(
{
"default": "releases",
"serializer": "json",
"stores": {
"releases": {"driver": "file", "path": str(release_cache_dir)},
"packages": {"driver": "dict"},
},
}
)
self._session = CacheControl(
session(), cache=FileCache(str(release_cache_dir / "_http"))
)
super(PyPiRepository, self).__init__()
def find_packages(
self,
name, # type: str
constraint=None, # type: Union[VersionConstraint, str, None]
extras=None, # type: Union[list, None]
allow_prereleases=False, # type: bool
): # type: (...) -> List[Package]
"""
Find packages on the remote server.
"""
if constraint is None:
constraint = "*"
if not isinstance(constraint, VersionConstraint):
constraint = parse_constraint(constraint)
info = self.get_package_info(name)
packages = []
for version, release in info["releases"].items():
if not release:
# Bad release
self._log(
"No release information found for {}-{}, skipping".format(
name, version
),
level="debug",
)
continue
package = Package(name, version)
if (
package.is_prerelease()
and not allow_prereleases
and not constraint.allows(package.version)
):
continue
if not constraint or (constraint and constraint.allows(package.version)):
if extras is not None:
package.requires_extras = extras
packages.append(package)
self._log(
"{} packages found for {} {}".format(len(packages), name, str(constraint)),
level="debug",
)
return packages
def package(
self,
name, # type: str
version, # type: str
extras=None, # type: (Union[list, None])
): # type: (...) -> Union[Package, None]
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.extras:
for extra in dependency.extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get("summary", "")
if release_info["requires_python"]:
package.python_versions = release_info["requires_python"]
if release_info["platform"]:
package.platform = release_info["platform"]
# Adding hashes information
package.hashes = release_info["digests"]
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
return package
def search(self, query, mode=0):
results = []
search = {"name": query}
if mode == self.SEARCH_FULLTEXT:
search["summary"] = query
client = ServerProxy("https://pypi.python.org/pypi")
hits = client.search(search, "or")
for hit in hits:
result = Package(hit["name"], hit["version"], hit["version"])
result.description = to_str(hit["summary"])
results.append(result)
return results
def get_package_info(self, name): # type: (str) -> dict
"""
Return the package information given its name.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_package_info(name)
return self._cache.store("packages").remember_forever(
name, lambda: self._get_package_info(name)
)
def _get_package_info(self, name): # type: (str) -> dict
data = self._get("pypi/{}/json".format(name))
if data is None:
raise ValueError("Package [{}] not found.".format(name))
return data
def get_release_info(self, name, version): # type: (str, str) -> dict
"""
Return the release information given a package name and a version.
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_release_info(name, version)
return self._cache.remember_forever(
"{}:{}".format(name, version), lambda: self._get_release_info(name, version)
)
def _get_release_info(self, name, version): # type: (str, str) -> dict
self._log("Getting info for {} ({}) from PyPI".format(name, version), "debug")
json_data = self._get("pypi/{}/{}/json".format(name, version))
if json_data is None:
raise ValueError("Package [{}] not found.".format(name))
info = json_data["info"]
data = {
"name": info["name"],
"version": info["version"],
"summary": info["summary"],
"platform": info["platform"],
"requires_dist": info["requires_dist"],
"requires_python": info["requires_python"],
"digests": [],
"_fallback": False,
}
try:
version_info = json_data["releases"][version]
except KeyError:
version_info = []
for file_info in version_info:
data["digests"].append(file_info["digests"]["sha256"])
if self._fallback and data["requires_dist"] is None:
self._log("No dependencies found, downloading archives", level="debug")
# No dependencies set (along with other information)
# This might be due to actually no dependencies
# or badly set metadata when uploading
# So, we need to make sure there is actually no
# dependencies by introspecting packages
urls = {}
for url in json_data["urls"]:
# Only get sdist and universal wheels if they exist
dist_type = url["packagetype"]
if dist_type not in ["sdist", "bdist_wheel"]:
continue
if dist_type == "sdist" and "dist" not in urls:
urls[url["packagetype"]] = url["url"]
continue
if "bdist_wheel" in urls:
continue
# If bdist_wheel, check if it's universal
python_version = url["python_version"]
if python_version not in ["py2.py3", "py3", "py2"]:
continue
parts = urlparse.urlparse(url["url"])
filename = os.path.basename(parts.path)
if "-none-any" not in filename:
continue
if not urls:
# If we don't have urls, we try to take the first one
# we find and go from there
if not json_data["urls"]:
return data
for url in json_data["urls"]:
# Only get sdist and universal wheels if they exist
dist_type = url["packagetype"]
if dist_type != "bdist_wheel":
continue
urls[url["packagetype"]] = url["url"]
break
if not urls:
return data
info = self._get_info_from_urls(urls)
data["requires_dist"] = info["requires_dist"]
if not data["requires_python"]:
data["requires_python"] = info["requires_python"]
return data
def _get(self, endpoint): # type: (str) -> Union[dict, None]
json_response = self._session.get(self._url + endpoint)
if json_response.status_code == 404:
return None
json_data = json_response.json()
return json_data
def _get_info_from_urls(
self, urls
): # type: (Dict[str, str]) -> Dict[str, Union[str, List, None]]
if "bdist_wheel" in urls:
return self._get_info_from_wheel(urls["bdist_wheel"])
return self._get_info_from_sdist(urls["sdist"])
def _get_info_from_wheel(
self, url
): # type: (str) -> Dict[str, Union[str, List, None]]
info = {"summary": "", "requires_python": None, "requires_dist": None}
filename = os.path.basename(urlparse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = os.path.join(temp_dir, filename)
self._download(url, filepath)
try:
meta = pkginfo.Wheel(filepath)
except ValueError:
# Unable to determine dependencies
# Assume none
return info
if meta.summary:
info["summary"] = meta.summary or ""
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = meta.requires_dist
return info
def _get_info_from_sdist(
self, url
): # type: (str) -> Dict[str, Union[str, List, None]]
info = {"summary": "", "requires_python": None, "requires_dist": None}
filename = os.path.basename(urlparse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
try:
meta = pkginfo.SDist(str(filepath))
if meta.summary:
info["summary"] = meta.summary
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
return info
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
# Still not dependencies found
# So, we unpack and introspect
suffix = filepath.suffix
gz = None
if suffix == ".zip":
tar = zipfile.ZipFile(str(filepath))
else:
if suffix == ".bz2":
gz = BZ2File(str(filepath))
else:
gz = GzipFile(str(filepath))
tar = tarfile.TarFile(str(filepath), fileobj=gz)
try:
tar.extractall(os.path.join(temp_dir, "unpacked"))
finally:
if gz:
gz.close()
tar.close()
unpacked = Path(temp_dir) / "unpacked"
sdist_dir = unpacked / Path(filename).name.rstrip(".tar.gz")
# Checking for .egg-info at root
eggs = list(sdist_dir.glob("*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Searching for .egg-info in sub directories
eggs = list(sdist_dir.glob("**/*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Still nothing, assume no dependencies
# We could probably get them by executing
# python setup.py egg-info but I don't feel
# confortable executing a file just for the sake
# of getting dependencies.
return info
def _inspect_sdist_with_setup(self, sdist_dir):
info = {"requires_python": None, "requires_dist": None}
setup = sdist_dir / "setup.py"
if not setup.exists():
return info
venv = Venv.create(NullIO())
current_dir = os.getcwd()
os.chdir(sdist_dir.as_posix())
try:
venv.run("python", "setup.py", "egg_info")
egg_info = list(sdist_dir.glob("**/*.egg-info"))[0]
meta = pkginfo.UnpackedSDist(str(egg_info))
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
else:
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
info["requires_dist"] = parse_requires(f.read())
except Exception:
pass
os.chdir(current_dir)
return info
def _download(self, url, dest): # type: (str, str) -> None
r = get(url, stream=True)
with open(dest, "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
def _log(self, msg, level="info"):
getattr(logger, level)("<comment>{}:</comment> {}".format(self._name, msg))