def _fetch_specified_metadata(remote, project_specifiers):
"""
Fetch metadata for content units matching project specifiers from the remote.
Args:
project_specifiers (dict): Information about a project and which versions of a project
to filter
Returns:
list: of contentunit metadata.
"""
remote_units = []
for project in project_specifiers:
digests = python_models.DistributionDigest.objects.filter(
project_specifier=project)
metadata_url = urljoin(remote.url, 'pypi/%s/json' % project.name)
downloader = remote.get_downloader(metadata_url)
downloader.fetch()
metadata = json.load(open(downloader.path))
for version, packages in metadata['releases'].items():
for package in packages:
# If neither specifiers nor digests have been set, then we should add the unit
if not project.version_specifier and not digests.exists():
remote_units.append(
parse_metadata(metadata['info'], version, package))
continue
specifier = specifiers.SpecifierSet(project.version_specifier)
# Note: SpecifierSet("").contains(version) will return true for released versions
# SpecifierSet("").contains('3.0.0') returns True
# SpecifierSet("").contains('3.0.0b1') returns False
if specifier.contains(version):
# add the package if the project specifier does not have an associated digest
if not digests.exists():
remote_units.append(
parse_metadata(metadata['info'], version, package))
# otherwise check each digest to see if it matches the specifier
else:
for type, digest in package['digests'].items():
if digests.filter(type=type,
digest=digest).exists():
remote_units.append(
parse_metadata(metadata['info'], version,
package))
break
return remote_units
async def create_content(self, pkg):
"""
Take the filtered package, separate into releases and
create a Content Unit to put into the pipeline
"""
for version, dists in pkg.releases.items():
for package in dists:
entry = parse_metadata(pkg.info, version, package)
url = entry.pop("url")
artifact = Artifact(sha256=entry["sha256"])
package = PythonPackageContent(**entry)
da = DeclarativeArtifact(
artifact,
url,
entry["filename"],
self.python_stage.remote,
deferred_download=self.deferred_download,
)
dc = DeclarativeContent(content=package, d_artifacts=[da])
await self.python_stage.put(dc)
async def get_relevant_packages(self, metadata, includes, excludes,
prereleases):
"""
Provided project metadata and specifiers, return the matching packages.
Compare the defined specifiers against the project metadata and create a deduplicated
list of metadata for the packages matching the criteria.
Args:
metadata (dict): Metadata about the project from PyPI.
includes (iterable): An iterable of project_specifiers for package versions to include.
excludes (iterable): An iterable of project_specifiers for package versions to exclude.
prereleases (bool): Whether or not to include pre-release package versions in the sync.
Returns:
list: List of dictionaries containing Python package metadata
"""
# The set of project release metadata, in the format {"version": [package1, package2, ...]}
releases = metadata['releases']
# The packages we want to return
remote_packages = []
# Delete versions/packages matching the exclude specifiers.
for exclude_specifier in excludes:
# Fast path: If one of the specifiers matches all versions and we don't have any
# digests to reference, clear the whole dict, we're done.
if not exclude_specifier.version_specifier:
releases.clear()
break
# Slow path: We have to check all the metadata.
for version, packages in list(
releases.items()): # Prevent iterator invalidation.
specifier = specifiers.SpecifierSet(
exclude_specifier.version_specifier,
prereleases=prereleases)
# First check the version specifer, if it matches, check the digests and delete
# matching packages. If there are no digests, delete them all.
if specifier.contains(version):
del releases[version]
for version, packages in releases.items():
for include_specifier in includes:
# Fast path: If one of the specifiers matches all versions and we don't have any
# digests to reference, return all of the packages for the version.
if prereleases and not include_specifier.version_specifier:
for package in packages:
remote_packages.append(
parse_metadata(metadata['info'], version, package))
# This breaks the inner loop, e.g. don't check any other include_specifiers.
# We want to continue the outer loop.
break
specifier = specifiers.SpecifierSet(
include_specifier.version_specifier,
prereleases=prereleases)
# First check the version specifer, if it matches, check the digests and include
# matching packages. If there are no digests, include them all.
if specifier.contains(version):
for package in packages:
remote_packages.append(
parse_metadata(metadata['info'], version, package))
return remote_packages
async def get_relevant_packages(self, metadata, includes, excludes, prereleases):
"""
Provided project metadata and specifiers, return the matching packages.
Compare the defined specifiers against the project metadata and create a deduplicated
list of metadata for the packages matching the criteria.
Args:
metadata (dict): Metadata about the project from PyPI.
includes (iterable): An iterable of project_specifiers for package versions to include.
excludes (iterable): An iterable of project_specifiers for package versions to exclude.
prereleases (bool): Whether or not to include pre-release package versions in the sync.
Returns:
list: List of dictionaries containing Python package metadata
"""
# The set of project release metadata, in the format {"version": [package1, package2, ...]}
releases = metadata['releases']
# The packages we want to return
remote_packages = []
# Delete versions/packages matching the exclude specifiers.
for exclude_specifier in excludes:
# Fast path: If one of the specifiers matches all versions and we don't have any
# digests to reference, clear the whole dict, we're done.
if not exclude_specifier.version_specifier:
releases.clear()
break
# Slow path: We have to check all the metadata.
for version, packages in list(releases.items()): # Prevent iterator invalidation.
specifier = specifiers.SpecifierSet(
exclude_specifier.version_specifier,
prereleases=prereleases
)
# First check the version specifer, if it matches, check the digests and delete
# matching packages. If there are no digests, delete them all.
if specifier.contains(version):
del releases[version]
for version, packages in releases.items():
for include_specifier in includes:
# Fast path: If one of the specifiers matches all versions and we don't have any
# digests to reference, return all of the packages for the version.
if prereleases and not include_specifier.version_specifier:
for package in packages:
remote_packages.append(parse_metadata(metadata['info'], version, package))
# This breaks the inner loop, e.g. don't check any other include_specifiers.
# We want to continue the outer loop.
break
specifier = specifiers.SpecifierSet(
include_specifier.version_specifier,
prereleases=prereleases
)
# First check the version specifer, if it matches, check the digests and include
# matching packages. If there are no digests, include them all.
if specifier.contains(version):
for package in packages:
remote_packages.append(
parse_metadata(metadata['info'], version, package)
)
return remote_packages