async def __call__(self, in_q, out_q):
"""
Build and emit `DeclarativeContent` from the remote metadata.
Fetch and parse the remote metadata, use the Project Specifiers on the Remote
to determine which Python packages should be synced.
Args:
in_q (asyncio.Queue): Unused because the first stage doesn't read from an input queue.
out_q (asyncio.Queue): The out_q to send `DeclarativeContent` objects to.
"""
ps = ProjectSpecifier.objects.filter(remote=self.remote)
with ProgressBar(message='Fetching Project Metadata') as pb:
# Group multiple specifiers to the same project together, so that we only have to fetch
# the metadata once, and can re-use it if there are multiple specifiers.
for name, project_specifiers in groupby_unsorted(
ps, key=lambda x: x.name):
# Fetch the metadata from PyPI
pb.increment()
try:
metadata = await self.get_project_metadata(name)
except ClientResponseError as e:
# Project doesn't exist, log a message and move on
log.info(
_("HTTP 404 'Not Found' for url '{url}'\n"
"Does project '{name}' exist on the remote repository?"
).format(url=e.request_info.url, name=name))
continue
project_specifiers = list(project_specifiers)
# Determine which packages from the project match the criteria in the specifiers
packages = await self.get_relevant_packages(
metadata=metadata,
includes=[
specifier for specifier in project_specifiers
if not specifier.exclude
],
excludes=[
specifier for specifier in project_specifiers
if specifier.exclude
],
prereleases=self.remote.prereleases)
# For each package, create Declarative objects to pass into the next stage
for entry in packages:
url = entry.pop('url')
artifact = Artifact(sha256=entry.pop('sha256_digest'))
package = PythonPackageContent(**entry)
da = DeclarativeArtifact(artifact, url, entry['filename'],
self.remote)
dc = DeclarativeContent(content=package, d_artifacts=[da])
await out_q.put(dc)
await out_q.put(None)
async def create_content(self, pkg):
"""
Take the filtered package, separate into releases and
create a Content Unit to put into the pipeline
"""
for version, dists in pkg.releases.items():
for package in dists:
entry = parse_metadata(pkg.info, version, package)
url = entry.pop("url")
artifact = Artifact(sha256=entry["sha256"])
package = PythonPackageContent(**entry)
da = DeclarativeArtifact(
artifact,
url,
entry["filename"],
self.python_stage.remote,
deferred_download=self.deferred_download,
)
dc = DeclarativeContent(content=package, d_artifacts=[da])
await self.python_stage.put(dc)