async def run(self):
"""
Build and emit `DeclarativeContent` from the ansible metadata.
"""
with ProgressBar(message='Parsing Role Metadata') as pb:
pending = []
async for metadata in self._fetch_roles():
role = AnsibleRole(name=metadata['name'],
namespace=metadata['namespace'])
d_content = DeclarativeContent(content=role,
d_artifacts=[],
does_batch=False)
pending.append(
asyncio.ensure_future(
self._add_role_versions(
d_content.get_or_create_future(),
metadata,
)))
await self.put(d_content)
pb.increment()
await asyncio.gather(*pending)
async def run(self):
"""
Build and emit `DeclarativeContent` from the Release data.
"""
# TODO Merge into one list of futures
future_releases = []
future_package_indices = []
future_installer_file_indices = []
with ProgressReport(
message="Creating download requests for Release files",
code="download.release",
total=self.num_distributions,
) as pb:
for distribution in self.distributions:
log.info(
'Downloading Release file for distribution: "{}"'.format(
distribution))
release_relpath = os.path.join("dists", distribution,
"Release")
release_path = os.path.join(self.parsed_url.path,
release_relpath)
release_da = DeclarativeArtifact(
Artifact(),
urlunparse(self.parsed_url._replace(path=release_path)),
release_relpath,
self.remote,
deferred_download=False,
)
release_gpg_relpath = os.path.join("dists", distribution,
"Release.gpg")
release_gpg_path = os.path.join(self.parsed_url.path,
release_gpg_relpath)
release_gpg_da = DeclarativeFailsafeArtifact(
Artifact(),
urlunparse(
self.parsed_url._replace(path=release_gpg_path)),
release_gpg_relpath,
self.remote,
deferred_download=False,
)
inrelease_relpath = os.path.join("dists", distribution,
"InRelease")
inrelease_path = os.path.join(self.parsed_url.path,
inrelease_relpath)
inrelease_da = DeclarativeFailsafeArtifact(
Artifact(),
urlunparse(self.parsed_url._replace(path=inrelease_path)),
inrelease_relpath,
self.remote,
deferred_download=False,
)
release_unit = Release(distribution=distribution,
relative_path=release_relpath)
release_dc = DeclarativeContent(
content=release_unit,
d_artifacts=[release_da, release_gpg_da, inrelease_da],
does_batch=False,
)
future_releases.append(release_dc.get_or_create_future())
await self.put(release_dc)
pb.increment()
with ProgressReport(message="Parsing Release files",
code="parsing.release",
total=self.num_distributions) as pb:
for release_future in asyncio.as_completed(future_releases):
release = await release_future
if release is None:
continue
log.info('Parsing Release file for release: "{}"'.format(
release.codename))
release_artifact = release._artifacts.get(
sha256=release.sha256)
release_dict = deb822.Release(release_artifact.file)
async for d_content in self._read_release_file(
release, release_dict):
if isinstance(d_content.content, PackageIndex):
future_package_indices.append(
d_content.get_or_create_future())
if isinstance(d_content.content, InstallerFileIndex):
future_installer_file_indices.append(
d_content.get_or_create_future())
await self.put(d_content)
pb.increment()
with ProgressReport(message="Parsing package index files",
code="parsing.packageindex") as pb:
for package_index_future in asyncio.as_completed(
future_package_indices):
package_index = await package_index_future
if package_index is None:
continue
package_index_artifact = package_index.main_artifact
log.debug("Parsing package index for {}:{}.".format(
package_index.component, package_index.architecture))
async for package_dc in self._read_package_index(
package_index_artifact.file):
await self.put(package_dc)
pb.increment()
with ProgressReport(message="Parsing installer file index files",
code="parsing.installer") as pb:
for installer_file_index_future in asyncio.as_completed(
future_installer_file_indices):
installer_file_index = await installer_file_index_future
if installer_file_index is None:
continue
log.debug("Parsing installer file index for {}:{}.".format(
installer_file_index.component,
installer_file_index.architecture))
async for d_content in self._read_installer_file_index(
installer_file_index):
await self.put(d_content)
pb.increment()
