async def run(self): """ Build and emit `DeclarativeContent` from the ansible metadata. """ with ProgressBar(message='Parsing Role Metadata') as pb: pending = [] async for metadata in self._fetch_roles(): role = AnsibleRole(name=metadata['name'], namespace=metadata['namespace']) d_content = DeclarativeContent(content=role, d_artifacts=[], does_batch=False) pending.append( asyncio.ensure_future( self._add_role_versions( d_content.get_or_create_future(), metadata, ))) await self.put(d_content) pb.increment() await asyncio.gather(*pending)
async def run(self): """ Build and emit `DeclarativeContent` from the Release data. """ # TODO Merge into one list of futures future_releases = [] future_package_indices = [] future_installer_file_indices = [] with ProgressReport( message="Creating download requests for Release files", code="download.release", total=self.num_distributions, ) as pb: for distribution in self.distributions: log.info( 'Downloading Release file for distribution: "{}"'.format( distribution)) release_relpath = os.path.join("dists", distribution, "Release") release_path = os.path.join(self.parsed_url.path, release_relpath) release_da = DeclarativeArtifact( Artifact(), urlunparse(self.parsed_url._replace(path=release_path)), release_relpath, self.remote, deferred_download=False, ) release_gpg_relpath = os.path.join("dists", distribution, "Release.gpg") release_gpg_path = os.path.join(self.parsed_url.path, release_gpg_relpath) release_gpg_da = DeclarativeFailsafeArtifact( Artifact(), urlunparse( self.parsed_url._replace(path=release_gpg_path)), release_gpg_relpath, self.remote, deferred_download=False, ) inrelease_relpath = os.path.join("dists", distribution, "InRelease") inrelease_path = os.path.join(self.parsed_url.path, inrelease_relpath) inrelease_da = DeclarativeFailsafeArtifact( Artifact(), urlunparse(self.parsed_url._replace(path=inrelease_path)), inrelease_relpath, self.remote, deferred_download=False, ) release_unit = Release(distribution=distribution, relative_path=release_relpath) release_dc = DeclarativeContent( content=release_unit, d_artifacts=[release_da, release_gpg_da, inrelease_da], does_batch=False, ) future_releases.append(release_dc.get_or_create_future()) await self.put(release_dc) pb.increment() with ProgressReport(message="Parsing Release files", code="parsing.release", total=self.num_distributions) as pb: for release_future in asyncio.as_completed(future_releases): release = await release_future if release is None: continue log.info('Parsing Release file for release: "{}"'.format( release.codename)) release_artifact = release._artifacts.get( sha256=release.sha256) release_dict = deb822.Release(release_artifact.file) async for d_content in self._read_release_file( release, release_dict): if isinstance(d_content.content, PackageIndex): future_package_indices.append( d_content.get_or_create_future()) if isinstance(d_content.content, InstallerFileIndex): future_installer_file_indices.append( d_content.get_or_create_future()) await self.put(d_content) pb.increment() with ProgressReport(message="Parsing package index files", code="parsing.packageindex") as pb: for package_index_future in asyncio.as_completed( future_package_indices): package_index = await package_index_future if package_index is None: continue package_index_artifact = package_index.main_artifact log.debug("Parsing package index for {}:{}.".format( package_index.component, package_index.architecture)) async for package_dc in self._read_package_index( package_index_artifact.file): await self.put(package_dc) pb.increment() with ProgressReport(message="Parsing installer file index files", code="parsing.installer") as pb: for installer_file_index_future in asyncio.as_completed( future_installer_file_indices): installer_file_index = await installer_file_index_future if installer_file_index is None: continue log.debug("Parsing installer file index for {}:{}.".format( installer_file_index.component, installer_file_index.architecture)) async for d_content in self._read_installer_file_index( installer_file_index): await self.put(d_content) pb.increment()