async def _handle_distribution(self, distribution):
log.info(
_('Downloading Release file for distribution: "{}"').format(
distribution))
# Create release_file
if distribution[-1] == "/":
release_file_dir = distribution.strip("/")
else:
release_file_dir = os.path.join("dists", distribution)
release_file_dc = DeclarativeContent(
content=ReleaseFile(distribution=distribution),
d_artifacts=[
self._to_d_artifact(os.path.join(release_file_dir, filename))
for filename in ["Release", "InRelease", "Release.gpg"]
],
)
release_file = await self._create_unit(release_file_dc)
if release_file is None:
return
# Create release object
release_unit = Release(codename=release_file.codename,
suite=release_file.suite,
distribution=distribution)
release_dc = DeclarativeContent(content=release_unit)
release = await self._create_unit(release_dc)
# Create release architectures
architectures = _filter_split_architectures(release_file.architectures,
self.remote.architectures,
distribution)
for architecture in architectures:
release_architecture_dc = DeclarativeContent(
content=ReleaseArchitecture(architecture=architecture,
release=release))
await self.put(release_architecture_dc)
# Parse release file
log.info(
_('Parsing Release file at distribution="{}"').format(
distribution))
release_artifact = await _get_main_artifact_blocking(release_file)
release_file_dict = deb822.Release(release_artifact.file)
# collect file references in new dict
file_references = defaultdict(deb822.Deb822Dict)
for digest_name in ["SHA512", "SHA256", "SHA1", "MD5sum"]:
if digest_name in release_file_dict:
for unit in release_file_dict[digest_name]:
file_references[unit["Name"]].update(unit)
await asyncio.gather(*[
self._handle_component(component, release, release_file,
file_references, architectures)
for component in _filter_split_components(
release_file.components, self.remote.components, distribution)
])
async def run(self):
"""
Build and emit `DeclarativeContent` from the Release data.
"""
# TODO Merge into one list of futures
future_releases = []
future_package_indices = []
future_installer_file_indices = []
with ProgressReport(
message="Creating download requests for Release files",
code="download.release",
total=self.num_distributions,
) as pb:
for distribution in self.distributions:
log.info(
'Downloading Release file for distribution: "{}"'.format(
distribution))
release_relpath = os.path.join("dists", distribution,
"Release")
release_path = os.path.join(self.parsed_url.path,
release_relpath)
release_da = DeclarativeArtifact(
Artifact(),
urlunparse(self.parsed_url._replace(path=release_path)),
release_relpath,
self.remote,
deferred_download=False,
)
release_gpg_relpath = os.path.join("dists", distribution,
"Release.gpg")
release_gpg_path = os.path.join(self.parsed_url.path,
release_gpg_relpath)
release_gpg_da = DeclarativeFailsafeArtifact(
Artifact(),
urlunparse(
self.parsed_url._replace(path=release_gpg_path)),
release_gpg_relpath,
self.remote,
deferred_download=False,
)
inrelease_relpath = os.path.join("dists", distribution,
"InRelease")
inrelease_path = os.path.join(self.parsed_url.path,
inrelease_relpath)
inrelease_da = DeclarativeFailsafeArtifact(
Artifact(),
urlunparse(self.parsed_url._replace(path=inrelease_path)),
inrelease_relpath,
self.remote,
deferred_download=False,
)
release_unit = Release(distribution=distribution,
relative_path=release_relpath)
release_dc = DeclarativeContent(
content=release_unit,
d_artifacts=[release_da, release_gpg_da, inrelease_da],
does_batch=False,
)
future_releases.append(release_dc.get_or_create_future())
await self.put(release_dc)
pb.increment()
with ProgressReport(message="Parsing Release files",
code="parsing.release",
total=self.num_distributions) as pb:
for release_future in asyncio.as_completed(future_releases):
release = await release_future
if release is None:
continue
log.info('Parsing Release file for release: "{}"'.format(
release.codename))
release_artifact = release._artifacts.get(
sha256=release.sha256)
release_dict = deb822.Release(release_artifact.file)
async for d_content in self._read_release_file(
release, release_dict):
if isinstance(d_content.content, PackageIndex):
future_package_indices.append(
d_content.get_or_create_future())
if isinstance(d_content.content, InstallerFileIndex):
future_installer_file_indices.append(
d_content.get_or_create_future())
await self.put(d_content)
pb.increment()
with ProgressReport(message="Parsing package index files",
code="parsing.packageindex") as pb:
for package_index_future in asyncio.as_completed(
future_package_indices):
package_index = await package_index_future
if package_index is None:
continue
package_index_artifact = package_index.main_artifact
log.debug("Parsing package index for {}:{}.".format(
package_index.component, package_index.architecture))
async for package_dc in self._read_package_index(
package_index_artifact.file):
await self.put(package_dc)
pb.increment()
with ProgressReport(message="Parsing installer file index files",
code="parsing.installer") as pb:
for installer_file_index_future in asyncio.as_completed(
future_installer_file_indices):
installer_file_index = await installer_file_index_future
if installer_file_index is None:
continue
log.debug("Parsing installer file index for {}:{}.".format(
installer_file_index.component,
installer_file_index.architecture))
async for d_content in self._read_installer_file_index(
installer_file_index):
await self.put(d_content)
pb.increment()
async def _handle_distribution(self, distribution):
log.info(
_('Downloading Release file for distribution: "{}"').format(
distribution))
# Create release_file
if distribution[-1] == "/":
release_file_dir = distribution.strip("/")
else:
release_file_dir = os.path.join("dists", distribution)
release_file_dc = DeclarativeContent(
content=ReleaseFile(distribution=distribution),
d_artifacts=[
self._to_d_artifact(os.path.join(release_file_dir, filename))
for filename in ["Release", "InRelease", "Release.gpg"]
],
)
release_file = await self._create_unit(release_file_dc)
if release_file is None:
return
# Create release object
release_unit = Release(codename=release_file.codename,
suite=release_file.suite,
distribution=distribution)
release_dc = DeclarativeContent(content=release_unit)
release = await self._create_unit(release_dc)
# Create release architectures
if release_file.architectures:
architectures = _filter_split_architectures(
release_file.architectures, self.remote.architectures,
distribution)
elif distribution[-1] == "/":
message = (
"The ReleaseFile content unit architecrures are unset for the flat repo with "
"distribution '{}'. ReleaseArchitecture content creation is deferred!"
)
log.warning(_(message).format(distribution))
architectures = []
for architecture in architectures:
release_architecture_dc = DeclarativeContent(
content=ReleaseArchitecture(architecture=architecture,
release=release))
await self.put(release_architecture_dc)
# Parse release file
log.info(
_('Parsing Release file at distribution="{}"').format(
distribution))
release_artifact = await _get_main_artifact_blocking(release_file)
release_file_dict = deb822.Release(release_artifact.file)
# Retrieve and interpret any 'No-Support-for-Architecture-all' value:
# We will refer to the presence of 'No-Support-for-Architecture-all: Packages' in a Release
# file as indicating "hybrid format". For more info, see:
# https://wiki.debian.org/DebianRepository/Format#No-Support-for-Architecture-all
no_support_for_arch_all = release_file_dict.get(
"No-Support-for-Architecture-all", "")
if no_support_for_arch_all.strip() == "Packages":
hybrid_format = True
elif not no_support_for_arch_all:
hybrid_format = False
else:
raise UnknownNoSupportForArchitectureAllValue(
release_file.relative_path, no_support_for_arch_all)
# collect file references in new dict
file_references = defaultdict(deb822.Deb822Dict)
for digest_name in ["SHA512", "SHA256", "SHA1", "MD5sum"]:
if digest_name in release_file_dict:
for unit in release_file_dict[digest_name]:
file_references[unit["Name"]].update(unit)
if distribution[-1] == "/":
# Handle flat repo
sub_tasks = [
self._handle_flat_repo(file_references, release_file, release)
]
else:
# Handle components
sub_tasks = [
self._handle_component(
component,
release,
release_file,
file_references,
architectures,
hybrid_format,
) for component in _filter_split_components(
release_file.components, self.remote.components,
distribution)
]
await asyncio.gather(*sub_tasks)