async def run(self):
"""
Build and emit `DeclarativeContent` from the ansible metadata.
"""
with ProgressReport(message="Parsing Collection Metadata",
code="parsing.metadata") as pb:
async for metadata in self._fetch_collections():
url = metadata["download_url"]
collection_version = CollectionVersion(
namespace=metadata["namespace"]["name"],
name=metadata["collection"]["name"],
version=metadata["version"],
)
artifact = metadata["artifact"]
d_artifact = DeclarativeArtifact(
artifact=Artifact(sha256=artifact["sha256"],
size=artifact["size"]),
url=url,
relative_path=collection_version.relative_path,
remote=self.remote,
deferred_download=self.deferred_download,
)
d_content = DeclarativeContent(content=collection_version,
d_artifacts=[d_artifact])
pb.increment()
await self.put(d_content)
def create_collection_from_importer(importer_result, metadata_only=False):
"""
Process results from importer.
"""
collection_info = importer_result["metadata"]
with transaction.atomic():
collection, created = Collection.objects.get_or_create(
namespace=collection_info["namespace"],
name=collection_info["name"])
tags = collection_info.pop("tags")
# Remove fields not used by this model
collection_info.pop("license_file")
collection_info.pop("readme")
# the importer returns many None values. We need to let the defaults in the model prevail
for key in [
"description", "documentation", "homepage", "issues",
"repository"
]:
if collection_info[key] is None:
collection_info.pop(key)
collection_version = CollectionVersion(
collection=collection,
**collection_info,
requires_ansible=importer_result.get("requires_ansible"),
contents=importer_result["contents"],
docs_blob=importer_result["docs_blob"],
)
serializer_fields = CollectionVersionSerializer.Meta.fields
data = {
k: v
for k, v in collection_version.__dict__.items()
if k in serializer_fields
}
data["id"] = collection_version.pulp_id
if not metadata_only:
data["artifact"] = importer_result["artifact_url"]
serializer = CollectionVersionSerializer(data=data)
serializer.is_valid(raise_exception=True)
collection_version.save()
for name in tags:
tag, created = Tag.objects.get_or_create(name=name)
collection_version.tags.add(tag)
_update_highest_version(collection_version)
collection_version.save() # Save the FK updates
return collection_version
async def run(self):
"""
Build and emit `DeclarativeContent` from the ansible metadata.
"""
msg = "Parsing CollectionVersion Metadata"
with ProgressReport(message=msg, code="parsing.metadata") as pb:
async for metadata in self._fetch_collections():
url = metadata["download_url"]
collection_version = CollectionVersion(
namespace=metadata["namespace"]["name"],
name=metadata["collection"]["name"],
version=metadata["version"],
)
info = metadata["metadata"]
info.pop("tags")
for attr_name, attr_value in info.items():
if attr_value is None or attr_name not in collection_version.__dict__:
continue
setattr(collection_version, attr_name, attr_value)
artifact = metadata["artifact"]
d_artifact = DeclarativeArtifact(
artifact=Artifact(sha256=artifact["sha256"], size=artifact["size"]),
url=url,
relative_path=collection_version.relative_path,
remote=self.remote,
deferred_download=self.deferred_download,
)
extradata = dict(
docs_blob_url=metadata["docs_blob_url"],
deprecated=metadata["deprecated"],
)
d_content = DeclarativeContent(
content=collection_version,
d_artifacts=[d_artifact],
extra_data=extradata,
)
pb.increment()
await self.put(d_content)
def sign(repository_href, content_hrefs, signing_service_href):
"""The signing task."""
repository = AnsibleRepository.objects.get(pk=repository_href)
if content_hrefs == ["*"]:
filtered = repository.latest_version().content.filter(
pulp_type=CollectionVersion.get_pulp_type())
content = CollectionVersion.objects.filter(pk__in=filtered)
else:
content = CollectionVersion.objects.filter(pk__in=content_hrefs)
signing_service = SigningService.objects.get(pk=signing_service_href)
filtered_sigs = repository.latest_version().content.filter(
pulp_type=CollectionVersionSignature.get_pulp_type())
repos_current_signatures = CollectionVersionSignature.objects.filter(
pk__in=filtered_sigs)
first_stage = CollectionSigningFirstStage(content, signing_service,
repos_current_signatures)
SigningDeclarativeVersion(first_stage, repository).create()
async def _add_collection_version(self, api_version,
collection_version_url, metadata):
"""Add CollectionVersion to the sync pipeline."""
url = metadata["download_url"]
collection_version = CollectionVersion(
namespace=metadata["namespace"]["name"],
name=metadata["collection"]["name"],
version=metadata["version"],
)
info = metadata["metadata"]
info.pop("tags")
for attr_name, attr_value in info.items():
if attr_value is None or attr_name not in collection_version.__dict__:
continue
setattr(collection_version, attr_name, attr_value)
artifact = metadata["artifact"]
d_artifact = DeclarativeArtifact(
artifact=Artifact(sha256=artifact["sha256"],
size=artifact["size"]),
url=url,
relative_path=collection_version.relative_path,
remote=self.remote,
deferred_download=self.deferred_download,
)
extra_data = {}
if api_version != 2: # V2 never implemented the docs-blob requests
extra_data["docs_blob_url"] = f"{collection_version_url}docs-blob/"
d_content = DeclarativeContent(
content=collection_version,
d_artifacts=[d_artifact],
extra_data=extra_data,
)
self.parsing_metadata_progress_bar.increment()
await self.put(d_content)
def create_collection_from_importer(importer_result):
"""
Process results from importer.
"""
collection_info = importer_result["metadata"]
with transaction.atomic():
collection, created = Collection.objects.get_or_create(
namespace=collection_info["namespace"],
name=collection_info["name"])
tags = collection_info.pop("tags")
# Remove fields not used by this model
collection_info.pop("license_file")
collection_info.pop("readme")
# the importer returns many None values. We need to let the defaults in the model prevail
for key in [
"description", "documentation", "homepage", "issues",
"repository"
]:
if collection_info[key] is None:
collection_info.pop(key)
collection_version = CollectionVersion(
collection=collection,
**collection_info,
contents=importer_result["contents"],
docs_blob=importer_result["docs_blob"],
)
collection_version.save()
for name in tags:
tag, created = Tag.objects.get_or_create(name=name)
collection_version.tags.add(tag)
_update_highest_version(collection_version)
collection_version.save() # Save the FK updates
return collection_version
async def _add_collection_version(self, api_version,
collection_version_url, metadata):
"""Add CollectionVersion to the sync pipeline."""
url = metadata["download_url"]
collection_version = CollectionVersion(
namespace=metadata["namespace"]["name"],
name=metadata["collection"]["name"],
version=metadata["version"],
)
cv_unique = attrgetter("namespace", "name",
"version")(collection_version)
if cv_unique in self.already_synced:
return
self.already_synced.add(cv_unique)
info = metadata["metadata"]
if self.add_dependents:
dependencies = info["dependencies"]
tasks = []
loop = asyncio.get_event_loop()
for full_name, version in dependencies.items():
namespace, name = full_name.split(".")
if not (namespace, name, version) in self.already_synced:
new_req = RequirementsFileEntry(
name=full_name,
version=version,
source=None,
)
tasks.append(
loop.create_task(
self._fetch_collection_metadata(new_req)))
await asyncio.gather(*tasks)
info.pop("tags")
for attr_name, attr_value in info.items():
if attr_value is None or attr_name not in collection_version.__dict__:
continue
setattr(collection_version, attr_name, attr_value)
artifact = metadata["artifact"]
d_artifact = DeclarativeArtifact(
artifact=Artifact(sha256=artifact["sha256"],
size=artifact["size"]),
url=url,
relative_path=collection_version.relative_path,
remote=self.remote,
deferred_download=self.deferred_download,
)
extra_data = {}
if api_version != 2: # V2 never implemented the docs-blob requests
extra_data["docs_blob_url"] = f"{collection_version_url}docs-blob/"
d_content = DeclarativeContent(
content=collection_version,
d_artifacts=[d_artifact],
extra_data=extra_data,
)
self.parsing_metadata_progress_bar.increment()
await self.put(d_content)
async def _add_collection_version(self, api_version,
collection_version_url, metadata):
"""Add CollectionVersion to the sync pipeline."""
url = metadata["download_url"]
collection_version = CollectionVersion(
namespace=metadata["namespace"]["name"],
name=metadata["collection"]["name"],
version=metadata["version"],
)
cv_unique = attrgetter("namespace", "name",
"version")(collection_version)
fullname, version = f"{cv_unique[0]}.{cv_unique[1]}", cv_unique[2]
if fullname in self.exclude_info and version in self.exclude_info[
fullname]:
return
if cv_unique in self.already_synced:
return
self.already_synced.add(cv_unique)
info = metadata["metadata"]
signatures = metadata.get("signatures")
if self.signed_only and not signatures:
return
if self.add_dependents:
dependencies = info["dependencies"]
tasks = []
loop = asyncio.get_event_loop()
for full_name, version in dependencies.items():
namespace, name = full_name.split(".")
req = (namespace, name, version)
new_req = RequirementsFileEntry(full_name,
version=version,
source=None)
if not any([
req in self.already_synced, new_req
in self.collection_info
]):
self.collection_info.append(new_req)
tasks.append(
loop.create_task(
self._fetch_collection_metadata(new_req)))
await asyncio.gather(*tasks)
info.pop("tags")
for attr_name, attr_value in info.items():
if attr_value is None or attr_name not in collection_version.__dict__:
continue
setattr(collection_version, attr_name, attr_value)
artifact = metadata["artifact"]
d_artifact = DeclarativeArtifact(
artifact=Artifact(sha256=artifact["sha256"],
size=artifact["size"]),
url=url,
relative_path=collection_version.relative_path,
remote=self.remote,
deferred_download=self.deferred_download,
)
extra_data = {}
if api_version != 2: # V2 never implemented the docs-blob requests
extra_data["docs_blob_url"] = f"{collection_version_url}docs-blob/"
d_content = DeclarativeContent(
content=collection_version,
d_artifacts=[d_artifact],
extra_data=extra_data,
)
await self.parsing_metadata_progress_bar.aincrement()
await self.put(d_content)
if signatures:
collection_version = await d_content.resolution()
for signature in signatures:
sig = signature["signature"].encode("utf8")
cv_signature = CollectionVersionSignature(
signed_collection=collection_version,
data=sig,
digest=hashlib.sha256(sig).hexdigest(),
pubkey_fingerprint=signature["pubkey_fingerprint"],
)
await self.put(DeclarativeContent(content=cv_signature))
def import_collection(
artifact_pk,
repository_pk=None,
expected_namespace=None,
expected_name=None,
expected_version=None,
):
"""
Create a Collection from an uploaded artifact and optionally validate its expected metadata.
This task provides optional validation of the `namespace`, `name`, and `version` metadata
attributes. If the Artifact fails validation or parsing, the Artifact is deleted and the
Collection is not created.
This task performs a CollectionImport object get_or_create() to allow import messages to be
logged.
Args:
artifact_pk (str): The pk of the Artifact to create the Collection from.
Keyword Args:
repository_pk (str): Optional. If specified, a new RepositoryVersion will be created for the
Repository and any new Collection content associated with it.
expected_namespace (str): Optional. The namespace is validated against the namespace
specified in the Collection's metadata. If it does not match a ImporterError is
raised.
expected_name (str): Optional. The name is validated against the name specified in the
Collection's metadata. If it does not match a ImporterError is raised.
expected_version (str): Optional. The version is validated against the version specified in
the Collection's metadata. If it does not match a ImporterError is raised.
Raises:
ImporterError: If the `expected_namespace`, `expected_name`, or `expected_version` do not
match the metadata in the tarball.
"""
CollectionImport.objects.get_or_create(task_id=get_current_job().id)
artifact = Artifact.objects.get(pk=artifact_pk)
filename = CollectionFilename(expected_namespace, expected_name, expected_version)
log.info(f"Processing collection from {artifact.file.name}")
import_logger = logging.getLogger("pulp_ansible.app.tasks.collection.import_collection")
with _artifact_guard(artifact):
try:
with artifact.file.open() as artifact_file:
importer_result = process_collection(
artifact_file, filename=filename, logger=import_logger
)
except ImporterError as exc:
log.info(f"Collection processing was not successfull: {exc}")
raise
collection_info = importer_result["metadata"]
with transaction.atomic():
collection, created = Collection.objects.get_or_create(
namespace=collection_info["namespace"], name=collection_info["name"]
)
tags = collection_info.pop("tags")
# Remove fields not used by this model
collection_info.pop("license_file")
collection_info.pop("readme")
# the importer returns many None values. We need to let the defaults in the model prevail
for key in ["description", "documentation", "homepage", "issues", "repository"]:
if collection_info[key] is None:
collection_info.pop(key)
collection_version = CollectionVersion(
collection=collection,
**collection_info,
contents=importer_result["contents"],
docs_blob=importer_result["docs_blob"],
)
collection_version.save()
for name in tags:
tag, created = Tag.objects.get_or_create(name=name)
collection_version.tags.add(tag)
_update_highest_version(collection_version)
collection_version.save() # Save the FK updates
ContentArtifact.objects.create(
artifact=artifact,
content=collection_version,
relative_path=collection_version.relative_path,
)
CreatedResource.objects.create(content_object=collection_version)
if repository_pk:
repository = Repository.objects.get(pk=repository_pk)
content_q = CollectionVersion.objects.filter(pk=collection_version.pk)
with RepositoryVersion.create(repository) as new_version:
new_version.add_content(content_q)
CreatedResource.objects.create(content_object=repository)