async def run(self): """ Build and emit `DeclarativeContent` from the ansible metadata. """ with ProgressReport(message="Parsing Collection Metadata", code="parsing.metadata") as pb: async for metadata in self._fetch_collections(): url = metadata["download_url"] collection_version = CollectionVersion( namespace=metadata["namespace"]["name"], name=metadata["collection"]["name"], version=metadata["version"], ) artifact = metadata["artifact"] d_artifact = DeclarativeArtifact( artifact=Artifact(sha256=artifact["sha256"], size=artifact["size"]), url=url, relative_path=collection_version.relative_path, remote=self.remote, deferred_download=self.deferred_download, ) d_content = DeclarativeContent(content=collection_version, d_artifacts=[d_artifact]) pb.increment() await self.put(d_content)
def create_collection_from_importer(importer_result, metadata_only=False): """ Process results from importer. """ collection_info = importer_result["metadata"] with transaction.atomic(): collection, created = Collection.objects.get_or_create( namespace=collection_info["namespace"], name=collection_info["name"]) tags = collection_info.pop("tags") # Remove fields not used by this model collection_info.pop("license_file") collection_info.pop("readme") # the importer returns many None values. We need to let the defaults in the model prevail for key in [ "description", "documentation", "homepage", "issues", "repository" ]: if collection_info[key] is None: collection_info.pop(key) collection_version = CollectionVersion( collection=collection, **collection_info, requires_ansible=importer_result.get("requires_ansible"), contents=importer_result["contents"], docs_blob=importer_result["docs_blob"], ) serializer_fields = CollectionVersionSerializer.Meta.fields data = { k: v for k, v in collection_version.__dict__.items() if k in serializer_fields } data["id"] = collection_version.pulp_id if not metadata_only: data["artifact"] = importer_result["artifact_url"] serializer = CollectionVersionSerializer(data=data) serializer.is_valid(raise_exception=True) collection_version.save() for name in tags: tag, created = Tag.objects.get_or_create(name=name) collection_version.tags.add(tag) _update_highest_version(collection_version) collection_version.save() # Save the FK updates return collection_version
async def run(self): """ Build and emit `DeclarativeContent` from the ansible metadata. """ msg = "Parsing CollectionVersion Metadata" with ProgressReport(message=msg, code="parsing.metadata") as pb: async for metadata in self._fetch_collections(): url = metadata["download_url"] collection_version = CollectionVersion( namespace=metadata["namespace"]["name"], name=metadata["collection"]["name"], version=metadata["version"], ) info = metadata["metadata"] info.pop("tags") for attr_name, attr_value in info.items(): if attr_value is None or attr_name not in collection_version.__dict__: continue setattr(collection_version, attr_name, attr_value) artifact = metadata["artifact"] d_artifact = DeclarativeArtifact( artifact=Artifact(sha256=artifact["sha256"], size=artifact["size"]), url=url, relative_path=collection_version.relative_path, remote=self.remote, deferred_download=self.deferred_download, ) extradata = dict( docs_blob_url=metadata["docs_blob_url"], deprecated=metadata["deprecated"], ) d_content = DeclarativeContent( content=collection_version, d_artifacts=[d_artifact], extra_data=extradata, ) pb.increment() await self.put(d_content)
def create_collection_from_importer(importer_result): """ Process results from importer. """ collection_info = importer_result["metadata"] with transaction.atomic(): collection, created = Collection.objects.get_or_create( namespace=collection_info["namespace"], name=collection_info["name"]) tags = collection_info.pop("tags") # Remove fields not used by this model collection_info.pop("license_file") collection_info.pop("readme") # the importer returns many None values. We need to let the defaults in the model prevail for key in [ "description", "documentation", "homepage", "issues", "repository" ]: if collection_info[key] is None: collection_info.pop(key) collection_version = CollectionVersion( collection=collection, **collection_info, contents=importer_result["contents"], docs_blob=importer_result["docs_blob"], ) collection_version.save() for name in tags: tag, created = Tag.objects.get_or_create(name=name) collection_version.tags.add(tag) _update_highest_version(collection_version) collection_version.save() # Save the FK updates return collection_version
async def _add_collection_version(self, api_version, collection_version_url, metadata): """Add CollectionVersion to the sync pipeline.""" url = metadata["download_url"] collection_version = CollectionVersion( namespace=metadata["namespace"]["name"], name=metadata["collection"]["name"], version=metadata["version"], ) info = metadata["metadata"] info.pop("tags") for attr_name, attr_value in info.items(): if attr_value is None or attr_name not in collection_version.__dict__: continue setattr(collection_version, attr_name, attr_value) artifact = metadata["artifact"] d_artifact = DeclarativeArtifact( artifact=Artifact(sha256=artifact["sha256"], size=artifact["size"]), url=url, relative_path=collection_version.relative_path, remote=self.remote, deferred_download=self.deferred_download, ) extra_data = {} if api_version != 2: # V2 never implemented the docs-blob requests extra_data["docs_blob_url"] = f"{collection_version_url}docs-blob/" d_content = DeclarativeContent( content=collection_version, d_artifacts=[d_artifact], extra_data=extra_data, ) self.parsing_metadata_progress_bar.increment() await self.put(d_content)
async def _add_collection_version(self, api_version, collection_version_url, metadata): """Add CollectionVersion to the sync pipeline.""" url = metadata["download_url"] collection_version = CollectionVersion( namespace=metadata["namespace"]["name"], name=metadata["collection"]["name"], version=metadata["version"], ) cv_unique = attrgetter("namespace", "name", "version")(collection_version) if cv_unique in self.already_synced: return self.already_synced.add(cv_unique) info = metadata["metadata"] if self.add_dependents: dependencies = info["dependencies"] tasks = [] loop = asyncio.get_event_loop() for full_name, version in dependencies.items(): namespace, name = full_name.split(".") if not (namespace, name, version) in self.already_synced: new_req = RequirementsFileEntry( name=full_name, version=version, source=None, ) tasks.append( loop.create_task( self._fetch_collection_metadata(new_req))) await asyncio.gather(*tasks) info.pop("tags") for attr_name, attr_value in info.items(): if attr_value is None or attr_name not in collection_version.__dict__: continue setattr(collection_version, attr_name, attr_value) artifact = metadata["artifact"] d_artifact = DeclarativeArtifact( artifact=Artifact(sha256=artifact["sha256"], size=artifact["size"]), url=url, relative_path=collection_version.relative_path, remote=self.remote, deferred_download=self.deferred_download, ) extra_data = {} if api_version != 2: # V2 never implemented the docs-blob requests extra_data["docs_blob_url"] = f"{collection_version_url}docs-blob/" d_content = DeclarativeContent( content=collection_version, d_artifacts=[d_artifact], extra_data=extra_data, ) self.parsing_metadata_progress_bar.increment() await self.put(d_content)
async def _add_collection_version(self, api_version, collection_version_url, metadata): """Add CollectionVersion to the sync pipeline.""" url = metadata["download_url"] collection_version = CollectionVersion( namespace=metadata["namespace"]["name"], name=metadata["collection"]["name"], version=metadata["version"], ) cv_unique = attrgetter("namespace", "name", "version")(collection_version) fullname, version = f"{cv_unique[0]}.{cv_unique[1]}", cv_unique[2] if fullname in self.exclude_info and version in self.exclude_info[ fullname]: return if cv_unique in self.already_synced: return self.already_synced.add(cv_unique) info = metadata["metadata"] signatures = metadata.get("signatures") if self.signed_only and not signatures: return if self.add_dependents: dependencies = info["dependencies"] tasks = [] loop = asyncio.get_event_loop() for full_name, version in dependencies.items(): namespace, name = full_name.split(".") req = (namespace, name, version) new_req = RequirementsFileEntry(full_name, version=version, source=None) if not any([ req in self.already_synced, new_req in self.collection_info ]): self.collection_info.append(new_req) tasks.append( loop.create_task( self._fetch_collection_metadata(new_req))) await asyncio.gather(*tasks) info.pop("tags") for attr_name, attr_value in info.items(): if attr_value is None or attr_name not in collection_version.__dict__: continue setattr(collection_version, attr_name, attr_value) artifact = metadata["artifact"] d_artifact = DeclarativeArtifact( artifact=Artifact(sha256=artifact["sha256"], size=artifact["size"]), url=url, relative_path=collection_version.relative_path, remote=self.remote, deferred_download=self.deferred_download, ) extra_data = {} if api_version != 2: # V2 never implemented the docs-blob requests extra_data["docs_blob_url"] = f"{collection_version_url}docs-blob/" d_content = DeclarativeContent( content=collection_version, d_artifacts=[d_artifact], extra_data=extra_data, ) await self.parsing_metadata_progress_bar.aincrement() await self.put(d_content) if signatures: collection_version = await d_content.resolution() for signature in signatures: sig = signature["signature"].encode("utf8") cv_signature = CollectionVersionSignature( signed_collection=collection_version, data=sig, digest=hashlib.sha256(sig).hexdigest(), pubkey_fingerprint=signature["pubkey_fingerprint"], ) await self.put(DeclarativeContent(content=cv_signature))
def import_collection( artifact_pk, repository_pk=None, expected_namespace=None, expected_name=None, expected_version=None, ): """ Create a Collection from an uploaded artifact and optionally validate its expected metadata. This task provides optional validation of the `namespace`, `name`, and `version` metadata attributes. If the Artifact fails validation or parsing, the Artifact is deleted and the Collection is not created. This task performs a CollectionImport object get_or_create() to allow import messages to be logged. Args: artifact_pk (str): The pk of the Artifact to create the Collection from. Keyword Args: repository_pk (str): Optional. If specified, a new RepositoryVersion will be created for the Repository and any new Collection content associated with it. expected_namespace (str): Optional. The namespace is validated against the namespace specified in the Collection's metadata. If it does not match a ImporterError is raised. expected_name (str): Optional. The name is validated against the name specified in the Collection's metadata. If it does not match a ImporterError is raised. expected_version (str): Optional. The version is validated against the version specified in the Collection's metadata. If it does not match a ImporterError is raised. Raises: ImporterError: If the `expected_namespace`, `expected_name`, or `expected_version` do not match the metadata in the tarball. """ CollectionImport.objects.get_or_create(task_id=get_current_job().id) artifact = Artifact.objects.get(pk=artifact_pk) filename = CollectionFilename(expected_namespace, expected_name, expected_version) log.info(f"Processing collection from {artifact.file.name}") import_logger = logging.getLogger("pulp_ansible.app.tasks.collection.import_collection") with _artifact_guard(artifact): try: with artifact.file.open() as artifact_file: importer_result = process_collection( artifact_file, filename=filename, logger=import_logger ) except ImporterError as exc: log.info(f"Collection processing was not successfull: {exc}") raise collection_info = importer_result["metadata"] with transaction.atomic(): collection, created = Collection.objects.get_or_create( namespace=collection_info["namespace"], name=collection_info["name"] ) tags = collection_info.pop("tags") # Remove fields not used by this model collection_info.pop("license_file") collection_info.pop("readme") # the importer returns many None values. We need to let the defaults in the model prevail for key in ["description", "documentation", "homepage", "issues", "repository"]: if collection_info[key] is None: collection_info.pop(key) collection_version = CollectionVersion( collection=collection, **collection_info, contents=importer_result["contents"], docs_blob=importer_result["docs_blob"], ) collection_version.save() for name in tags: tag, created = Tag.objects.get_or_create(name=name) collection_version.tags.add(tag) _update_highest_version(collection_version) collection_version.save() # Save the FK updates ContentArtifact.objects.create( artifact=artifact, content=collection_version, relative_path=collection_version.relative_path, ) CreatedResource.objects.create(content_object=collection_version) if repository_pk: repository = Repository.objects.get(pk=repository_pk) content_q = CollectionVersion.objects.filter(pk=collection_version.pk) with RepositoryVersion.create(repository) as new_version: new_version.add_content(content_q) CreatedResource.objects.create(content_object=repository)