async def create_artifact(self, pulp2_storage_path, expected_digests={}, expected_size=None, downloaded=True): """ Create a hard link if possible and then create an Artifact. If it's not possible to create a hard link, file is copied to the Pulp 3 storage. For non-downloaded content, artifact with its expected checksum and size is created. """ if not downloaded: if not expected_digests: raise ValueError( _('No digest is provided for on_demand content creation. Pulp 2 ' 'storage path: {}'.format(pulp2_storage_path))) artifact = Artifact(**expected_digests) artifact.size = expected_size return artifact artifact = Artifact.init_and_validate( pulp2_storage_path, expected_digests=expected_digests, expected_size=expected_size) pulp3_storage_relative_path = storage.get_artifact_path( artifact.sha256) pulp3_storage_path = os.path.join(settings.MEDIA_ROOT, pulp3_storage_relative_path) os.makedirs(os.path.dirname(pulp3_storage_path), exist_ok=True) is_copied = False try: os.link(pulp2_storage_path, pulp3_storage_path) except FileExistsError: pass except OSError: _logger.debug( _('Hard link cannot be created, file will be copied.')) shutil.copy2(pulp2_storage_path, pulp3_storage_path) is_copied = True if not expected_digests: expected_digests = {'sha256': artifact.sha256} if is_copied: # recalculate checksums to ensure that after being copied a file is still fine artifact = Artifact.init_and_validate( file=pulp3_storage_path, expected_digests=expected_digests, expected_size=expected_size) else: # a hard link has been created or a file has already been in the pulp 3 storage, so # artifact's path can be just updated and no checksum recalculation is needed. artifact.file = pulp3_storage_path return artifact
def setUp(self): with open(self.artifact01_path, 'w') as f: f.write('Temp Artifact File 01') with open(self.artifact02_path, 'w') as f: f.write('Temp Artifact File 02') self.artifact01 = Artifact.init_and_validate(self.artifact01_path) self.artifact01.save() self.artifact02 = Artifact.init_and_validate(self.artifact02_path) self.artifact02.save()
def setUp(self): with open(self.artifact01_path, "w") as f: f.write("Temp Artifact File 01") with open(self.artifact02_path, "w") as f: f.write("Temp Artifact File 02") self.artifact01 = Artifact.init_and_validate(self.artifact01_path) self.artifact01.save() self.artifact02 = Artifact.init_and_validate(self.artifact02_path) self.artifact02.save()
def get_or_create_blob(layer_json, manifest, path): """ Creates Blob from json snippet of manifest.json Args: layer_json (json): json manifest (class:`pulp_container.app.models.Manifest`): The manifest path (str): Path of the directory that contains layer Returns: class:`pulp_container.app.models.Blob` """ try: blob = Blob.objects.get(digest=layer_json["digest"]) except Blob.DoesNotExist: layer_file_name = "{}{}".format(path, layer_json["digest"][7:]) layer_artifact = Artifact.init_and_validate(layer_file_name) layer_artifact.save() blob = Blob(digest=layer_json["digest"], media_type=layer_json["mediaType"]) blob.save() ContentArtifact(artifact=layer_artifact, content=blob, relative_path=layer_json["digest"]).save() if blob.media_type != MEDIA_TYPE.CONFIG_BLOB_OCI: BlobManifest(manifest=manifest, manifest_blob=blob).save() return blob
def create(self, request): """ Dispatch a Collection creation task. """ serializer = CollectionOneShotSerializer(data=request.data, context={'request': request}) serializer.is_valid(raise_exception=True) expected_digests = {} if serializer.validated_data['sha256']: expected_digests['sha256'] = serializer.validated_data['sha256'] try: artifact = Artifact.init_and_validate(serializer.validated_data['file'], expected_digests=expected_digests) except DigestValidationError: raise serializers.ValidationError( _("The provided sha256 value does not match the sha256 of the uploaded file.") ) artifact.save() async_result = enqueue_with_reservation( import_collection, [str(artifact.pk)], kwargs={ 'artifact_pk': artifact.pk, } ) return OperationPostponedResponse(async_result, request)
def post(self, request): """Upload an RPM package.""" serializer = OneShotUploadSerializer( data=request.data, context={'request': request}) serializer.is_valid(raise_exception=True) artifact = Artifact.init_and_validate(request.data['file']) if 'repository' in request.data: repository = serializer.validated_data['repository'] else: repository = None try: artifact.save() except IntegrityError: # if artifact already exists, let's use it artifact = Artifact.objects.get(sha256=artifact.sha256) async_result = enqueue_with_reservation( one_shot_upload, [artifact], kwargs={ 'artifact': artifact, 'repository': repository, }) return OperationPostponedResponse(async_result, request)
def create(self, request): """Upload an RPM package.""" artifact = Artifact.init_and_validate(request.data['file']) filename = request.data['file'].name if 'repository' in request.data: serializer = OneShotUploadSerializer(data=request.data, context={'request': request}) serializer.is_valid(raise_exception=True) repository = serializer.validated_data['repository'] repository_pk = repository.pk else: repository_pk = None try: artifact.save() except IntegrityError: # if artifact already exists, let's use it artifact = Artifact.objects.get(sha256=artifact.sha256) async_result = enqueue_with_reservation(tasks.one_shot_upload, [artifact], kwargs={ 'artifact_pk': artifact.pk, 'filename': filename, 'repository_pk': repository_pk, }) return OperationPostponedResponse(async_result, request)
def validate(self, data): """Validates the request.""" action = data.get(":action") if action != "file_upload": raise serializers.ValidationError( _("We do not support the :action {}").format(action)) file = data.get("content") for ext, packagetype in DIST_EXTENSIONS.items(): if file.name.endswith(ext): break else: raise serializers.ValidationError( _("Extension on {} is not a valid python extension " "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format( file.name)) sha256 = data.get("sha256_digest") digests = {"sha256": sha256} if sha256 else None artifact = Artifact.init_and_validate(file, expected_digests=digests) try: artifact.save() except IntegrityError: artifact = Artifact.objects.get(sha256=artifact.sha256) artifact.touch() log.info(f"Artifact for {file.name} already existed in database") data["content"] = (artifact, file.name) return data
def validate(self, data): """Validates that all the fields make sense.""" data = super().validate(data) if "containerfile" in data: if "containerfile_artifact" in data: raise serializers.ValidationError( _("Only one of 'containerfile' and 'containerfile_artifact' may be specified.") ) data["containerfile_artifact"] = Artifact.init_and_validate(data.pop("containerfile")) elif "containerfile_artifact" not in data: raise serializers.ValidationError(_("'containerfile' or 'containerfile_artifact' must " "be specified.")) artifacts = {} if 'artifacts' in data: for url, relative_path in data['artifacts'].items(): if os.path.isabs(relative_path): raise serializers.ValidationError(_("Relative path cannot start with '/'. " "{0}").format(relative_path)) artifactfield = RelatedField(view_name='artifacts-detail', queryset=Artifact.objects.all(), source='*', initial=url) try: artifact = artifactfield.run_validation(data=url) artifacts[artifact.pk] = relative_path except serializers.ValidationError as e: # Append the URL of missing Artifact to the error message e.detail[0] = "%s %s" % (e.detail[0], url) raise e data['artifacts'] = artifacts return data
def create(self, request): """ Dispatch a Collection creation task. """ serializer = CollectionOneShotSerializer(data=request.data, context={"request": request}) serializer.is_valid(raise_exception=True) expected_digests = {} if serializer.validated_data["sha256"]: expected_digests["sha256"] = serializer.validated_data["sha256"] try: artifact = Artifact.init_and_validate( serializer.validated_data["file"], expected_digests=expected_digests) except DigestValidationError: raise serializers.ValidationError( _("The provided sha256 value does not match the sha256 of the uploaded file." )) try: artifact.save() except IntegrityError: raise serializers.ValidationError(_("Artifact already exists.")) async_result = self._dispatch_import_collection_task(artifact.pk) return OperationPostponedResponse(async_result, request)
async def create_artifact(self, pulp2_storage_path, expected_digests={}, expected_size=None): """ Create a hard link if possible and then create an Artifact. If it's not possible to create a hard link, file is copied to the Pulp 3 storage. """ if not expected_digests.get('sha256'): # TODO: all checksums are calculated for the pulp 2 storage path, is it ok? artifact = Artifact.init_and_validate(pulp2_storage_path, size=expected_size) sha256digest = expected_digests.get('sha256') or artifact.sha256 pulp3_storage_relative_path = storage.get_artifact_path(sha256digest) pulp3_storage_path = os.path.join(settings.MEDIA_ROOT, pulp3_storage_relative_path) os.makedirs(os.path.dirname(pulp3_storage_path), exist_ok=True) is_copied = False try: os.link(pulp2_storage_path, pulp3_storage_path) except FileExistsError: pass except OSError: _logger.debug('Hard link cannot be created, file will be copied.') shutil.copy2(pulp2_storage_path, pulp3_storage_path) is_copied = True expected_digests = {'sha256': sha256digest} if is_copied: # recalculate checksums to ensure that after being copied a file is still fine artifact = Artifact.init_and_validate( file=pulp3_storage_path, expected_digests=expected_digests, expected_size=expected_size) else: # a hard link has been created or a file has already been in the pulp 3 storage, so # artifact's path can be just updated and no checksum recalculation is needed. artifact.file = pulp3_storage_path return artifact
def create(self, request, path): """ Dispatch a Collection creation task. """ distro = get_object_or_404(AnsibleDistribution, base_path=path) serializer = CollectionOneShotSerializer(data=request.data, context={"request": request}) serializer.is_valid(raise_exception=True) expected_digests = {} if serializer.validated_data["sha256"]: expected_digests["sha256"] = serializer.validated_data["sha256"] try: artifact = Artifact.init_and_validate( serializer.validated_data["file"], expected_digests=expected_digests) except DigestValidationError: raise serializers.ValidationError( _("The provided sha256 value does not match the sha256 of the uploaded file." )) try: artifact.save() except IntegrityError: raise serializers.ValidationError(_("Artifact already exists.")) kwargs = {} if serializer.validated_data["expected_namespace"]: kwargs["expected_namespace"] = serializer.validated_data[ "expected_namespace"] if serializer.validated_data["expected_name"]: kwargs["expected_name"] = serializer.validated_data[ "expected_name"] if serializer.validated_data["expected_version"]: kwargs["expected_version"] = serializer.validated_data[ "expected_version"] async_result = self._dispatch_import_collection_task( artifact.pk, distro.repository, **kwargs) CollectionImport.objects.create(task_id=async_result.id) data = { "task": reverse( "collection-imports-detail", kwargs={ "path": path, "pk": async_result.id }, request=None, ) } return Response(data, status=http_status.HTTP_202_ACCEPTED)
def validate(self, data): """Validate the GemContent data.""" data = super().validate(data) if "file" in data: if "artifact" in data: raise ValidationError(_("Only one of 'file' and 'artifact' may be specified.")) data["artifact"] = Artifact.init_and_validate(data.pop("file")) elif "artifact" not in data: raise ValidationError(_("One of 'file' and 'artifact' must be specified.")) if "request" not in self.context: data = self.deferred_validate(data) return data
def post(self, request, path): """ Queues a task that creates a new Collection from an uploaded artifact. """ distro = get_object_or_404(AnsibleDistribution, base_path=path) serializer = GalaxyCollectionUploadSerializer( data=request.data, context={"request": request}) serializer.is_valid(raise_exception=True) artifact = Artifact.init_and_validate( serializer.validated_data["file"]) artifact.save() async_result = self._dispatch_import_collection_task( artifact.pk, distro.repository) return OperationPostponedResponse(async_result, request)
def _create_snippet(snippet_string): """ Create snippet of modulemd[-defaults] as artifact. Args: snippet_string (string): Snippet with modulemd[-defaults] yaml Returns: Snippet as unsaved Artifact object """ tmp_file = tempfile.NamedTemporaryFile(dir=os.getcwd(), delete=False) with open(tmp_file.name, "w") as snippet: snippet.write(snippet_string) return Artifact.init_and_validate(tmp_file.name)
def put(self, request, path, pk=None): """ Create a blob from uploaded chunks. """ _, repository = self.get_dr_push(request, path) digest = request.query_params["digest"] upload = models.Upload.objects.get(pk=pk, repository=repository) chunks = UploadChunk.objects.filter(upload=upload).order_by("offset") with NamedTemporaryFile("ab") as temp_file: for chunk in chunks: temp_file.write(chunk.file.read()) temp_file.flush() uploaded_file = PulpTemporaryUploadedFile.from_file( File(open(temp_file.name, "rb"))) if uploaded_file.hashers["sha256"].hexdigest() == digest[len("sha256:" ):]: try: artifact = Artifact.init_and_validate(uploaded_file) artifact.save() except IntegrityError: artifact = Artifact.objects.get(sha256=artifact.sha256) try: blob = models.Blob(digest=digest, media_type=models.MEDIA_TYPE.REGULAR_BLOB) blob.save() except IntegrityError: blob = models.Blob.objects.get(digest=digest) try: blob_artifact = ContentArtifact(artifact=artifact, content=blob, relative_path=digest) blob_artifact.save() except IntegrityError: pass with repository.new_version() as new_version: new_version.add_content(models.Blob.objects.filter(pk=blob.pk)) upload.delete() return BlobResponse(blob, path, 201, request) else: raise Exception("The digest did not match")
def setUp(self): with open(self.artifact_path, 'w') as f: f.write('Temp Artifact File') self.artifact = Artifact.init_and_validate(self.artifact_path) self.artifact.save() collection = Collection.objects.create(namespace='my_ns', name='my_name') self.collection_version = CollectionVersion.objects.create( collection=collection) self.collection_version.save() content_artifact = ContentArtifact.objects.create( artifact=self.artifact, content=self.collection_version, ) content_artifact.save()
def add_image_from_directory_to_repository(path, repository, tag): """ Creates a Manifest and all blobs from a directory with OCI image Args: path (str): Path to directory with the OCI image repository (class:`pulpcore.plugin.models.Repository`): The destination repository tag (str): Tag name for the new image in the repository Returns: A class:`pulpcore.plugin.models.RepositoryVersion` that contains the new OCI container image and tag. """ manifest_path = "{}manifest.json".format(path) manifest_artifact = Artifact.init_and_validate(manifest_path) manifest_artifact.save() manifest_digest = "sha256:{}".format(manifest_artifact.sha256) manifest = Manifest(digest=manifest_digest, schema_version=2, media_type=MEDIA_TYPE.MANIFEST_OCI) manifest.save() ContentArtifact(artifact=manifest_artifact, content=manifest, relative_path=manifest_digest).save() tag = Tag(name=tag, tagged_manifest=manifest) tag.save() ContentArtifact(artifact=manifest_artifact, content=tag, relative_path=tag.name).save() with repository.new_version() as new_repo_version: new_repo_version.add_content(Manifest.objects.filter(pk=manifest.pk)) new_repo_version.add_content(Tag.objects.filter(pk=tag.pk)) with open(manifest_artifact.file.path, "r") as manifest_file: manifest_json = json.load(manifest_file) config_blob = get_or_create_blob(manifest_json["config"], manifest, path) manifest.config_blob = config_blob manifest.save() new_repo_version.add_content( Blob.objects.filter(pk=config_blob.pk)) for layer in manifest_json["layers"]: blob = get_or_create_blob(layer, manifest, path) new_repo_version.add_content(Blob.objects.filter(pk=blob.pk)) return new_repo_version
def import_collection_from_path(path): """ Import a single collection by path. This method will not fail if the Artifact already exists. Args: path: The path to the tarball to import. """ artifact = Artifact.init_and_validate(path) try: artifact.save() except IntegrityError: artifact = Artifact.objects.get(sha256=artifact.sha256) import_collection(artifact.pk)
async def run(self): """ Parse PackageIndex content units. Ensure, that an uncompressed artifact is available. """ with ProgressReport(message="Update PackageIndex units", code="update.packageindex") as pb: async for d_content in self.items(): if isinstance(d_content.content, PackageIndex): if not d_content.d_artifacts: d_content.content = None d_content.resolve() continue content = d_content.content if not [ da for da in d_content.d_artifacts if da.artifact.sha256 == content.sha256 ]: # No main_artifact found, uncompress one relative_dir = os.path.dirname( d_content.content.relative_path) filename = _uncompress_artifact( d_content.d_artifacts, relative_dir) da = DeclarativeArtifact( Artifact.init_and_validate( filename, expected_digests={"sha256": content.sha256}), filename, content.relative_path, d_content.d_artifacts[0].remote, ) d_content.d_artifacts.append(da) da.artifact.save() log.info( "*** Expected: {} *** Uncompressed: {} ***".format( content.sha256, da.artifact.sha256)) pb.increment() await self.put(d_content)
def init_content_data(self, serializer, request): """Initialize a temporary Artifact.""" shared_resources = [] task_payload = {k: v for k, v in request.data.items()} file_content = task_payload.pop("file", None) if file_content: # in the upload code path make sure, the artifact exists, and the 'file' # parameter is replaced by an Artifact; this Artifact will be afterwards # deleted because it serves as a temporary storage for file contents artifact = Artifact.init_and_validate(file_content) try: artifact.save() except IntegrityError: # if artifact already exists, let's use it artifact = Artifact.objects.get(sha256=artifact.sha256) task_payload["artifact"] = ArtifactSerializer( artifact, context={"request": request} ).data["pulp_href"] shared_resources.append(artifact) return ContentUploadData(shared_resources, task_payload)
def sync(remote_pk, repository_pk): """ Sync Collections with ``remote_pk``, and save a new RepositoryVersion for ``repository_pk``. Args: remote_pk (str): The remote PK. repository_pk (str): The repository PK. Raises: ValueError: If the remote does not specify a URL to sync or a ``whitelist`` of Collections to sync. """ remote = CollectionRemote.objects.get(pk=remote_pk) repository = Repository.objects.get(pk=repository_pk) if not remote.url: raise ValueError( _("A CollectionRemote must have a 'url' specified to synchronize.") ) if not remote.whitelist: raise ValueError( _("A CollectionRemote must have a 'whitelist' specified to synchronize." )) repository_spec_strings = remote.whitelist.split(' ') def nowhere(*args, **kwargs): pass collections_created_pks = [] with tempfile.TemporaryDirectory() as temp_ansible_path: galaxy_context = GalaxyContext( collections_path=temp_ansible_path, server={ 'url': remote.url, 'ignore_certs': False, }, ) install_repository_specs_loop( display_callback=nowhere, galaxy_context=galaxy_context, repository_spec_strings=repository_spec_strings, ) content_walk_generator = os.walk(temp_ansible_path) for dirpath, dirnames, filenames in content_walk_generator: if 'MANIFEST.json' in filenames: with open(dirpath + os.path.sep + 'MANIFEST.json') as manifest_file: manifest_data = json.load(manifest_file) info = manifest_data['collection_info'] filename = '{namespace}-{name}-{version}'.format( namespace=info['namespace'], name=info['name'], version=info['version'], ) tarfile_path = temp_ansible_path + os.path.sep + filename + '.tar.gz' with tarfile.open(name=tarfile_path, mode='w|gz') as newtar: newtar.add(dirpath, arcname=filename) with transaction.atomic(): collection, created = Collection.objects.get_or_create( namespace=info['namespace'], name=info['name'], version=info['version']) if created: artifact = Artifact.init_and_validate(newtar.name) artifact.save() ContentArtifact.objects.create( artifact=artifact, content=collection, relative_path=collection.relative_path, ) collections_created_pks.append(collection) if collections_created_pks: with RepositoryVersion.create(repository) as new_version: collections = Collection.objects.filter( pk__in=collections_created_pks) new_version.add_content(collections)
def put(self, request, path, pk=None): """ Create a blob from uploaded chunks. """ _, repository = self.get_dr_push(request, path) digest = request.query_params["digest"] # Try to see if the client came back after we told it to backoff with the ``Throttled`` # exception. In that case we answer based on the task state, or make it backoff again. # This mechanism seems to work with podman but not with docker. However we let the task run # anyway, since all clients will look with a HEAD request before attemting to upload a blob # again. try: upload = models.Upload.objects.get(pk=pk, repository=repository) except models.Upload.DoesNotExist as e_upload: # Upload has been deleted => task has started or even finished try: task = Task.objects.filter( name__endswith="add_and_remove", reserved_resources_record__resource=f"upload:{pk}", ).last() except Task.DoesNotExist: # No upload and no task for it => the upload probably never existed # return 404 raise e_upload if task.state == "completed": task.delete() blob = models.Blob.objects.get(digest=digest) return BlobResponse(blob, path, 201, request) elif task.state in ["waiting", "running"]: raise Throttled() else: error = task.error task.delete() raise Exception(str(error)) chunks = UploadChunk.objects.filter(upload=upload).order_by("offset") with NamedTemporaryFile("ab") as temp_file: for chunk in chunks: temp_file.write(chunk.file.read()) temp_file.flush() uploaded_file = PulpTemporaryUploadedFile.from_file(File(open(temp_file.name, "rb"))) if uploaded_file.hashers["sha256"].hexdigest() == digest[len("sha256:") :]: try: artifact = Artifact.init_and_validate(uploaded_file) artifact.save() except IntegrityError: artifact = Artifact.objects.get(sha256=artifact.sha256) try: blob = models.Blob(digest=digest, media_type=models.MEDIA_TYPE.REGULAR_BLOB) blob.save() except IntegrityError: blob = models.Blob.objects.get(digest=digest) try: blob_artifact = ContentArtifact( artifact=artifact, content=blob, relative_path=digest ) blob_artifact.save() except IntegrityError: pass upload.delete() dispatched_task = dispatch( add_and_remove, [f"upload:{pk}", repository], kwargs={ "repository_pk": str(repository.pk), "add_content_units": [str(blob.pk)], "remove_content_units": [], }, ) # Wait a small amount of time for dummy in range(3): time.sleep(1) task = Task.objects.get(pk=dispatched_task.pk) if task.state == "completed": task.delete() return BlobResponse(blob, path, 201, request) elif task.state in ["waiting", "running"]: continue else: error = task.error task.delete() raise Exception(str(error)) raise Throttled() else: raise Exception("The digest did not match")
async def create_artifact(self, pulp2_storage_path, expected_digests={}, expected_size=None, downloaded=True): """ Create a hard link if possible and then create an Artifact. If it's not possible to create a hard link, file is copied to the Pulp 3 storage. For non-downloaded content, artifact with its expected checksum and size is created. """ if not downloaded: if not expected_digests: raise ValueError( _('No digest is provided for on_demand content creation. Pulp 2 ' 'storage path: {}'.format(pulp2_storage_path))) artifact = Artifact(**expected_digests) artifact.size = expected_size return artifact try: artifact = Artifact.init_and_validate( pulp2_storage_path, expected_digests=expected_digests, expected_size=expected_size) except (DigestValidationError, FileNotFoundError, SizeValidationError): if self.skip_corrupted: _logger.warn( f'The content located in {pulp2_storage_path} is missing or ' f'corrupted. It was skipped during Pulp 2to3 migration.') return raise ArtifactValidationError( f'The content located in {pulp2_storage_path} is ' f'missing or corrupted. Repair it in pulp2 and re-run ' f'the migration. Alternatively, run migration with ' f'skip_corrupted=True.') pulp3_storage_relative_path = storage.get_artifact_path( artifact.sha256) pulp3_storage_path = os.path.join(settings.MEDIA_ROOT, pulp3_storage_relative_path) os.makedirs(os.path.dirname(pulp3_storage_path), exist_ok=True) is_copied = False try: os.link(pulp2_storage_path, pulp3_storage_path) except FileExistsError: pass except OSError: _logger.debug( _('Hard link cannot be created, file will be copied.')) shutil.copy2(pulp2_storage_path, pulp3_storage_path) is_copied = True if not expected_digests: expected_digests = {'sha256': artifact.sha256} if is_copied: # recalculate checksums to ensure that after being copied a file is still fine artifact = Artifact.init_and_validate( file=pulp3_storage_path, expected_digests=expected_digests, expected_size=expected_size) else: # a hard link has been created or a file has already been in the pulp 3 storage, so # artifact's path can be just updated and no checksum recalculation is needed. artifact.file = pulp3_storage_path return artifact
async def declarative_content_from_git_repo(remote, url, git_ref=None, metadata_only=False): """Returns a DeclarativeContent for the Collection in a Git repository.""" if git_ref: try: gitrepo = Repo.clone_from(url, uuid4(), depth=1, branch=git_ref) except GitCommandError: gitrepo = Repo.clone_from(url, uuid4()) gitrepo.git.checkout(git_ref) else: gitrepo = Repo.clone_from(url, uuid4(), depth=1) commit_sha = gitrepo.head.commit.hexsha metadata, artifact_path = sync_collection(gitrepo.working_dir, ".") if not metadata_only: artifact = Artifact.init_and_validate(artifact_path) try: await sync_to_async(artifact.save)() except IntegrityError: artifact = Artifact.objects.get(sha256=artifact.sha256) metadata["artifact_url"] = reverse("artifacts-detail", args=[artifact.pk]) metadata["artifact"] = artifact else: metadata["artifact"] = None metadata["artifact_url"] = None metadata["remote_artifact_url"] = "{}/commit/{}".format( url.rstrip("/"), commit_sha) artifact = metadata["artifact"] try: collection_version = await sync_to_async( create_collection_from_importer)(metadata, metadata_only=metadata_only) await sync_to_async(ContentArtifact.objects.get_or_create)( artifact=artifact, content=collection_version, relative_path=collection_version.relative_path, ) except ValidationError as e: if e.args[0]["non_field_errors"][0].code == "unique": namespace = metadata["metadata"]["namespace"] name = metadata["metadata"]["name"] version = metadata["metadata"]["version"] else: raise e collection_version = await sync_to_async(CollectionVersion.objects.get )(namespace=namespace, name=name, version=version) if artifact is None: artifact = Artifact() d_artifact = DeclarativeArtifact( artifact=artifact, url=metadata["remote_artifact_url"], relative_path=collection_version.relative_path, remote=remote, deferred_download=metadata_only, ) # TODO: docs blob support?? d_content = DeclarativeContent( content=collection_version, d_artifacts=[d_artifact], ) return d_content