Esempio n. 1
0
 def _save_manifest(self,
                    artifact,
                    manifest_digest,
                    content_type,
                    config_blob=None):
     manifest = models.Manifest(
         digest=manifest_digest,
         schema_version=2,
         media_type=content_type,
         config_blob=config_blob,
     )
     try:
         manifest.save()
     except IntegrityError:
         manifest = models.Manifest.objects.get(digest=manifest.digest)
         manifest.touch()
     ca = ContentArtifact(artifact=artifact,
                          content=manifest,
                          relative_path=manifest.digest)
     try:
         ca.save()
     except IntegrityError:
         ca = ContentArtifact.objects.get(content=manifest,
                                          relative_path=manifest.digest)
         if not ca.artifact:
             ca.artifact = artifact
             ca.save(update_fields=["artifact"])
     return manifest
Esempio n. 2
0
    def save(self):
        """
        Update the DB:
         - Create (or fetch) the Artifact.
         - Create (or fetch) the ContentArtifact.
         - Create (or update) the RemoteArtifact.
        """
        if self._stored_model:
            try:
                with transaction.atomic():
                    self._stored_model.save()
            except IntegrityError:
                q = self.artifact_q()
                self._stored_model = Artifact.objects.get(q)

        try:
            with transaction.atomic():
                content_artifact = ContentArtifact(
                    relative_path=self.relative_path,
                    content=self.content.stored_model,
                    artifact=self._stored_model)
                content_artifact.save()
        except IntegrityError:
            content_artifact = ContentArtifact.objects.get(
                relative_path=self.relative_path,
                content=self.content.stored_model)
            if self._stored_model:
                content_artifact.artifact = self._stored_model
                content_artifact.save()

        digests = {f: getattr(self._model, f) for f in Artifact.DIGEST_FIELDS}

        try:
            with transaction.atomic():
                remote_artifact = RemoteArtifact(
                    url=self.url,
                    remote=self.remote,
                    content_artifact=content_artifact,
                    size=self._model.size,
                    **digests)
                remote_artifact.save()
        except IntegrityError:
            q_set = RemoteArtifact.objects.filter(
                remote=self.remote,
                content_artifact=content_artifact)
            q_set.update(
                url=self.url,
                size=self._model.size,
                **digests)
Esempio n. 3
0
            def process_batch():
                content_artifact_bulk = []
                to_update_ca_query = ContentArtifact.objects.none()
                to_update_ca_bulk = []
                to_update_ca_artifact = {}
                with transaction.atomic():
                    self._pre_save(batch)
                    # Process the batch in dc.content.natural_keys order.
                    # This prevents deadlocks when we're processing the same/similar content
                    # in concurrent workers.
                    batch.sort(key=lambda x: "".join(
                        map(str, x.content.natural_key())))
                    for d_content in batch:
                        # Are we saving to the database for the first time?
                        content_already_saved = not d_content.content._state.adding
                        if not content_already_saved:
                            try:
                                with transaction.atomic():
                                    d_content.content.save()
                            except IntegrityError as e:
                                try:
                                    d_content.content = d_content.content.__class__.objects.get(
                                        d_content.content.q())
                                except ObjectDoesNotExist:
                                    raise e
                            else:
                                for d_artifact in d_content.d_artifacts:
                                    if not d_artifact.artifact._state.adding:
                                        artifact = d_artifact.artifact
                                    else:
                                        # set to None for on-demand synced artifacts
                                        artifact = None
                                    content_artifact = ContentArtifact(
                                        content=d_content.content,
                                        artifact=artifact,
                                        relative_path=d_artifact.relative_path,
                                    )
                                    content_artifact_bulk.append(
                                        content_artifact)
                                continue
                        # When the Content already exists, check if ContentArtifacts need to be
                        # updated
                        for d_artifact in d_content.d_artifacts:
                            if not d_artifact.artifact._state.adding:
                                # the artifact is already present in the database; update references
                                # Creating one large query and one large dictionary
                                to_update_ca_query |= ContentArtifact.objects.filter(
                                    content=d_content.content,
                                    relative_path=d_artifact.relative_path,
                                )
                                key = (d_content.content.pk,
                                       d_artifact.relative_path)
                                to_update_ca_artifact[
                                    key] = d_artifact.artifact
                    # Query db once and update each object in memory for bulk_update call
                    for content_artifact in to_update_ca_query.iterator():
                        key = (content_artifact.content_id,
                               content_artifact.relative_path)
                        # Maybe remove dict elements after to reduce memory?
                        content_artifact.artifact = to_update_ca_artifact[key]
                        to_update_ca_bulk.append(content_artifact)

                    # Sort the lists we're about to do bulk updates/creates on.
                    # We know to_update_ca_bulk entries already are in the DB, so we can enforce
                    # order just using pulp_id.
                    to_update_ca_bulk.sort(key=lambda x: x.pulp_id)
                    content_artifact_bulk.sort(
                        key=lambda x: ContentArtifact.sort_key(x))

                    ContentArtifact.objects.bulk_update(
                        to_update_ca_bulk, ["artifact"])
                    ContentArtifact.objects.bulk_get_or_create(
                        content_artifact_bulk)
                    self._post_save(batch)