def _save_manifest(self,
artifact,
manifest_digest,
content_type,
config_blob=None):
manifest = models.Manifest(
digest=manifest_digest,
schema_version=2,
media_type=content_type,
config_blob=config_blob,
)
try:
manifest.save()
except IntegrityError:
manifest = models.Manifest.objects.get(digest=manifest.digest)
manifest.touch()
ca = ContentArtifact(artifact=artifact,
content=manifest,
relative_path=manifest.digest)
try:
ca.save()
except IntegrityError:
ca = ContentArtifact.objects.get(content=manifest,
relative_path=manifest.digest)
if not ca.artifact:
ca.artifact = artifact
ca.save(update_fields=["artifact"])
return manifest
def save(self):
"""
Update the DB:
- Create (or fetch) the Artifact.
- Create (or fetch) the ContentArtifact.
- Create (or update) the RemoteArtifact.
"""
if self._stored_model:
try:
with transaction.atomic():
self._stored_model.save()
except IntegrityError:
q = self.artifact_q()
self._stored_model = Artifact.objects.get(q)
try:
with transaction.atomic():
content_artifact = ContentArtifact(
relative_path=self.relative_path,
content=self.content.stored_model,
artifact=self._stored_model)
content_artifact.save()
except IntegrityError:
content_artifact = ContentArtifact.objects.get(
relative_path=self.relative_path,
content=self.content.stored_model)
if self._stored_model:
content_artifact.artifact = self._stored_model
content_artifact.save()
digests = {f: getattr(self._model, f) for f in Artifact.DIGEST_FIELDS}
try:
with transaction.atomic():
remote_artifact = RemoteArtifact(
url=self.url,
remote=self.remote,
content_artifact=content_artifact,
size=self._model.size,
**digests)
remote_artifact.save()
except IntegrityError:
q_set = RemoteArtifact.objects.filter(
remote=self.remote,
content_artifact=content_artifact)
q_set.update(
url=self.url,
size=self._model.size,
**digests)
def process_batch():
content_artifact_bulk = []
to_update_ca_query = ContentArtifact.objects.none()
to_update_ca_bulk = []
to_update_ca_artifact = {}
with transaction.atomic():
self._pre_save(batch)
# Process the batch in dc.content.natural_keys order.
# This prevents deadlocks when we're processing the same/similar content
# in concurrent workers.
batch.sort(key=lambda x: "".join(
map(str, x.content.natural_key())))
for d_content in batch:
# Are we saving to the database for the first time?
content_already_saved = not d_content.content._state.adding
if not content_already_saved:
try:
with transaction.atomic():
d_content.content.save()
except IntegrityError as e:
try:
d_content.content = d_content.content.__class__.objects.get(
d_content.content.q())
except ObjectDoesNotExist:
raise e
else:
for d_artifact in d_content.d_artifacts:
if not d_artifact.artifact._state.adding:
artifact = d_artifact.artifact
else:
# set to None for on-demand synced artifacts
artifact = None
content_artifact = ContentArtifact(
content=d_content.content,
artifact=artifact,
relative_path=d_artifact.relative_path,
)
content_artifact_bulk.append(
content_artifact)
continue
# When the Content already exists, check if ContentArtifacts need to be
# updated
for d_artifact in d_content.d_artifacts:
if not d_artifact.artifact._state.adding:
# the artifact is already present in the database; update references
# Creating one large query and one large dictionary
to_update_ca_query |= ContentArtifact.objects.filter(
content=d_content.content,
relative_path=d_artifact.relative_path,
)
key = (d_content.content.pk,
d_artifact.relative_path)
to_update_ca_artifact[
key] = d_artifact.artifact
# Query db once and update each object in memory for bulk_update call
for content_artifact in to_update_ca_query.iterator():
key = (content_artifact.content_id,
content_artifact.relative_path)
# Maybe remove dict elements after to reduce memory?
content_artifact.artifact = to_update_ca_artifact[key]
to_update_ca_bulk.append(content_artifact)
# Sort the lists we're about to do bulk updates/creates on.
# We know to_update_ca_bulk entries already are in the DB, so we can enforce
# order just using pulp_id.
to_update_ca_bulk.sort(key=lambda x: x.pulp_id)
content_artifact_bulk.sort(
key=lambda x: ContentArtifact.sort_key(x))
ContentArtifact.objects.bulk_update(
to_update_ca_bulk, ["artifact"])
ContentArtifact.objects.bulk_get_or_create(
content_artifact_bulk)
self._post_save(batch)