def _associate_manifest(tag, oci_manifest): with db_transaction(): tag_manifest = TagManifest.create( tag=tag, digest=oci_manifest.digest, json_data=oci_manifest.manifest_bytes) TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=oci_manifest) return tag_manifest
def _backfill_manifest(tag_manifest): logger.info('Backfilling manifest for tag manifest %s', tag_manifest.id) # Ensure that a mapping row doesn't already exist. If it does, we've been preempted. if lookup_manifest_map_row(tag_manifest): return False # Parse the manifest. If we cannot parse, then we treat the manifest as broken and just emit it # without additional rows or data, as it will eventually not be useful. is_broken = False try: manifest = DockerSchema1Manifest(Bytes.for_string_or_unicode( tag_manifest.json_data), validate=False) except ManifestException: logger.exception('Exception when trying to parse manifest %s', tag_manifest.id) manifest = BrokenManifest(tag_manifest.digest, tag_manifest.json_data) is_broken = True # Lookup the storages for the digests. root_image = tag_manifest.tag.image repository = tag_manifest.tag.repository image_storage_id_map = { root_image.storage.content_checksum: root_image.storage.id } try: parent_images = get_parent_images(repository.namespace_user.username, repository.name, root_image) except DataModelException: logger.exception( 'Exception when trying to load parent images for manifest `%s`', tag_manifest.id) parent_images = {} is_broken = True for parent_image in parent_images: image_storage_id_map[ parent_image.storage.content_checksum] = parent_image.storage.id # Ensure that all the expected blobs have been found. If not, we lookup the blob under the repo # and add its storage ID. If the blob is not found, we mark the manifest as broken. storage_ids = set() try: for blob_digest in manifest.get_blob_digests_for_translation(): if blob_digest in image_storage_id_map: storage_ids.add(image_storage_id_map[blob_digest]) else: logger.debug( 'Blob `%s` not found in images for manifest `%s`; checking repo', blob_digest, tag_manifest.id) try: blob_storage = get_repo_blob_by_digest( repository.namespace_user.username, repository.name, blob_digest) storage_ids.add(blob_storage.id) except BlobDoesNotExist: logger.debug( 'Blob `%s` not found in repo for manifest `%s`', blob_digest, tag_manifest.id) is_broken = True except MalformedSchema1Manifest: logger.warning( 'Found malformed schema 1 manifest during blob backfill') is_broken = True with db_transaction(): # Re-retrieve the tag manifest to ensure it still exists and we're pointing at the correct tag. try: tag_manifest = TagManifest.get(id=tag_manifest.id) except TagManifest.DoesNotExist: return True # Ensure it wasn't already created. if lookup_manifest_map_row(tag_manifest): return False # Check for a pre-existing manifest matching the digest in the repository. This can happen # if we've already created the manifest row (typically for tag reverision). try: manifest_row = Manifest.get(digest=manifest.digest, repository=tag_manifest.tag.repository) except Manifest.DoesNotExist: # Create the new-style rows for the manifest. try: manifest_row = populate_manifest(tag_manifest.tag.repository, manifest, tag_manifest.tag.image, storage_ids) except IntegrityError: # Pre-empted. return False # Create the mapping row. If we find another was created for this tag manifest in the # meantime, then we've been preempted. try: TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest_row, broken=is_broken) except IntegrityError: return False # Backfill any labels on the manifest. _backfill_labels(tag_manifest, manifest_row, repository) return True
def retarget_tag(tag_name, manifest_id, is_reversion=False, now_ms=None, adjust_old_model=True): """ Creates or updates a tag with the specified name to point to the given manifest under its repository. If this action is a reversion to a previous manifest, is_reversion should be set to True. Returns the newly created tag row or None on error. """ try: manifest = (Manifest.select( Manifest, MediaType).join(MediaType).where(Manifest.id == manifest_id).get()) except Manifest.DoesNotExist: return None # CHECK: Make sure that we are not mistargeting a schema 1 manifest to a tag with a different # name. if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES: try: parsed = DockerSchema1Manifest(Bytes.for_string_or_unicode( manifest.manifest_bytes), validate=False) if parsed.tag != tag_name: logger.error( "Tried to re-target schema1 manifest with tag `%s` to tag `%s", parsed.tag, tag_name, ) return None except MalformedSchema1Manifest: logger.exception("Could not parse schema1 manifest") return None legacy_image = get_legacy_image_for_manifest(manifest) now_ms = now_ms or get_epoch_timestamp_ms() now_ts = int(now_ms / 1000) with db_transaction(): # Lookup an existing tag in the repository with the same name and, if present, mark it # as expired. existing_tag = get_tag(manifest.repository_id, tag_name) if existing_tag is not None: _, okay = set_tag_end_ms(existing_tag, now_ms) # TODO: should we retry here and/or use a for-update? if not okay: return None # Create a new tag pointing to the manifest with a lifetime start of now. created = Tag.create( name=tag_name, repository=manifest.repository_id, lifetime_start_ms=now_ms, reversion=is_reversion, manifest=manifest, tag_kind=Tag.tag_kind.get_id("tag"), ) # TODO: Remove the linkage code once RepositoryTag is gone. # If this is a schema 1 manifest, then add a TagManifest linkage to it. Otherwise, it will only # be pullable via the new OCI model. if adjust_old_model: if (manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES and legacy_image is not None): old_style_tag = RepositoryTag.create( repository=manifest.repository_id, image=legacy_image, name=tag_name, lifetime_start_ts=now_ts, reversion=is_reversion, ) TagToRepositoryTag.create(tag=created, repository_tag=old_style_tag, repository=manifest.repository_id) tag_manifest = TagManifest.create( tag=old_style_tag, digest=manifest.digest, json_data=manifest.manifest_bytes) TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest, repository=manifest.repository_id) return created