def test_manifestbackfillworker_repeat_digest(clear_rows, initialized_db): """ Tests that a manifest with a shared digest will be properly linked. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository("devtable", "gargantuan") tag_v30 = model.tag.get_active_tag("devtable", "gargantuan", "v3.0") tag_v50 = model.tag.get_active_tag("devtable", "gargantuan", "v5.0") # Build a manifest and assign it to both tags (this is allowed in the old model). builder = DockerSchema1ManifestBuilder("devtable", "gargantuan", "sometag") builder.add_layer("sha256:deadbeef", '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) manifest_1 = TagManifest.create(json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v30) manifest_2 = TagManifest.create(json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50) # Backfill "both" manifests and ensure both are pointed to by a single resulting row. assert _backfill_manifest(manifest_1) assert _backfill_manifest(manifest_2) map_row1 = TagManifestToManifest.get(tag_manifest=manifest_1) map_row2 = TagManifestToManifest.get(tag_manifest=manifest_2) assert map_row1.manifest == map_row2.manifest
def test_manifestbackfillworker_mislinked_invalid_manifest(clear_rows, initialized_db): """ Tests that a manifest whose image is mislinked will attempt to have its storages relinked properly. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository("devtable", "complex") tag_v50 = model.tag.get_active_tag("devtable", "gargantuan", "v5.0") # Add a mislinked manifest, by having its layer point to an invalid blob but its image # be the v5.0 image. builder = DockerSchema1ManifestBuilder("devtable", "gargantuan", "sometag") builder.add_layer("sha256:deadbeef", '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) broken_manifest = TagManifest.create( json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50 ) # Backfill the manifest and ensure it is marked as broken. assert _backfill_manifest(broken_manifest) map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) assert map_row.broken manifest_row = map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert legacy_image == tag_v50.image manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) assert len(manifest_blobs) == 0
def test_manifest_backfill_broken_tag(clear_rows, initialized_db): """ Tests backfilling a broken tag. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() # Create a tag with an image referenced missing parent images. repo = model.repository.get_repository("devtable", "gargantuan") broken_image = Image.create( docker_image_id="foo", repository=repo, ancestors="/348723847234/", storage=ImageStorage.get(), ) broken_image_tag = RepositoryTag.create(repository=repo, image=broken_image, name="broken") # Backfill the tag. assert backfill_tag(broken_image_tag) # Ensure we backfilled, even though we reference a broken manifest. tag_manifest = TagManifest.get(tag=broken_image_tag) map_row = TagManifestToManifest.get(tag_manifest=tag_manifest) manifest = map_row.manifest assert manifest.manifest_bytes == tag_manifest.json_data tag = TagToRepositoryTag.get(repository_tag=broken_image_tag).tag assert tag.name == "broken" assert tag.manifest == manifest
def test_manifestbackfillworker_mislinked_manifest(clear_rows, initialized_db): """ Tests that a manifest whose image is mislinked will have its storages relinked properly. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository('devtable', 'complex') tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0') tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0') # Add a mislinked manifest, by having its layer point to a blob in v3.0 but its image # be the v5.0 image. builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag') builder.add_layer(tag_v30.image.storage.content_checksum, '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) mislinked_manifest = TagManifest.create(json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50) # Backfill the manifest and ensure its proper content checksum was linked. assert _backfill_manifest(mislinked_manifest) map_row = TagManifestToManifest.get(tag_manifest=mislinked_manifest) assert not map_row.broken manifest_row = map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert legacy_image == tag_v50.image manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) assert len(manifest_blobs) == 1 assert manifest_blobs[0].blob.content_checksum == tag_v30.image.storage.content_checksum
def delete_manifests(): ManifestLegacyImage.delete().execute() ManifestBlob.delete().execute() Manifest.delete().execute() TagManifestToManifest.delete().execute() TagManifest.delete().execute() return "OK"
def clear_rows(initialized_db): # Remove all new-style rows so we can backfill. TagToRepositoryTag.delete().execute() Tag.delete().execute() TagManifestLabelMap.delete().execute() ManifestLabel.delete().execute() ManifestBlob.delete().execute() ManifestLegacyImage.delete().execute() TagManifestToManifest.delete().execute() Manifest.delete().execute() TagManifestLabel.delete().execute() TagManifest.delete().execute()
def _get_manifest_id(repositorytag): repository_tag_datatype = TagDataType.for_repository_tag(repositorytag) # Retrieve the TagManifest for the RepositoryTag, backfilling if necessary. with db_transaction(): manifest_datatype = None try: manifest_datatype = pre_oci_model.get_manifest_for_tag( repository_tag_datatype, backfill_if_necessary=True) except MalformedSchema1Manifest: logger.exception('Error backfilling manifest for tag `%s`', repositorytag.id) if manifest_datatype is None: logger.error('Could not load or backfill manifest for tag `%s`', repositorytag.id) # Create a broken manifest for the tag. tag_manifest = TagManifest.create(tag=repositorytag, digest='BROKEN-%s' % repositorytag.id, json_data='{}') else: # Retrieve the new-style Manifest for the TagManifest, if any. try: tag_manifest = TagManifest.get(id=manifest_datatype._db_id) except TagManifest.DoesNotExist: logger.exception('Could not find tag manifest') return None try: found = TagManifestToManifest.get(tag_manifest=tag_manifest).manifest # Verify that the new-style manifest has the same contents as the old-style manifest. # If not, update and then return. This is an extra check put in place to ensure unicode # manifests have been correctly copied. if found.manifest_bytes != tag_manifest.json_data: logger.warning('Fixing manifest `%s`', found.id) found.manifest_bytes = tag_manifest.json_data found.save() return found.id except TagManifestToManifest.DoesNotExist: # Could not find the new style manifest, so backfill. _backfill_manifest(tag_manifest) # Try to retrieve the manifest again, since we've performed a backfill. try: return TagManifestToManifest.get(tag_manifest=tag_manifest).manifest_id except TagManifestToManifest.DoesNotExist: return None
def _purge_pre_oci_tag(tag, context, allow_non_expired=False): assert tag.repository_id == context.repository.id if not allow_non_expired: assert tag.lifetime_end_ts is not None assert tag.lifetime_end_ts <= pre_oci_tag.get_epoch_timestamp() # If it exists, GC the tag manifest. try: tag_manifest = TagManifest.select().where(TagManifest.tag == tag).get() _garbage_collect_legacy_manifest(tag_manifest.id, context) except TagManifest.DoesNotExist: pass # Add the tag's legacy image to be GCed. context.add_legacy_image_id(tag.image_id) with db_transaction(): # Reload the tag and verify its lifetime_end_ts has not changed. try: reloaded_tag = db_for_update(RepositoryTag.select().where(RepositoryTag.id == tag.id)).get() except RepositoryTag.DoesNotExist: return False assert reloaded_tag.id == tag.id assert reloaded_tag.repository_id == context.repository.id if reloaded_tag.lifetime_end_ts != tag.lifetime_end_ts: return False # Delete mapping rows. TagToRepositoryTag.delete().where(TagToRepositoryTag.repository_tag == reloaded_tag).execute() # Delete the tag. reloaded_tag.delete_instance()
def store_tag_manifest_for_repo(repository_id, tag_name, manifest, leaf_layer_id, storage_id_map, reversion=False): """ Stores a tag manifest for a specific tag name in the database. Returns the TagManifest object, as well as a boolean indicating whether the TagManifest was created. """ # Create the new-style OCI manifest and its blobs. oci_manifest = _populate_manifest_and_blobs(repository_id, manifest, storage_id_map, leaf_layer_id=leaf_layer_id) # Create the tag for the tag manifest. tag = create_or_update_tag_for_repo(repository_id, tag_name, leaf_layer_id, reversion=reversion, oci_manifest=oci_manifest) # Add a tag manifest pointing to that tag. try: manifest = TagManifest.get(digest=manifest.digest) manifest.tag = tag manifest.save() return manifest, False except TagManifest.DoesNotExist: created = _associate_manifest(tag, oci_manifest) return created, True
def _namespace_from_kwargs(self, args_dict): if "namespace_name" in args_dict: return args_dict["namespace_name"] if "repository_ref" in args_dict: return args_dict["repository_ref"].namespace_name if "tag" in args_dict: return args_dict["tag"].repository.namespace_name if "manifest" in args_dict: manifest = args_dict["manifest"] if manifest._is_tag_manifest: return TagManifest.get( id=manifest._db_id).tag.repository.namespace_user.username else: return Manifest.get( id=manifest._db_id).repository.namespace_user.username if "manifest_or_legacy_image" in args_dict: manifest_or_legacy_image = args_dict["manifest_or_legacy_image"] if isinstance(manifest_or_legacy_image, LegacyImage): return Image.get(id=manifest_or_legacy_image._db_id ).repository.namespace_user.username else: manifest = manifest_or_legacy_image if manifest._is_tag_manifest: return TagManifest.get( id=manifest._db_id ).tag.repository.namespace_user.username else: return Manifest.get( id=manifest._db_id).repository.namespace_user.username if "derived_image" in args_dict: return DerivedStorageForImage.get( id=args_dict["derived_image"]._db_id ).source_image.repository.namespace_user.username if "blob" in args_dict: return "" # Blob functions are shared, so no need to do anything. if "blob_upload" in args_dict: return "" # Blob functions are shared, so no need to do anything. raise Exception("Unknown namespace for dict `%s`" % args_dict)
def _associate_manifest(tag, oci_manifest): with db_transaction(): tag_manifest = TagManifest.create( tag=tag, digest=oci_manifest.digest, json_data=oci_manifest.manifest_bytes) TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=oci_manifest) return tag_manifest
def _namespace_from_kwargs(self, args_dict): if 'namespace_name' in args_dict: return args_dict['namespace_name'] if 'repository_ref' in args_dict: return args_dict['repository_ref'].namespace_name if 'tag' in args_dict: return args_dict['tag'].repository.namespace_name if 'manifest' in args_dict: manifest = args_dict['manifest'] if manifest._is_tag_manifest: return TagManifest.get(id=manifest._db_id).tag.repository.namespace_user.username else: return Manifest.get(id=manifest._db_id).repository.namespace_user.username if 'manifest_or_legacy_image' in args_dict: manifest_or_legacy_image = args_dict['manifest_or_legacy_image'] if isinstance(manifest_or_legacy_image, LegacyImage): return Image.get(id=manifest_or_legacy_image._db_id).repository.namespace_user.username else: manifest = manifest_or_legacy_image if manifest._is_tag_manifest: return TagManifest.get(id=manifest._db_id).tag.repository.namespace_user.username else: return Manifest.get(id=manifest._db_id).repository.namespace_user.username if 'derived_image' in args_dict: return (DerivedStorageForImage .get(id=args_dict['derived_image']._db_id) .source_image .repository .namespace_user .username) if 'blob' in args_dict: return '' # Blob functions are shared, so no need to do anything. if 'blob_upload' in args_dict: return '' # Blob functions are shared, so no need to do anything. raise Exception('Unknown namespace for dict `%s`' % args_dict)
def get_tag_manifest_digests(tags): """ Returns a map from tag ID to its associated manifest digest, if any. """ if not tags: return dict() manifests = TagManifest.select( TagManifest.tag, TagManifest.digest).where(TagManifest.tag << [t.id for t in tags]) return {manifest.tag_id: manifest.digest for manifest in manifests}
def _load_repo_manifests(namespace, repo_name, allow_dead=False): query = (TagManifest.select( TagManifest, RepositoryTag).join(RepositoryTag).join(Image).join(Repository).join( Namespace, on=(Namespace.id == Repository.namespace_user)).where( Repository.name == repo_name, Namespace.username == namespace)) if not allow_dead: query = _tag_alive(query) return query
def test_manifestbackfillworker_broken_manifest(clear_rows, initialized_db): # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() # Add a broken manifest. broken_manifest = TagManifest.create(json_data='wat?', digest='sha256:foobar', tag=RepositoryTag.get()) # Ensure the backfill works. assert _backfill_manifest(broken_manifest) # Ensure the mapping is marked as broken. map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) assert map_row.broken manifest_row = map_row.manifest assert manifest_row.manifest_bytes == broken_manifest.json_data assert manifest_row.digest == broken_manifest.digest assert manifest_row.repository == broken_manifest.tag.repository legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert broken_manifest.tag.image == legacy_image
def verify_backfill(namespace_name): logger.info('Checking namespace %s', namespace_name) namespace_user = model.user.get_namespace_user(namespace_name) assert namespace_user repo_tags = (RepositoryTag .select() .join(Repository) .where(Repository.namespace_user == namespace_user) .where(RepositoryTag.hidden == False)) repo_tags = list(repo_tags) logger.info('Found %s tags', len(repo_tags)) for index, repo_tag in enumerate(repo_tags): logger.info('Checking tag %s under repository %s (%s/%s)', repo_tag.name, repo_tag.repository.name, index + 1, len(repo_tags)) tag = TagToRepositoryTag.get(repository_tag=repo_tag).tag assert not tag.hidden assert tag.repository == repo_tag.repository assert tag.name == repo_tag.name, _vs(tag.name, repo_tag.name) assert tag.repository == repo_tag.repository, _vs(tag.repository_id, repo_tag.repository_id) assert tag.reversion == repo_tag.reversion, _vs(tag.reversion, repo_tag.reversion) start_check = int(tag.lifetime_start_ms / 1000) == repo_tag.lifetime_start_ts assert start_check, _vs(tag.lifetime_start_ms, repo_tag.lifetime_start_ts) if repo_tag.lifetime_end_ts is not None: end_check = int(tag.lifetime_end_ms / 1000) == repo_tag.lifetime_end_ts assert end_check, _vs(tag.lifetime_end_ms, repo_tag.lifetime_end_ts) else: assert tag.lifetime_end_ms is None try: tag_manifest = tag.manifest repo_tag_manifest = TagManifest.get(tag=repo_tag) digest_check = tag_manifest.digest == repo_tag_manifest.digest assert digest_check, _vs(tag_manifest.digest, repo_tag_manifest.digest) bytes_check = tag_manifest.manifest_bytes == repo_tag_manifest.json_data assert bytes_check, _vs(tag_manifest.manifest_bytes, repo_tag_manifest.json_data) except TagManifest.DoesNotExist: logger.info('No tag manifest found for repository tag %s', repo_tag.id) mli = ManifestLegacyImage.get(manifest=tag_manifest) assert mli.repository == repo_tag.repository manifest_legacy_image = mli.image assert manifest_legacy_image == repo_tag.image, _vs(manifest_legacy_image.id, repo_tag.image_id)
def _garbage_collect_legacy_manifest(legacy_manifest_id, context): assert legacy_manifest_id is not None # Add the labels to be GCed. query = TagManifestLabel.select().where( TagManifestLabel.annotated == legacy_manifest_id) for manifest_label in query: context.add_label_id(manifest_label.label_id) # Delete the tag manifest. with db_transaction(): try: tag_manifest = TagManifest.select().where( TagManifest.id == legacy_manifest_id).get() except TagManifest.DoesNotExist: return False assert tag_manifest.id == legacy_manifest_id assert tag_manifest.tag.repository_id == context.repository.id # Delete any label mapping rows. (TagManifestLabelMap.delete().where( TagManifestLabelMap.tag_manifest == legacy_manifest_id).execute()) # Delete the label rows. TagManifestLabel.delete().where( TagManifestLabel.annotated == legacy_manifest_id).execute() # Delete the mapping row if it exists. try: tmt = (TagManifestToManifest.select().where( TagManifestToManifest.tag_manifest == tag_manifest).get()) context.add_manifest_id(tmt.manifest_id) tmt_deleted = tmt.delete_instance() if tmt_deleted: gc_table_rows_deleted.labels( table="TagManifestToManifest").inc() except TagManifestToManifest.DoesNotExist: pass # Delete the tag manifest. tag_manifest_deleted = tag_manifest.delete_instance() if tag_manifest_deleted: gc_table_rows_deleted.labels(table="TagManifest").inc() return True
def compute_layer_id(layer): """ Returns the ID for the layer in the security scanner. """ # NOTE: this is temporary until we switch to Clair V3. if isinstance(layer, ManifestDataType): if layer._is_tag_manifest: layer = TagManifest.get(id=layer._db_id).tag.image else: manifest = Manifest.get(id=layer._db_id) try: layer = ManifestLegacyImage.get(manifest=manifest).image except ManifestLegacyImage.DoesNotExist: return None elif isinstance(layer, LegacyImage): layer = Image.get(id=layer._db_id) assert layer.docker_image_id assert layer.storage.uuid return '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
def test_retarget_tag(initialized_db): repo = get_repository("devtable", "history") results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name="latest") assert len(results) == 2 assert results[0].lifetime_end_ms is None assert results[1].lifetime_end_ms is not None # Revert back to the original manifest. created = retarget_tag("latest", results[0].manifest, is_reversion=True, now_ms=results[1].lifetime_end_ms + 10000) assert created.lifetime_end_ms is None assert created.reversion assert created.name == "latest" assert created.manifest == results[0].manifest # Verify in the history. results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name="latest") assert len(results) == 3 assert results[0].lifetime_end_ms is None assert results[1].lifetime_end_ms is not None assert results[2].lifetime_end_ms is not None assert results[0] == created # Verify old-style tables. repository_tag = TagToRepositoryTag.get(tag=created).repository_tag assert repository_tag.lifetime_start_ts == int(created.lifetime_start_ms / 1000) tag_manifest = TagManifest.get(tag=repository_tag) assert TagManifestToManifest.get( tag_manifest=tag_manifest).manifest == created.manifest
def restore_tag_to_manifest(repo_obj, tag_name, manifest_digest): """ Restores a tag to a specific manifest digest. """ with db_transaction(): # Verify that the manifest digest already existed under this repository under the # tag. try: tag_manifest = (TagManifest.select( TagManifest, RepositoryTag, Image).join(RepositoryTag).join(Image).where( RepositoryTag.repository == repo_obj).where( RepositoryTag.name == tag_name).where( TagManifest.digest == manifest_digest).get()) except TagManifest.DoesNotExist: raise DataModelException( "Cannot restore to unknown or invalid digest") # Lookup the existing image, if any. try: existing_image = get_repo_tag_image(repo_obj, tag_name) except DataModelException: existing_image = None docker_image_id = tag_manifest.tag.image.docker_image_id oci_manifest = None try: oci_manifest = Manifest.get(repository=repo_obj, digest=manifest_digest) except Manifest.DoesNotExist: pass # Change the tag and tag manifest to point to the updated image. updated_tag = create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id, reversion=True, oci_manifest=oci_manifest) tag_manifest.tag = updated_tag tag_manifest.save() return existing_image
def upgrade(op, tables, tester): # Backfill migration removed. if not tester.is_testing: assert TagManifest.select().count() == 0
def retarget_tag(tag_name, manifest_id, is_reversion=False, now_ms=None, adjust_old_model=True): """ Creates or updates a tag with the specified name to point to the given manifest under its repository. If this action is a reversion to a previous manifest, is_reversion should be set to True. Returns the newly created tag row or None on error. """ try: manifest = (Manifest.select( Manifest, MediaType).join(MediaType).where(Manifest.id == manifest_id).get()) except Manifest.DoesNotExist: return None # CHECK: Make sure that we are not mistargeting a schema 1 manifest to a tag with a different # name. if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES: try: parsed = DockerSchema1Manifest(Bytes.for_string_or_unicode( manifest.manifest_bytes), validate=False) if parsed.tag != tag_name: logger.error( "Tried to re-target schema1 manifest with tag `%s` to tag `%s", parsed.tag, tag_name, ) return None except MalformedSchema1Manifest: logger.exception("Could not parse schema1 manifest") return None legacy_image = get_legacy_image_for_manifest(manifest) now_ms = now_ms or get_epoch_timestamp_ms() now_ts = int(now_ms / 1000) with db_transaction(): # Lookup an existing tag in the repository with the same name and, if present, mark it # as expired. existing_tag = get_tag(manifest.repository_id, tag_name) if existing_tag is not None: _, okay = set_tag_end_ms(existing_tag, now_ms) # TODO: should we retry here and/or use a for-update? if not okay: return None # Create a new tag pointing to the manifest with a lifetime start of now. created = Tag.create( name=tag_name, repository=manifest.repository_id, lifetime_start_ms=now_ms, reversion=is_reversion, manifest=manifest, tag_kind=Tag.tag_kind.get_id("tag"), ) # TODO: Remove the linkage code once RepositoryTag is gone. # If this is a schema 1 manifest, then add a TagManifest linkage to it. Otherwise, it will only # be pullable via the new OCI model. if adjust_old_model: if (manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES and legacy_image is not None): old_style_tag = RepositoryTag.create( repository=manifest.repository_id, image=legacy_image, name=tag_name, lifetime_start_ts=now_ts, reversion=is_reversion, ) TagToRepositoryTag.create(tag=created, repository_tag=old_style_tag, repository=manifest.repository_id) tag_manifest = TagManifest.create( tag=old_style_tag, digest=manifest.digest, json_data=manifest.manifest_bytes) TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest, repository=manifest.repository_id) return created
def get_tag_manifest(tag): try: return TagManifest.get(tag=tag) except TagManifest.DoesNotExist: return None
def test_tagbackfillworker(clear_all_rows, initialized_db): # Remove the new-style rows so we can backfill. TagToRepositoryTag.delete().execute() Tag.delete().execute() if clear_all_rows: TagManifestLabelMap.delete().execute() ManifestLabel.delete().execute() ManifestBlob.delete().execute() ManifestLegacyImage.delete().execute() TagManifestToManifest.delete().execute() Manifest.delete().execute() found_dead_tag = False for repository_tag in list(RepositoryTag.select()): # Backfill the tag. assert backfill_tag(repository_tag) # Ensure if we try again, the backfill is skipped. assert not backfill_tag(repository_tag) # Ensure that we now have the expected tag rows. tag_to_repo_tag = TagToRepositoryTag.get(repository_tag=repository_tag) tag = tag_to_repo_tag.tag assert tag.name == repository_tag.name assert tag.repository == repository_tag.repository assert not tag.hidden assert tag.reversion == repository_tag.reversion if repository_tag.lifetime_start_ts is None: assert tag.lifetime_start_ms is None else: assert tag.lifetime_start_ms == (repository_tag.lifetime_start_ts * 1000) if repository_tag.lifetime_end_ts is None: assert tag.lifetime_end_ms is None else: assert tag.lifetime_end_ms == (repository_tag.lifetime_end_ts * 1000) found_dead_tag = True assert tag.manifest # Ensure that we now have the expected manifest rows. try: tag_manifest = TagManifest.get(tag=repository_tag) except TagManifest.DoesNotExist: continue map_row = TagManifestToManifest.get(tag_manifest=tag_manifest) assert not map_row.broken manifest_row = map_row.manifest assert manifest_row.manifest_bytes == tag_manifest.json_data assert manifest_row.digest == tag_manifest.digest assert manifest_row.repository == tag_manifest.tag.repository assert tag.manifest == map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert tag_manifest.tag.image == legacy_image expected_storages = {tag_manifest.tag.image.storage.id} for parent_image_id in tag_manifest.tag.image.ancestor_id_list(): expected_storages.add(Image.get(id=parent_image_id).storage_id) found_storages = { manifest_blob.blob_id for manifest_blob in ManifestBlob.select().where( ManifestBlob.manifest == manifest_row) } assert expected_storages == found_storages # Ensure the labels were copied over. tmls = list(TagManifestLabel.select().where( TagManifestLabel.annotated == tag_manifest)) expected_labels = {tml.label_id for tml in tmls} found_labels = { m.label_id for m in ManifestLabel.select().where( ManifestLabel.manifest == manifest_row) } assert found_labels == expected_labels # Verify at the repository level. for repository in list(Repository.select()): tags = RepositoryTag.select().where( RepositoryTag.repository == repository, RepositoryTag.hidden == False) oci_tags = Tag.select().where(Tag.repository == repository) assert len(tags) == len(oci_tags) assert {t.name for t in tags} == {t.name for t in oci_tags} for tag in tags: tag_manifest = TagManifest.get(tag=tag) ttr = TagToRepositoryTag.get(repository_tag=tag) manifest = ttr.tag.manifest assert tag_manifest.json_data == manifest.manifest_bytes assert tag_manifest.digest == manifest.digest assert tag.image == ManifestLegacyImage.get( manifest=manifest).image assert tag.lifetime_start_ts == (ttr.tag.lifetime_start_ms / 1000) if tag.lifetime_end_ts: assert tag.lifetime_end_ts == (ttr.tag.lifetime_end_ms / 1000) else: assert ttr.tag.lifetime_end_ms is None assert found_dead_tag
def _backfill_manifest(tag_manifest): logger.info('Backfilling manifest for tag manifest %s', tag_manifest.id) # Ensure that a mapping row doesn't already exist. If it does, we've been preempted. if lookup_manifest_map_row(tag_manifest): return False # Parse the manifest. If we cannot parse, then we treat the manifest as broken and just emit it # without additional rows or data, as it will eventually not be useful. is_broken = False try: manifest = DockerSchema1Manifest(Bytes.for_string_or_unicode( tag_manifest.json_data), validate=False) except ManifestException: logger.exception('Exception when trying to parse manifest %s', tag_manifest.id) manifest = BrokenManifest(tag_manifest.digest, tag_manifest.json_data) is_broken = True # Lookup the storages for the digests. root_image = tag_manifest.tag.image repository = tag_manifest.tag.repository image_storage_id_map = { root_image.storage.content_checksum: root_image.storage.id } try: parent_images = get_parent_images(repository.namespace_user.username, repository.name, root_image) except DataModelException: logger.exception( 'Exception when trying to load parent images for manifest `%s`', tag_manifest.id) parent_images = {} is_broken = True for parent_image in parent_images: image_storage_id_map[ parent_image.storage.content_checksum] = parent_image.storage.id # Ensure that all the expected blobs have been found. If not, we lookup the blob under the repo # and add its storage ID. If the blob is not found, we mark the manifest as broken. storage_ids = set() try: for blob_digest in manifest.get_blob_digests_for_translation(): if blob_digest in image_storage_id_map: storage_ids.add(image_storage_id_map[blob_digest]) else: logger.debug( 'Blob `%s` not found in images for manifest `%s`; checking repo', blob_digest, tag_manifest.id) try: blob_storage = get_repo_blob_by_digest( repository.namespace_user.username, repository.name, blob_digest) storage_ids.add(blob_storage.id) except BlobDoesNotExist: logger.debug( 'Blob `%s` not found in repo for manifest `%s`', blob_digest, tag_manifest.id) is_broken = True except MalformedSchema1Manifest: logger.warning( 'Found malformed schema 1 manifest during blob backfill') is_broken = True with db_transaction(): # Re-retrieve the tag manifest to ensure it still exists and we're pointing at the correct tag. try: tag_manifest = TagManifest.get(id=tag_manifest.id) except TagManifest.DoesNotExist: return True # Ensure it wasn't already created. if lookup_manifest_map_row(tag_manifest): return False # Check for a pre-existing manifest matching the digest in the repository. This can happen # if we've already created the manifest row (typically for tag reverision). try: manifest_row = Manifest.get(digest=manifest.digest, repository=tag_manifest.tag.repository) except Manifest.DoesNotExist: # Create the new-style rows for the manifest. try: manifest_row = populate_manifest(tag_manifest.tag.repository, manifest, tag_manifest.tag.image, storage_ids) except IntegrityError: # Pre-empted. return False # Create the mapping row. If we find another was created for this tag manifest in the # meantime, then we've been preempted. try: TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest_row, broken=is_broken) except IntegrityError: return False # Backfill any labels on the manifest. _backfill_labels(tag_manifest, manifest_row, repository) return True