def test_manifestbackfillworker_repeat_digest(clear_rows, initialized_db): """ Tests that a manifest with a shared digest will be properly linked. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository("devtable", "gargantuan") tag_v30 = model.tag.get_active_tag("devtable", "gargantuan", "v3.0") tag_v50 = model.tag.get_active_tag("devtable", "gargantuan", "v5.0") # Build a manifest and assign it to both tags (this is allowed in the old model). builder = DockerSchema1ManifestBuilder("devtable", "gargantuan", "sometag") builder.add_layer("sha256:deadbeef", '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) manifest_1 = TagManifest.create(json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v30) manifest_2 = TagManifest.create(json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50) # Backfill "both" manifests and ensure both are pointed to by a single resulting row. assert _backfill_manifest(manifest_1) assert _backfill_manifest(manifest_2) map_row1 = TagManifestToManifest.get(tag_manifest=manifest_1) map_row2 = TagManifestToManifest.get(tag_manifest=manifest_2) assert map_row1.manifest == map_row2.manifest
def test_manifest_backfill_broken_tag(clear_rows, initialized_db): """ Tests backfilling a broken tag. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() # Create a tag with an image referenced missing parent images. repo = model.repository.get_repository("devtable", "gargantuan") broken_image = Image.create( docker_image_id="foo", repository=repo, ancestors="/348723847234/", storage=ImageStorage.get(), ) broken_image_tag = RepositoryTag.create(repository=repo, image=broken_image, name="broken") # Backfill the tag. assert backfill_tag(broken_image_tag) # Ensure we backfilled, even though we reference a broken manifest. tag_manifest = TagManifest.get(tag=broken_image_tag) map_row = TagManifestToManifest.get(tag_manifest=tag_manifest) manifest = map_row.manifest assert manifest.manifest_bytes == tag_manifest.json_data tag = TagToRepositoryTag.get(repository_tag=broken_image_tag).tag assert tag.name == "broken" assert tag.manifest == manifest
def test_manifestbackfillworker_mislinked_invalid_manifest(clear_rows, initialized_db): """ Tests that a manifest whose image is mislinked will attempt to have its storages relinked properly. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository("devtable", "complex") tag_v50 = model.tag.get_active_tag("devtable", "gargantuan", "v5.0") # Add a mislinked manifest, by having its layer point to an invalid blob but its image # be the v5.0 image. builder = DockerSchema1ManifestBuilder("devtable", "gargantuan", "sometag") builder.add_layer("sha256:deadbeef", '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) broken_manifest = TagManifest.create( json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50 ) # Backfill the manifest and ensure it is marked as broken. assert _backfill_manifest(broken_manifest) map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) assert map_row.broken manifest_row = map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert legacy_image == tag_v50.image manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) assert len(manifest_blobs) == 0
def test_manifestbackfillworker_mislinked_manifest(clear_rows, initialized_db): """ Tests that a manifest whose image is mislinked will have its storages relinked properly. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository('devtable', 'complex') tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0') tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0') # Add a mislinked manifest, by having its layer point to a blob in v3.0 but its image # be the v5.0 image. builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag') builder.add_layer(tag_v30.image.storage.content_checksum, '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) mislinked_manifest = TagManifest.create(json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50) # Backfill the manifest and ensure its proper content checksum was linked. assert _backfill_manifest(mislinked_manifest) map_row = TagManifestToManifest.get(tag_manifest=mislinked_manifest) assert not map_row.broken manifest_row = map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert legacy_image == tag_v50.image manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) assert len(manifest_blobs) == 1 assert manifest_blobs[0].blob.content_checksum == tag_v30.image.storage.content_checksum
def clear_rows(initialized_db): # Remove all new-style rows so we can backfill. TagToRepositoryTag.delete().execute() Tag.delete().execute() TagManifestLabelMap.delete().execute() ManifestLabel.delete().execute() ManifestBlob.delete().execute() ManifestLegacyImage.delete().execute() TagManifestToManifest.delete().execute() Manifest.delete().execute() TagManifestLabel.delete().execute() TagManifest.delete().execute()
def delete_manifest_label(label_uuid, manifest): """ Deletes the manifest label on the tag manifest with the given ID. Returns the label deleted or None if none. """ # Find the label itself. label = get_manifest_label(label_uuid, manifest) if label is None: return None if not label.source_type.mutable: raise DataModelException('Cannot delete immutable label') # Delete the mapping records and label. # TODO: Remove this code once the TagManifest table is gone. with db_transaction(): (TagManifestLabelMap .delete() .where(TagManifestLabelMap.label == label) .execute()) deleted_count = TagManifestLabel.delete().where(TagManifestLabel.label == label).execute() if deleted_count != 1: logger.warning('More than a single label deleted for matching label %s', label_uuid) deleted_count = ManifestLabel.delete().where(ManifestLabel.label == label).execute() if deleted_count != 1: logger.warning('More than a single label deleted for matching label %s', label_uuid) label.delete_instance(recursive=False) return label
def delete_manifest_label(label_uuid, tag_manifest): """ Deletes the manifest label on the tag manifest with the given ID. """ # Find the label itself. label = get_manifest_label(label_uuid, tag_manifest) if label is None: return None if not label.source_type.mutable: raise DataModelException("Cannot delete immutable label") # Delete the mapping records and label. (TagManifestLabelMap.delete().where( TagManifestLabelMap.label == label).execute()) deleted_count = TagManifestLabel.delete().where( TagManifestLabel.label == label).execute() if deleted_count != 1: logger.warning( "More than a single label deleted for matching label %s", label_uuid) deleted_count = ManifestLabel.delete().where( ManifestLabel.label == label).execute() if deleted_count != 1: logger.warning( "More than a single label deleted for matching label %s", label_uuid) label.delete_instance(recursive=False) return label
def _garbage_collect_legacy_manifest(legacy_manifest_id, context): assert legacy_manifest_id is not None # Add the labels to be GCed. query = TagManifestLabel.select().where( TagManifestLabel.annotated == legacy_manifest_id) for manifest_label in query: context.add_label_id(manifest_label.label_id) # Delete the tag manifest. with db_transaction(): try: tag_manifest = TagManifest.select().where( TagManifest.id == legacy_manifest_id).get() except TagManifest.DoesNotExist: return False assert tag_manifest.id == legacy_manifest_id assert tag_manifest.tag.repository_id == context.repository.id # Delete any label mapping rows. (TagManifestLabelMap.delete().where( TagManifestLabelMap.tag_manifest == legacy_manifest_id).execute()) # Delete the label rows. TagManifestLabel.delete().where( TagManifestLabel.annotated == legacy_manifest_id).execute() # Delete the mapping row if it exists. try: tmt = (TagManifestToManifest.select().where( TagManifestToManifest.tag_manifest == tag_manifest).get()) context.add_manifest_id(tmt.manifest_id) tmt_deleted = tmt.delete_instance() if tmt_deleted: gc_table_rows_deleted.labels( table="TagManifestToManifest").inc() except TagManifestToManifest.DoesNotExist: pass # Delete the tag manifest. tag_manifest_deleted = tag_manifest.delete_instance() if tag_manifest_deleted: gc_table_rows_deleted.labels(table="TagManifest").inc() return True
def test_manifestbackfillworker_broken_manifest(clear_rows, initialized_db): # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() # Add a broken manifest. broken_manifest = TagManifest.create(json_data='wat?', digest='sha256:foobar', tag=RepositoryTag.get()) # Ensure the backfill works. assert _backfill_manifest(broken_manifest) # Ensure the mapping is marked as broken. map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) assert map_row.broken manifest_row = map_row.manifest assert manifest_row.manifest_bytes == broken_manifest.json_data assert manifest_row.digest == broken_manifest.digest assert manifest_row.repository == broken_manifest.tag.repository legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert broken_manifest.tag.image == legacy_image