def test_manifestbackfillworker_mislinked_invalid_manifest(clear_rows, initialized_db): """ Tests that a manifest whose image is mislinked will attempt to have its storages relinked properly. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository("devtable", "complex") tag_v50 = model.tag.get_active_tag("devtable", "gargantuan", "v5.0") # Add a mislinked manifest, by having its layer point to an invalid blob but its image # be the v5.0 image. builder = DockerSchema1ManifestBuilder("devtable", "gargantuan", "sometag") builder.add_layer("sha256:deadbeef", '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) broken_manifest = TagManifest.create( json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50 ) # Backfill the manifest and ensure it is marked as broken. assert _backfill_manifest(broken_manifest) map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) assert map_row.broken manifest_row = map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert legacy_image == tag_v50.image manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) assert len(manifest_blobs) == 0
def test_manifestbackfillworker_mislinked_manifest(clear_rows, initialized_db): """ Tests that a manifest whose image is mislinked will have its storages relinked properly. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository('devtable', 'complex') tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0') tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0') # Add a mislinked manifest, by having its layer point to a blob in v3.0 but its image # be the v5.0 image. builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag') builder.add_layer(tag_v30.image.storage.content_checksum, '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) mislinked_manifest = TagManifest.create(json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50) # Backfill the manifest and ensure its proper content checksum was linked. assert _backfill_manifest(mislinked_manifest) map_row = TagManifestToManifest.get(tag_manifest=mislinked_manifest) assert not map_row.broken manifest_row = map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert legacy_image == tag_v50.image manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) assert len(manifest_blobs) == 1 assert manifest_blobs[0].blob.content_checksum == tag_v30.image.storage.content_checksum
def test_manifest_backfill_broken_tag(clear_rows, initialized_db): """ Tests backfilling a broken tag. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() # Create a tag with an image referenced missing parent images. repo = model.repository.get_repository("devtable", "gargantuan") broken_image = Image.create( docker_image_id="foo", repository=repo, ancestors="/348723847234/", storage=ImageStorage.get(), ) broken_image_tag = RepositoryTag.create(repository=repo, image=broken_image, name="broken") # Backfill the tag. assert backfill_tag(broken_image_tag) # Ensure we backfilled, even though we reference a broken manifest. tag_manifest = TagManifest.get(tag=broken_image_tag) map_row = TagManifestToManifest.get(tag_manifest=tag_manifest) manifest = map_row.manifest assert manifest.manifest_bytes == tag_manifest.json_data tag = TagToRepositoryTag.get(repository_tag=broken_image_tag).tag assert tag.name == "broken" assert tag.manifest == manifest
def test_manifestbackfillworker_repeat_digest(clear_rows, initialized_db): """ Tests that a manifest with a shared digest will be properly linked. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository("devtable", "gargantuan") tag_v30 = model.tag.get_active_tag("devtable", "gargantuan", "v3.0") tag_v50 = model.tag.get_active_tag("devtable", "gargantuan", "v5.0") # Build a manifest and assign it to both tags (this is allowed in the old model). builder = DockerSchema1ManifestBuilder("devtable", "gargantuan", "sometag") builder.add_layer("sha256:deadbeef", '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) manifest_1 = TagManifest.create(json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v30) manifest_2 = TagManifest.create(json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50) # Backfill "both" manifests and ensure both are pointed to by a single resulting row. assert _backfill_manifest(manifest_1) assert _backfill_manifest(manifest_2) map_row1 = TagManifestToManifest.get(tag_manifest=manifest_1) map_row2 = TagManifestToManifest.get(tag_manifest=manifest_2) assert map_row1.manifest == map_row2.manifest
def clear_rows(initialized_db): # Remove all new-style rows so we can backfill. TagToRepositoryTag.delete().execute() Tag.delete().execute() TagManifestLabelMap.delete().execute() ManifestLabel.delete().execute() ManifestBlob.delete().execute() ManifestLegacyImage.delete().execute() TagManifestToManifest.delete().execute() Manifest.delete().execute() TagManifestLabel.delete().execute() TagManifest.delete().execute()
def delete_manifest_label(label_uuid, manifest): """ Deletes the manifest label on the tag manifest with the given ID. Returns the label deleted or None if none. """ # Find the label itself. label = get_manifest_label(label_uuid, manifest) if label is None: return None if not label.source_type.mutable: raise DataModelException('Cannot delete immutable label') # Delete the mapping records and label. # TODO: Remove this code once the TagManifest table is gone. with db_transaction(): (TagManifestLabelMap .delete() .where(TagManifestLabelMap.label == label) .execute()) deleted_count = TagManifestLabel.delete().where(TagManifestLabel.label == label).execute() if deleted_count != 1: logger.warning('More than a single label deleted for matching label %s', label_uuid) deleted_count = ManifestLabel.delete().where(ManifestLabel.label == label).execute() if deleted_count != 1: logger.warning('More than a single label deleted for matching label %s', label_uuid) label.delete_instance(recursive=False) return label
def _backfill_labels(tag_manifest, manifest, repository): tmls = list(TagManifestLabel.select().where( TagManifestLabel.annotated == tag_manifest)) if not tmls: return for tag_manifest_label in tmls: label = tag_manifest_label.label try: TagManifestLabelMap.get(tag_manifest_label=tag_manifest_label) continue except TagManifestLabelMap.DoesNotExist: pass try: manifest_label = ManifestLabel.create(manifest=manifest, label=label, repository=repository) TagManifestLabelMap.create( manifest_label=manifest_label, tag_manifest_label=tag_manifest_label, label=label, manifest=manifest, tag_manifest=tag_manifest_label.annotated) except IntegrityError: continue
def delete_manifest_label(label_uuid, tag_manifest): """ Deletes the manifest label on the tag manifest with the given ID. """ # Find the label itself. label = get_manifest_label(label_uuid, tag_manifest) if label is None: return None if not label.source_type.mutable: raise DataModelException("Cannot delete immutable label") # Delete the mapping records and label. (TagManifestLabelMap.delete().where( TagManifestLabelMap.label == label).execute()) deleted_count = TagManifestLabel.delete().where( TagManifestLabel.label == label).execute() if deleted_count != 1: logger.warning( "More than a single label deleted for matching label %s", label_uuid) deleted_count = ManifestLabel.delete().where( ManifestLabel.label == label).execute() if deleted_count != 1: logger.warning( "More than a single label deleted for matching label %s", label_uuid) label.delete_instance(recursive=False) return label
def _garbage_collect_legacy_manifest(legacy_manifest_id, context): assert legacy_manifest_id is not None # Add the labels to be GCed. query = TagManifestLabel.select().where( TagManifestLabel.annotated == legacy_manifest_id) for manifest_label in query: context.add_label_id(manifest_label.label_id) # Delete the tag manifest. with db_transaction(): try: tag_manifest = TagManifest.select().where( TagManifest.id == legacy_manifest_id).get() except TagManifest.DoesNotExist: return False assert tag_manifest.id == legacy_manifest_id assert tag_manifest.tag.repository_id == context.repository.id # Delete any label mapping rows. (TagManifestLabelMap.delete().where( TagManifestLabelMap.tag_manifest == legacy_manifest_id).execute()) # Delete the label rows. TagManifestLabel.delete().where( TagManifestLabel.annotated == legacy_manifest_id).execute() # Delete the mapping row if it exists. try: tmt = (TagManifestToManifest.select().where( TagManifestToManifest.tag_manifest == tag_manifest).get()) context.add_manifest_id(tmt.manifest_id) tmt_deleted = tmt.delete_instance() if tmt_deleted: gc_table_rows_deleted.labels( table="TagManifestToManifest").inc() except TagManifestToManifest.DoesNotExist: pass # Delete the tag manifest. tag_manifest_deleted = tag_manifest.delete_instance() if tag_manifest_deleted: gc_table_rows_deleted.labels(table="TagManifest").inc() return True
def create_manifest_label(tag_manifest, key, value, source_type_name, media_type_name=None): """ Creates a new manifest label on a specific tag manifest. """ if not key: raise InvalidLabelKeyException("Missing key on label") # Note that we don't prevent invalid label names coming from the manifest to be stored, as Docker # does not currently prevent them from being put into said manifests. if not validate_label_key(key) and source_type_name != "manifest": raise InvalidLabelKeyException( "Label key `%s` is invalid or reserved" % key) # Find the matching media type. If none specified, we infer. if media_type_name is None: media_type_name = "text/plain" if is_json(value): media_type_name = "application/json" media_type_id = _get_media_type_id(media_type_name) if media_type_id is None: raise InvalidMediaTypeException() source_type_id = _get_label_source_type_id(source_type_name) with db_transaction(): label = Label.create(key=key, value=value, source_type=source_type_id, media_type=media_type_id) tag_manifest_label = TagManifestLabel.create( annotated=tag_manifest, label=label, repository=tag_manifest.tag.repository) try: mapping_row = TagManifestToManifest.get(tag_manifest=tag_manifest) if mapping_row.manifest: manifest_label = ManifestLabel.create( manifest=mapping_row.manifest, label=label, repository=tag_manifest.tag.repository, ) TagManifestLabelMap.create( manifest_label=manifest_label, tag_manifest_label=tag_manifest_label, label=label, manifest=mapping_row.manifest, tag_manifest=tag_manifest, ) except TagManifestToManifest.DoesNotExist: pass return label
def _check_label_used(label_id): assert label_id is not None with db_transaction(): # Check if the label is referenced by another manifest or tag manifest. try: ManifestLabel.select().where(ManifestLabel.label == label_id).get() return True except ManifestLabel.DoesNotExist: pass try: TagManifestLabel.select().where(TagManifestLabel.label == label_id).get() return True except TagManifestLabel.DoesNotExist: pass return False
def _candidates_to_backfill(self): def missing_tmt_query(): return (TagManifestLabel.select().join( TagManifestLabelMap, JOIN.LEFT_OUTER).where(TagManifestLabelMap.id >> None)) min_id = (TagManifestLabel.select(fn.Min(TagManifestLabel.id)).join( TagManifestLabelMap, JOIN.LEFT_OUTER).where(TagManifestLabelMap.id >> None).scalar()) max_id = TagManifestLabel.select(fn.Max(TagManifestLabel.id)).scalar() iterator = yield_random_entries( missing_tmt_query, TagManifestLabel.id, 100, max_id, min_id, ) return iterator
def test_manifestbackfillworker_broken_manifest(clear_rows, initialized_db): # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() # Add a broken manifest. broken_manifest = TagManifest.create(json_data='wat?', digest='sha256:foobar', tag=RepositoryTag.get()) # Ensure the backfill works. assert _backfill_manifest(broken_manifest) # Ensure the mapping is marked as broken. map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) assert map_row.broken manifest_row = map_row.manifest assert manifest_row.manifest_bytes == broken_manifest.json_data assert manifest_row.digest == broken_manifest.digest assert manifest_row.repository == broken_manifest.tag.repository legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert broken_manifest.tag.image == legacy_image
def missing_tmt_query(): return (TagManifestLabel.select().join( TagManifestLabelMap, JOIN.LEFT_OUTER).where(TagManifestLabelMap.id >> None))
def _get_dangling_labels(): label_ids = set([current.id for current in Label.select()]) referenced_by_manifest = set( [mlabel.label_id for mlabel in TagManifestLabel.select()]) return label_ids - referenced_by_manifest
def create_manifest_label(manifest_id, key, value, source_type_name, media_type_name=None, adjust_old_model=True): """ Creates a new manifest label on a specific tag manifest. """ if not key: raise InvalidLabelKeyException() # Note that we don't prevent invalid label names coming from the manifest to be stored, as Docker # does not currently prevent them from being put into said manifests. if not validate_label_key(key) and source_type_name != 'manifest': raise InvalidLabelKeyException('Key `%s` is invalid' % key) # Find the matching media type. If none specified, we infer. if media_type_name is None: media_type_name = 'text/plain' if is_json(value): media_type_name = 'application/json' try: media_type_id = Label.media_type.get_id(media_type_name) except MediaType.DoesNotExist: raise InvalidMediaTypeException() source_type_id = Label.source_type.get_id(source_type_name) # Ensure the manifest exists. try: manifest = (Manifest .select(Manifest, Repository) .join(Repository) .where(Manifest.id == manifest_id) .get()) except Manifest.DoesNotExist: return None repository = manifest.repository # TODO: Remove this code once the TagManifest table is gone. tag_manifest = None if adjust_old_model: try: mapping_row = (TagManifestToManifest .select(TagManifestToManifest, TagManifest) .join(TagManifest) .where(TagManifestToManifest.manifest == manifest) .get()) tag_manifest = mapping_row.tag_manifest except TagManifestToManifest.DoesNotExist: tag_manifest = None with db_transaction(): label = Label.create(key=key, value=value, source_type=source_type_id, media_type=media_type_id) manifest_label = ManifestLabel.create(manifest=manifest_id, label=label, repository=repository) # If there exists a mapping to a TagManifest, add the old-style label. # TODO: Remove this code once the TagManifest table is gone. if tag_manifest: tag_manifest_label = TagManifestLabel.create(annotated=tag_manifest, label=label, repository=repository) TagManifestLabelMap.create(manifest_label=manifest_label, tag_manifest_label=tag_manifest_label, label=label, manifest=manifest, tag_manifest=tag_manifest) return label
def test_tagbackfillworker(clear_all_rows, initialized_db): # Remove the new-style rows so we can backfill. TagToRepositoryTag.delete().execute() Tag.delete().execute() if clear_all_rows: TagManifestLabelMap.delete().execute() ManifestLabel.delete().execute() ManifestBlob.delete().execute() ManifestLegacyImage.delete().execute() TagManifestToManifest.delete().execute() Manifest.delete().execute() found_dead_tag = False for repository_tag in list(RepositoryTag.select()): # Backfill the tag. assert backfill_tag(repository_tag) # Ensure if we try again, the backfill is skipped. assert not backfill_tag(repository_tag) # Ensure that we now have the expected tag rows. tag_to_repo_tag = TagToRepositoryTag.get(repository_tag=repository_tag) tag = tag_to_repo_tag.tag assert tag.name == repository_tag.name assert tag.repository == repository_tag.repository assert not tag.hidden assert tag.reversion == repository_tag.reversion if repository_tag.lifetime_start_ts is None: assert tag.lifetime_start_ms is None else: assert tag.lifetime_start_ms == (repository_tag.lifetime_start_ts * 1000) if repository_tag.lifetime_end_ts is None: assert tag.lifetime_end_ms is None else: assert tag.lifetime_end_ms == (repository_tag.lifetime_end_ts * 1000) found_dead_tag = True assert tag.manifest # Ensure that we now have the expected manifest rows. try: tag_manifest = TagManifest.get(tag=repository_tag) except TagManifest.DoesNotExist: continue map_row = TagManifestToManifest.get(tag_manifest=tag_manifest) assert not map_row.broken manifest_row = map_row.manifest assert manifest_row.manifest_bytes == tag_manifest.json_data assert manifest_row.digest == tag_manifest.digest assert manifest_row.repository == tag_manifest.tag.repository assert tag.manifest == map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert tag_manifest.tag.image == legacy_image expected_storages = {tag_manifest.tag.image.storage.id} for parent_image_id in tag_manifest.tag.image.ancestor_id_list(): expected_storages.add(Image.get(id=parent_image_id).storage_id) found_storages = { manifest_blob.blob_id for manifest_blob in ManifestBlob.select().where( ManifestBlob.manifest == manifest_row) } assert expected_storages == found_storages # Ensure the labels were copied over. tmls = list(TagManifestLabel.select().where( TagManifestLabel.annotated == tag_manifest)) expected_labels = {tml.label_id for tml in tmls} found_labels = { m.label_id for m in ManifestLabel.select().where( ManifestLabel.manifest == manifest_row) } assert found_labels == expected_labels # Verify at the repository level. for repository in list(Repository.select()): tags = RepositoryTag.select().where( RepositoryTag.repository == repository, RepositoryTag.hidden == False) oci_tags = Tag.select().where(Tag.repository == repository) assert len(tags) == len(oci_tags) assert {t.name for t in tags} == {t.name for t in oci_tags} for tag in tags: tag_manifest = TagManifest.get(tag=tag) ttr = TagToRepositoryTag.get(repository_tag=tag) manifest = ttr.tag.manifest assert tag_manifest.json_data == manifest.manifest_bytes assert tag_manifest.digest == manifest.digest assert tag.image == ManifestLegacyImage.get( manifest=manifest).image assert tag.lifetime_start_ts == (ttr.tag.lifetime_start_ms / 1000) if tag.lifetime_end_ts: assert tag.lifetime_end_ts == (ttr.tag.lifetime_end_ms / 1000) else: assert ttr.tag.lifetime_end_ms is None assert found_dead_tag