def test_store_tag_manifest(get_storages, initialized_db): # Create a manifest with some layers. builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'sometag') storages = get_storages() assert storages repo = model.repository.get_repository('devtable', 'simple') storage_id_map = {} for index, storage in enumerate(storages): image_id = 'someimage%s' % index builder.add_layer(storage.content_checksum, json.dumps({'id': image_id})) find_create_or_link_image(image_id, repo, 'devtable', {}, 'local_us') storage_id_map[storage.content_checksum] = storage.id manifest = builder.build(docker_v2_signing_key) tag_manifest, _ = store_tag_manifest_for_testing('devtable', 'simple', 'sometag', manifest, manifest.leaf_layer_v1_image_id, storage_id_map) # Ensure we have the new-model expected rows. mapping_row = TagManifestToManifest.get(tag_manifest=tag_manifest) assert mapping_row.manifest is not None assert mapping_row.manifest.manifest_bytes == manifest.bytes.as_encoded_str() assert mapping_row.manifest.digest == str(manifest.digest) blob_rows = {m.blob_id for m in ManifestBlob.select().where(ManifestBlob.manifest == mapping_row.manifest)} assert blob_rows == {s.id for s in storages} assert ManifestLegacyImage.get(manifest=mapping_row.manifest).image == tag_manifest.tag.image
def test_manifestbackfillworker_mislinked_invalid_manifest(clear_rows, initialized_db): """ Tests that a manifest whose image is mislinked will attempt to have its storages relinked properly. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository("devtable", "complex") tag_v50 = model.tag.get_active_tag("devtable", "gargantuan", "v5.0") # Add a mislinked manifest, by having its layer point to an invalid blob but its image # be the v5.0 image. builder = DockerSchema1ManifestBuilder("devtable", "gargantuan", "sometag") builder.add_layer("sha256:deadbeef", '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) broken_manifest = TagManifest.create( json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50 ) # Backfill the manifest and ensure it is marked as broken. assert _backfill_manifest(broken_manifest) map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) assert map_row.broken manifest_row = map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert legacy_image == tag_v50.image manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) assert len(manifest_blobs) == 0
def test_manifestbackfillworker_mislinked_manifest(clear_rows, initialized_db): """ Tests that a manifest whose image is mislinked will have its storages relinked properly. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository('devtable', 'complex') tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0') tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0') # Add a mislinked manifest, by having its layer point to a blob in v3.0 but its image # be the v5.0 image. builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag') builder.add_layer(tag_v30.image.storage.content_checksum, '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) mislinked_manifest = TagManifest.create(json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50) # Backfill the manifest and ensure its proper content checksum was linked. assert _backfill_manifest(mislinked_manifest) map_row = TagManifestToManifest.get(tag_manifest=mislinked_manifest) assert not map_row.broken manifest_row = map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert legacy_image == tag_v50.image manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) assert len(manifest_blobs) == 1 assert manifest_blobs[0].blob.content_checksum == tag_v30.image.storage.content_checksum
def verify_backfill(namespace_name): logger.info('Checking namespace %s', namespace_name) namespace_user = model.user.get_namespace_user(namespace_name) assert namespace_user repo_tags = (RepositoryTag .select() .join(Repository) .where(Repository.namespace_user == namespace_user) .where(RepositoryTag.hidden == False)) repo_tags = list(repo_tags) logger.info('Found %s tags', len(repo_tags)) for index, repo_tag in enumerate(repo_tags): logger.info('Checking tag %s under repository %s (%s/%s)', repo_tag.name, repo_tag.repository.name, index + 1, len(repo_tags)) tag = TagToRepositoryTag.get(repository_tag=repo_tag).tag assert not tag.hidden assert tag.repository == repo_tag.repository assert tag.name == repo_tag.name, _vs(tag.name, repo_tag.name) assert tag.repository == repo_tag.repository, _vs(tag.repository_id, repo_tag.repository_id) assert tag.reversion == repo_tag.reversion, _vs(tag.reversion, repo_tag.reversion) start_check = int(tag.lifetime_start_ms / 1000) == repo_tag.lifetime_start_ts assert start_check, _vs(tag.lifetime_start_ms, repo_tag.lifetime_start_ts) if repo_tag.lifetime_end_ts is not None: end_check = int(tag.lifetime_end_ms / 1000) == repo_tag.lifetime_end_ts assert end_check, _vs(tag.lifetime_end_ms, repo_tag.lifetime_end_ts) else: assert tag.lifetime_end_ms is None try: tag_manifest = tag.manifest repo_tag_manifest = TagManifest.get(tag=repo_tag) digest_check = tag_manifest.digest == repo_tag_manifest.digest assert digest_check, _vs(tag_manifest.digest, repo_tag_manifest.digest) bytes_check = tag_manifest.manifest_bytes == repo_tag_manifest.json_data assert bytes_check, _vs(tag_manifest.manifest_bytes, repo_tag_manifest.json_data) except TagManifest.DoesNotExist: logger.info('No tag manifest found for repository tag %s', repo_tag.id) mli = ManifestLegacyImage.get(manifest=tag_manifest) assert mli.repository == repo_tag.repository manifest_legacy_image = mli.image assert manifest_legacy_image == repo_tag.image, _vs(manifest_legacy_image.id, repo_tag.image_id)
def compute_layer_id(layer): """ Returns the ID for the layer in the security scanner. """ assert isinstance(layer, ManifestDataType) manifest = Manifest.get(id=layer._db_id) try: layer = ManifestLegacyImage.get(manifest=manifest).image except ManifestLegacyImage.DoesNotExist: return None assert layer.docker_image_id assert layer.storage.uuid return "%s.%s" % (layer.docker_image_id, layer.storage.uuid)
def compute_layer_id(layer): """ Returns the ID for the layer in the security scanner. """ # NOTE: this is temporary until we switch to Clair V3. if isinstance(layer, ManifestDataType): if layer._is_tag_manifest: layer = TagManifest.get(id=layer._db_id).tag.image else: manifest = Manifest.get(id=layer._db_id) try: layer = ManifestLegacyImage.get(manifest=manifest).image except ManifestLegacyImage.DoesNotExist: return None elif isinstance(layer, LegacyImage): layer = Image.get(id=layer._db_id) assert layer.docker_image_id assert layer.storage.uuid return '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
def test_list_alive_tags(initialized_db): found = False for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())): tags = list_alive_tags(tag.repository) assert tag in tags with assert_query_count(1): legacy_images = get_legacy_images_for_tags(tags) for tag in tags: assert ManifestLegacyImage.get(manifest=tag.manifest).image == legacy_images[tag.id] found = True assert found # Ensure hidden tags cannot be listed. tag = Tag.get() tag.hidden = True tag.save() tags = list_alive_tags(tag.repository) assert tag not in tags
def test_manifestbackfillworker_broken_manifest(clear_rows, initialized_db): # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() # Add a broken manifest. broken_manifest = TagManifest.create(json_data='wat?', digest='sha256:foobar', tag=RepositoryTag.get()) # Ensure the backfill works. assert _backfill_manifest(broken_manifest) # Ensure the mapping is marked as broken. map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) assert map_row.broken manifest_row = map_row.manifest assert manifest_row.manifest_bytes == broken_manifest.json_data assert manifest_row.digest == broken_manifest.digest assert manifest_row.repository == broken_manifest.tag.repository legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert broken_manifest.tag.image == legacy_image
def _garbage_collect_manifest(manifest_id, context): assert manifest_id is not None # Make sure the manifest isn't referenced. if _check_manifest_used(manifest_id): return False # Add the manifest's blobs to the context to be GCed. for manifest_blob in ManifestBlob.select().where(ManifestBlob.manifest == manifest_id): context.add_blob_id(manifest_blob.blob_id) # Retrieve the manifest's associated image, if any. try: legacy_image_id = ManifestLegacyImage.get(manifest=manifest_id).image_id context.add_legacy_image_id(legacy_image_id) except ManifestLegacyImage.DoesNotExist: legacy_image_id = None # Add child manifests to be GCed. for connector in ManifestChild.select().where(ManifestChild.manifest == manifest_id): context.add_manifest_id(connector.child_manifest_id) # Add the labels to be GCed. for manifest_label in ManifestLabel.select().where(ManifestLabel.manifest == manifest_id): context.add_label_id(manifest_label.label_id) # Delete the manifest. with db_transaction(): try: manifest = Manifest.select().where(Manifest.id == manifest_id).get() except Manifest.DoesNotExist: return False assert manifest.id == manifest_id assert manifest.repository_id == context.repository.id if _check_manifest_used(manifest_id): return False # Delete any label mappings. (TagManifestLabelMap .delete() .where(TagManifestLabelMap.manifest == manifest_id) .execute()) # Delete any mapping rows for the manifest. TagManifestToManifest.delete().where(TagManifestToManifest.manifest == manifest_id).execute() # Delete any label rows. ManifestLabel.delete().where(ManifestLabel.manifest == manifest_id, ManifestLabel.repository == context.repository).execute() # Delete any child manifest rows. ManifestChild.delete().where(ManifestChild.manifest == manifest_id, ManifestChild.repository == context.repository).execute() # Delete the manifest blobs for the manifest. ManifestBlob.delete().where(ManifestBlob.manifest == manifest_id, ManifestBlob.repository == context.repository).execute() # Delete the manifest legacy image row. if legacy_image_id: (ManifestLegacyImage .delete() .where(ManifestLegacyImage.manifest == manifest_id, ManifestLegacyImage.repository == context.repository) .execute()) # Delete the manifest. manifest.delete_instance() context.mark_manifest_removed(manifest) return True
def _garbage_collect_manifest(manifest_id, context): assert manifest_id is not None # Make sure the manifest isn't referenced. if _check_manifest_used(manifest_id): return False # Add the manifest's blobs to the context to be GCed. for manifest_blob in ManifestBlob.select().where( ManifestBlob.manifest == manifest_id): context.add_blob_id(manifest_blob.blob_id) # Retrieve the manifest's associated image, if any. try: legacy_image_id = ManifestLegacyImage.get( manifest=manifest_id).image_id context.add_legacy_image_id(legacy_image_id) except ManifestLegacyImage.DoesNotExist: legacy_image_id = None # Add child manifests to be GCed. for connector in ManifestChild.select().where( ManifestChild.manifest == manifest_id): context.add_manifest_id(connector.child_manifest_id) # Add the labels to be GCed. for manifest_label in ManifestLabel.select().where( ManifestLabel.manifest == manifest_id): context.add_label_id(manifest_label.label_id) # Delete the manifest. with db_transaction(): try: manifest = Manifest.select().where( Manifest.id == manifest_id).get() except Manifest.DoesNotExist: return False assert manifest.id == manifest_id assert manifest.repository_id == context.repository.id if _check_manifest_used(manifest_id): return False # Delete any label mappings. deleted_tag_manifest_label_map = (TagManifestLabelMap.delete().where( TagManifestLabelMap.manifest == manifest_id).execute()) # Delete any mapping rows for the manifest. deleted_tag_manifest_to_manifest = ( TagManifestToManifest.delete().where( TagManifestToManifest.manifest == manifest_id).execute()) # Delete any label rows. deleted_manifest_label = (ManifestLabel.delete().where( ManifestLabel.manifest == manifest_id, ManifestLabel.repository == context.repository, ).execute()) # Delete any child manifest rows. deleted_manifest_child = (ManifestChild.delete().where( ManifestChild.manifest == manifest_id, ManifestChild.repository == context.repository, ).execute()) # Delete the manifest blobs for the manifest. deleted_manifest_blob = (ManifestBlob.delete().where( ManifestBlob.manifest == manifest_id, ManifestBlob.repository == context.repository).execute()) # Delete the security status for the manifest deleted_manifest_security = (ManifestSecurityStatus.delete().where( ManifestSecurityStatus.manifest == manifest_id, ManifestSecurityStatus.repository == context.repository, ).execute()) # Delete the manifest legacy image row. deleted_manifest_legacy_image = 0 if legacy_image_id: deleted_manifest_legacy_image = ( ManifestLegacyImage.delete().where( ManifestLegacyImage.manifest == manifest_id, ManifestLegacyImage.repository == context.repository, ).execute()) # Delete the manifest. manifest.delete_instance() context.mark_manifest_removed(manifest) gc_table_rows_deleted.labels( table="TagManifestLabelMap").inc(deleted_tag_manifest_label_map) gc_table_rows_deleted.labels( table="TagManifestToManifest").inc(deleted_tag_manifest_to_manifest) gc_table_rows_deleted.labels( table="ManifestLabel").inc(deleted_manifest_label) gc_table_rows_deleted.labels( table="ManifestChild").inc(deleted_manifest_child) gc_table_rows_deleted.labels( table="ManifestBlob").inc(deleted_manifest_blob) gc_table_rows_deleted.labels( table="ManifestSecurityStatus").inc(deleted_manifest_security) if deleted_manifest_legacy_image: gc_table_rows_deleted.labels( table="ManifestLegacyImage").inc(deleted_manifest_legacy_image) gc_table_rows_deleted.labels(table="Manifest").inc() return True
def test_tagbackfillworker(clear_all_rows, initialized_db): # Remove the new-style rows so we can backfill. TagToRepositoryTag.delete().execute() Tag.delete().execute() if clear_all_rows: TagManifestLabelMap.delete().execute() ManifestLabel.delete().execute() ManifestBlob.delete().execute() ManifestLegacyImage.delete().execute() TagManifestToManifest.delete().execute() Manifest.delete().execute() found_dead_tag = False for repository_tag in list(RepositoryTag.select()): # Backfill the tag. assert backfill_tag(repository_tag) # Ensure if we try again, the backfill is skipped. assert not backfill_tag(repository_tag) # Ensure that we now have the expected tag rows. tag_to_repo_tag = TagToRepositoryTag.get(repository_tag=repository_tag) tag = tag_to_repo_tag.tag assert tag.name == repository_tag.name assert tag.repository == repository_tag.repository assert not tag.hidden assert tag.reversion == repository_tag.reversion if repository_tag.lifetime_start_ts is None: assert tag.lifetime_start_ms is None else: assert tag.lifetime_start_ms == (repository_tag.lifetime_start_ts * 1000) if repository_tag.lifetime_end_ts is None: assert tag.lifetime_end_ms is None else: assert tag.lifetime_end_ms == (repository_tag.lifetime_end_ts * 1000) found_dead_tag = True assert tag.manifest # Ensure that we now have the expected manifest rows. try: tag_manifest = TagManifest.get(tag=repository_tag) except TagManifest.DoesNotExist: continue map_row = TagManifestToManifest.get(tag_manifest=tag_manifest) assert not map_row.broken manifest_row = map_row.manifest assert manifest_row.manifest_bytes == tag_manifest.json_data assert manifest_row.digest == tag_manifest.digest assert manifest_row.repository == tag_manifest.tag.repository assert tag.manifest == map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert tag_manifest.tag.image == legacy_image expected_storages = {tag_manifest.tag.image.storage.id} for parent_image_id in tag_manifest.tag.image.ancestor_id_list(): expected_storages.add(Image.get(id=parent_image_id).storage_id) found_storages = { manifest_blob.blob_id for manifest_blob in ManifestBlob.select().where( ManifestBlob.manifest == manifest_row) } assert expected_storages == found_storages # Ensure the labels were copied over. tmls = list(TagManifestLabel.select().where( TagManifestLabel.annotated == tag_manifest)) expected_labels = {tml.label_id for tml in tmls} found_labels = { m.label_id for m in ManifestLabel.select().where( ManifestLabel.manifest == manifest_row) } assert found_labels == expected_labels # Verify at the repository level. for repository in list(Repository.select()): tags = RepositoryTag.select().where( RepositoryTag.repository == repository, RepositoryTag.hidden == False) oci_tags = Tag.select().where(Tag.repository == repository) assert len(tags) == len(oci_tags) assert {t.name for t in tags} == {t.name for t in oci_tags} for tag in tags: tag_manifest = TagManifest.get(tag=tag) ttr = TagToRepositoryTag.get(repository_tag=tag) manifest = ttr.tag.manifest assert tag_manifest.json_data == manifest.manifest_bytes assert tag_manifest.digest == manifest.digest assert tag.image == ManifestLegacyImage.get( manifest=manifest).image assert tag.lifetime_start_ts == (ttr.tag.lifetime_start_ms / 1000) if tag.lifetime_end_ts: assert tag.lifetime_end_ts == (ttr.tag.lifetime_end_ms / 1000) else: assert ttr.tag.lifetime_end_ms is None assert found_dead_tag
def _get_legacy_image(namespace, repo, tag, include_storage=True): repo_ref = registry_model.lookup_repository(namespace, repo) repo_tag = registry_model.get_repo_tag(repo_ref, tag) manifest = registry_model.get_manifest_for_tag(repo_tag) return ManifestLegacyImage.get(manifest_id=manifest._db_id).image