def populate_manifest(repository, manifest, legacy_image, storage_ids): """ Populates the rows for the manifest, including its blobs and legacy image. """ media_type = Manifest.media_type.get_id(manifest.media_type) # Check for an existing manifest. If present, return it. try: return Manifest.get(repository=repository, digest=manifest.digest) except Manifest.DoesNotExist: pass with db_transaction(): try: manifest_row = Manifest.create( digest=manifest.digest, repository=repository, manifest_bytes=manifest.bytes.as_encoded_str(), media_type=media_type, ) except IntegrityError as ie: logger.debug( "Got integrity error when trying to write manifest: %s", ie) return Manifest.get(repository=repository, digest=manifest.digest) ManifestLegacyImage.create(manifest=manifest_row, repository=repository, image=legacy_image) blobs_to_insert = [ dict(manifest=manifest_row, repository=repository, blob=storage_id) for storage_id in storage_ids ] if blobs_to_insert: ManifestBlob.insert_many(blobs_to_insert).execute() return manifest_row
def delete_manifests(): ManifestLegacyImage.delete().execute() ManifestBlob.delete().execute() Manifest.delete().execute() TagManifestToManifest.delete().execute() TagManifest.delete().execute() return "OK"
def _is_storage_orphaned(candidate_id): """ Returns the whether the given candidate storage ID is orphaned. Must be executed under a transaction. """ with ensure_under_transaction(): try: ManifestBlob.get(blob=candidate_id) return False except ManifestBlob.DoesNotExist: pass try: Image.get(storage=candidate_id) return False except Image.DoesNotExist: pass try: UploadedBlob.get(blob=candidate_id) return False except UploadedBlob.DoesNotExist: pass return True
def connect_blobs(manifest: ManifestInterface, blob_ids: set[int], repository_id: int): manifest_blobs = [ dict(manifest=manifest, repository=repository_id, blob=blob_id) for blob_id in blob_ids ] try: ManifestBlob.insert_many(manifest_blobs).execute() except IntegrityError as e: raise _ManifestAlreadyExists(e)
def clear_rows(initialized_db): # Remove all new-style rows so we can backfill. TagToRepositoryTag.delete().execute() Tag.delete().execute() TagManifestLabelMap.delete().execute() ManifestLabel.delete().execute() ManifestBlob.delete().execute() ManifestLegacyImage.delete().execute() TagManifestToManifest.delete().execute() Manifest.delete().execute()
def test_perform_indexing_invalid_manifest(initialized_db, set_secscan_config): secscan = V4SecurityScanner(app, instance_keys, storage) secscan._secscan_api = mock.Mock() # Delete all ManifestBlob rows to cause the manifests to be invalid. ManifestBlob.delete().execute() secscan.perform_indexing() assert ManifestSecurityStatus.select().count() == Manifest.select().count() for mss in ManifestSecurityStatus.select(): assert mss.index_status == IndexStatus.MANIFEST_UNSUPPORTED
def test_store_tag_manifest(get_storages, initialized_db): # Create a manifest with some layers. builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'sometag') storages = get_storages() assert storages repo = model.repository.get_repository('devtable', 'simple') storage_id_map = {} for index, storage in enumerate(storages): image_id = 'someimage%s' % index builder.add_layer(storage.content_checksum, json.dumps({'id': image_id})) find_create_or_link_image(image_id, repo, 'devtable', {}, 'local_us') storage_id_map[storage.content_checksum] = storage.id manifest = builder.build(docker_v2_signing_key) tag_manifest, _ = store_tag_manifest_for_testing('devtable', 'simple', 'sometag', manifest, manifest.leaf_layer_v1_image_id, storage_id_map) # Ensure we have the new-model expected rows. mapping_row = TagManifestToManifest.get(tag_manifest=tag_manifest) assert mapping_row.manifest is not None assert mapping_row.manifest.manifest_bytes == manifest.bytes.as_encoded_str() assert mapping_row.manifest.digest == str(manifest.digest) blob_rows = {m.blob_id for m in ManifestBlob.select().where(ManifestBlob.manifest == mapping_row.manifest)} assert blob_rows == {s.id for s in storages} assert ManifestLegacyImage.get(manifest=mapping_row.manifest).image == tag_manifest.tag.image
def test_manifestbackfillworker_mislinked_invalid_manifest(clear_rows, initialized_db): """ Tests that a manifest whose image is mislinked will attempt to have its storages relinked properly. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository("devtable", "complex") tag_v50 = model.tag.get_active_tag("devtable", "gargantuan", "v5.0") # Add a mislinked manifest, by having its layer point to an invalid blob but its image # be the v5.0 image. builder = DockerSchema1ManifestBuilder("devtable", "gargantuan", "sometag") builder.add_layer("sha256:deadbeef", '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) broken_manifest = TagManifest.create( json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50 ) # Backfill the manifest and ensure it is marked as broken. assert _backfill_manifest(broken_manifest) map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) assert map_row.broken manifest_row = map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert legacy_image == tag_v50.image manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) assert len(manifest_blobs) == 0
def test_create_placeholder_blobs_for_new_manifest(self, create_repo): repo_ref = create_repo(self.orgname, self.upstream_repository, self.user) input_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(UBI8_8_4_MANIFEST_SCHEMA2), DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE, ) proxy_model = ProxyModel( self.orgname, self.upstream_repository, self.user, ) manifest, _ = proxy_model._create_manifest_and_retarget_tag( repo_ref, input_manifest, self.tag) assert manifest is not None blob_count = 1 # schema 2 manifests have one extra config blob blob_count += len(input_manifest.manifest_dict["layers"]) mblobs = ManifestBlob.select().where( ManifestBlob.manifest == manifest.id) assert blob_count == mblobs.count() expected_digests = [ layer["digest"] for layer in input_manifest.manifest_dict["layers"] ] expected_digests.append(input_manifest.config.digest) created_digests = [mblob.blob.content_checksum for mblob in mblobs] assert sorted(expected_digests) == sorted(created_digests)
def backfill_replication(): encountered = set() query = (ManifestBlob.select(ManifestBlob, Repository, User).join( ImageStorage).switch(ManifestBlob).join(Repository).join(User)) for manifest in query: if manifest.blob.uuid in encountered: continue namespace = manifest.repository.namespace_user locations = model.user.get_region_locations(namespace) locations_required = locations | set(storage.default_locations) query = (ImageStoragePlacement.select( ImageStoragePlacement, ImageStorageLocation).where( ImageStoragePlacement.storage == manifest.blob).join( ImageStorageLocation)) existing_locations = set([p.location.name for p in query]) locations_missing = locations_required - existing_locations if locations_missing: print("Enqueueing manifest blob %s to be replicated" % (manifest.blob.uuid)) encountered.add(manifest.blob.uuid) if not image_replication_queue.alive([manifest.blob.uuid]): queue_storage_replication( manifest.repository.namespace_user.username, manifest.blob)
def test_manifestbackfillworker_mislinked_manifest(clear_rows, initialized_db): """ Tests that a manifest whose image is mislinked will have its storages relinked properly. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository('devtable', 'complex') tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0') tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0') # Add a mislinked manifest, by having its layer point to a blob in v3.0 but its image # be the v5.0 image. builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag') builder.add_layer(tag_v30.image.storage.content_checksum, '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) mislinked_manifest = TagManifest.create(json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50) # Backfill the manifest and ensure its proper content checksum was linked. assert _backfill_manifest(mislinked_manifest) map_row = TagManifestToManifest.get(tag_manifest=mislinked_manifest) assert not map_row.broken manifest_row = map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert legacy_image == tag_v50.image manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) assert len(manifest_blobs) == 1 assert manifest_blobs[0].blob.content_checksum == tag_v30.image.storage.content_checksum
def _get_dangling_storage_count(): storage_ids = set([current.id for current in ImageStorage.select()]) referenced_by_image = set([image.storage_id for image in Image.select()]) referenced_by_manifest = set([blob.blob_id for blob in ManifestBlob.select()]) referenced_by_derived = set( [derived.derivative_id for derived in DerivedStorageForImage.select()] ) return len(storage_ids - referenced_by_image - referenced_by_derived - referenced_by_manifest)
def _create_blob(self, digest: str, size: int, manifest_id: int, repo_id: int): try: blob = ImageStorage.get(content_checksum=digest) except ImageStorage.DoesNotExist: # TODO: which size should we really be setting here? blob = ImageStorage.create(content_checksum=digest, image_size=size, compressed_size=size) try: ManifestBlob.get(manifest_id=manifest_id, blob=blob, repository_id=repo_id) except ManifestBlob.DoesNotExist: ManifestBlob.create(manifest_id=manifest_id, blob=blob, repository_id=repo_id) return blob
def purge_repository(repo, force=False): """ Completely delete all traces of the repository. Will return True upon complete success, and False upon partial or total failure. Garbage collection is incremental and repeatable, so this return value does not need to be checked or responded to. """ assert repo.state == RepositoryState.MARKED_FOR_DELETION or force # Delete the repository of all Appr-referenced entries. # Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags, # and MySQL is... particular... about such relationships when deleting. if repo.kind.name == "application": ApprTag.delete().where(ApprTag.repository == repo, ~(ApprTag.linked_tag >> None)).execute() ApprTag.delete().where(ApprTag.repository == repo).execute() else: # GC to remove the images and storage. _purge_repository_contents(repo) # Ensure there are no additional tags, manifests, images or blobs in the repository. assert ApprTag.select().where(ApprTag.repository == repo).count() == 0 assert Tag.select().where(Tag.repository == repo).count() == 0 assert RepositoryTag.select().where( RepositoryTag.repository == repo).count() == 0 assert Manifest.select().where(Manifest.repository == repo).count() == 0 assert ManifestBlob.select().where( ManifestBlob.repository == repo).count() == 0 assert Image.select().where(Image.repository == repo).count() == 0 # Delete any repository build triggers, builds, and any other large-ish reference tables for # the repository. _chunk_delete_all(repo, RepositoryPermission, force=force) _chunk_delete_all(repo, RepositoryBuild, force=force) _chunk_delete_all(repo, RepositoryBuildTrigger, force=force) _chunk_delete_all(repo, RepositoryActionCount, force=force) _chunk_delete_all(repo, Star, force=force) _chunk_delete_all(repo, AccessToken, force=force) _chunk_delete_all(repo, RepositoryNotification, force=force) _chunk_delete_all(repo, BlobUpload, force=force) _chunk_delete_all(repo, RepoMirrorConfig, force=force) _chunk_delete_all(repo, RepositoryAuthorizedEmail, force=force) # Delete any marker rows for the repository. DeletedRepository.delete().where( DeletedRepository.repository == repo).execute() # Delete the rest of the repository metadata. try: # Make sure the repository still exists. fetched = Repository.get(id=repo.id) except Repository.DoesNotExist: return False fetched.delete_instance(recursive=True, delete_nullable=False, force=force) return True
def test_purge_repository_storage_blob(default_tag_policy, initialized_db): with populate_storage_for_gc(): expected_blobs_removed_from_storage = set() preferred = storage.preferred_locations[0] # Check that existing uploadedblobs has an object in storage for repo in database.Repository.select().order_by(database.Repository.id): for uploadedblob in UploadedBlob.select().where(UploadedBlob.repository == repo): assert storage.exists( {preferred}, storage.blob_path(uploadedblob.blob.content_checksum) ) # Remove eveyrhing for repo in database.Repository.select(): # .order_by(database.Repository.id): for uploadedblob in UploadedBlob.select().where(UploadedBlob.repository == repo): # Check if only this repository is referencing the uploadedblob # If so, the blob should be removed from storage has_depedent_manifestblob = ( ManifestBlob.select() .where( ManifestBlob.blob == uploadedblob.blob, ManifestBlob.repository != repo, ) .count() ) has_dependent_image = ( Image.select() .where( Image.storage == uploadedblob.blob, Image.repository != repo, ) .count() ) has_dependent_uploadedblobs = ( UploadedBlob.select() .where( UploadedBlob == uploadedblob, UploadedBlob.repository != repo, ) .count() ) if ( not has_depedent_manifestblob and not has_dependent_image and not has_dependent_uploadedblobs ): expected_blobs_removed_from_storage.add(uploadedblob.blob) assert model.gc.purge_repository(repo, force=True) for removed_blob_from_storage in expected_blobs_removed_from_storage: assert not storage.exists( {preferred}, storage.blob_path(removed_blob_from_storage.content_checksum) )
def purge_repository(namespace_name, repository_name): """ Completely delete all traces of the repository. Will return True upon complete success, and False upon partial or total failure. Garbage collection is incremental and repeatable, so this return value does not need to be checked or responded to. """ try: repo = _basequery.get_existing_repository(namespace_name, repository_name) except Repository.DoesNotExist: return False assert repo.name == repository_name # Delete the repository of all Appr-referenced entries. # Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags, # and MySQL is... particular... about such relationships when deleting. if repo.kind.name == 'application': ApprTag.delete().where(ApprTag.repository == repo, ~(ApprTag.linked_tag >> None)).execute() ApprTag.delete().where(ApprTag.repository == repo).execute() else: # GC to remove the images and storage. _purge_repository_contents(repo) # Ensure there are no additional tags, manifests, images or blobs in the repository. assert ApprTag.select().where(ApprTag.repository == repo).count() == 0 assert Tag.select().where(Tag.repository == repo).count() == 0 assert RepositoryTag.select().where(RepositoryTag.repository == repo).count() == 0 assert Manifest.select().where(Manifest.repository == repo).count() == 0 assert ManifestBlob.select().where(ManifestBlob.repository == repo).count() == 0 assert Image.select().where(Image.repository == repo).count() == 0 # Delete the rest of the repository metadata. try: # Make sure the repository still exists. fetched = _basequery.get_existing_repository(namespace_name, repository_name) except Repository.DoesNotExist: return False fetched.delete_instance(recursive=True, delete_nullable=False) # Run callbacks for callback in config.repo_cleanup_callbacks: callback(namespace_name, repository_name) return True
def test_get_or_create_manifest(schema_version, initialized_db): repository = create_repository('devtable', 'newrepo', None) expected_labels = { 'Foo': 'Bar', 'Baz': 'Meh', } layer_json = json.dumps({ 'id': 'somelegacyid', 'config': { 'Labels': expected_labels, }, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, ], }) # Create a legacy image. find_create_or_link_image('somelegacyid', repository, 'devtable', {}, 'local_us') # Add a blob containing the config. _, config_digest = _populate_blob(layer_json) # Add a blob of random data. random_data = 'hello world' _, random_digest = _populate_blob(random_data) # Build the manifest. if schema_version == 1: builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag') builder.add_layer(random_digest, layer_json) sample_manifest_instance = builder.build(docker_v2_signing_key) elif schema_version == 2: builder = DockerSchema2ManifestBuilder() builder.set_config_digest(config_digest, len(layer_json)) builder.add_layer(random_digest, len(random_data)) sample_manifest_instance = builder.build() # Create a new manifest. created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage) created = created_manifest.manifest newly_created = created_manifest.newly_created assert newly_created assert created is not None assert created.media_type.name == sample_manifest_instance.media_type assert created.digest == sample_manifest_instance.digest assert created.manifest_bytes == sample_manifest_instance.bytes.as_encoded_str( ) assert created_manifest.labels_to_apply == expected_labels # Verify it has a temporary tag pointing to it. assert Tag.get(manifest=created, hidden=True).lifetime_end_ms # Verify the legacy image. legacy_image = get_legacy_image_for_manifest(created) assert legacy_image is not None assert legacy_image.storage.content_checksum == random_digest # Verify the linked blobs. blob_digests = [ mb.blob.content_checksum for mb in ManifestBlob.select().where(ManifestBlob.manifest == created) ] assert random_digest in blob_digests if schema_version == 2: assert config_digest in blob_digests # Retrieve it again and ensure it is the same manifest. created_manifest2 = get_or_create_manifest(repository, sample_manifest_instance, storage) created2 = created_manifest2.manifest newly_created2 = created_manifest2.newly_created assert not newly_created2 assert created2 == created # Ensure it again has a temporary tag. assert Tag.get(manifest=created2, hidden=True).lifetime_end_ms # Ensure the labels were added. labels = list(list_manifest_labels(created)) assert len(labels) == 2 labels_dict = {label.key: label.value for label in labels} assert labels_dict == expected_labels
def test_images_shared_cas(default_tag_policy, initialized_db): """ A repository, each two tags, pointing to the same image, which has image storage with the same *CAS path*, but *distinct records*. Deleting the first tag should delete the first image, and its storage, but not the file in storage, as it shares its CAS path. """ with assert_gc_integrity(expect_storage_removed=True): repository = create_repository() # Create two image storage records with the same content checksum. content = b"hello world" digest = "sha256:" + hashlib.sha256(content).hexdigest() preferred = storage.preferred_locations[0] storage.put_content({preferred}, storage.blob_path(digest), content) is1 = database.ImageStorage.create(content_checksum=digest) is2 = database.ImageStorage.create(content_checksum=digest) location = database.ImageStorageLocation.get(name=preferred) database.ImageStoragePlacement.create(location=location, storage=is1) database.ImageStoragePlacement.create(location=location, storage=is2) # Temp link so its available. model.blob.store_blob_record_and_temp_link_in_repo( repository, digest, location, len(content), 120) # Ensure the CAS path exists. assert storage.exists({preferred}, storage.blob_path(digest)) repo_ref = RepositoryReference.for_repo_obj(repository) # Store a manifest pointing to that path as `first`. builder = DockerSchema1ManifestBuilder( repository.namespace_user.username, repository.name, "first") builder.insert_layer( digest, json.dumps({ "id": "i1", }), ) manifest = builder.build(docker_v2_signing_key) registry_model.create_manifest_and_retarget_tag(repo_ref, manifest, "first", storage, raise_on_error=True) tag_ref = registry_model.get_repo_tag(repo_ref, "first") manifest_ref = registry_model.get_manifest_for_tag(tag_ref) registry_model.populate_legacy_images_for_testing( manifest_ref, storage) # Store another as `second`. builder = DockerSchema1ManifestBuilder( repository.namespace_user.username, repository.name, "second") builder.insert_layer( digest, json.dumps({ "id": "i2", }), ) manifest = builder.build(docker_v2_signing_key) created, _ = registry_model.create_manifest_and_retarget_tag( repo_ref, manifest, "second", storage, raise_on_error=True) tag_ref = registry_model.get_repo_tag(repo_ref, "second") manifest_ref = registry_model.get_manifest_for_tag(tag_ref) registry_model.populate_legacy_images_for_testing( manifest_ref, storage) # Manually retarget the second manifest's blob to the second row. try: second_blob = ManifestBlob.get(manifest=created._db_id, blob=is1) second_blob.blob = is2 second_blob.save() except ManifestBlob.DoesNotExist: second_blob = ManifestBlob.get(manifest=created._db_id, blob=is2) second_blob.blob = is1 second_blob.save() # Delete the temp reference. _delete_temp_links(repository) # Ensure the legacy images exist. assert_not_deleted(repository, "i1", "i2") # Delete the first tag. delete_tag(repository, "first") assert_deleted(repository, "i1") assert_not_deleted(repository, "i2") # Ensure the CAS path still exists. assert storage.exists({preferred}, storage.blob_path(digest))
def _garbage_collect_manifest(manifest_id, context): assert manifest_id is not None # Make sure the manifest isn't referenced. if _check_manifest_used(manifest_id): return False # Add the manifest's blobs to the context to be GCed. for manifest_blob in ManifestBlob.select().where(ManifestBlob.manifest == manifest_id): context.add_blob_id(manifest_blob.blob_id) # Retrieve the manifest's associated image, if any. try: legacy_image_id = ManifestLegacyImage.get(manifest=manifest_id).image_id context.add_legacy_image_id(legacy_image_id) except ManifestLegacyImage.DoesNotExist: legacy_image_id = None # Add child manifests to be GCed. for connector in ManifestChild.select().where(ManifestChild.manifest == manifest_id): context.add_manifest_id(connector.child_manifest_id) # Add the labels to be GCed. for manifest_label in ManifestLabel.select().where(ManifestLabel.manifest == manifest_id): context.add_label_id(manifest_label.label_id) # Delete the manifest. with db_transaction(): try: manifest = Manifest.select().where(Manifest.id == manifest_id).get() except Manifest.DoesNotExist: return False assert manifest.id == manifest_id assert manifest.repository_id == context.repository.id if _check_manifest_used(manifest_id): return False # Delete any label mappings. (TagManifestLabelMap .delete() .where(TagManifestLabelMap.manifest == manifest_id) .execute()) # Delete any mapping rows for the manifest. TagManifestToManifest.delete().where(TagManifestToManifest.manifest == manifest_id).execute() # Delete any label rows. ManifestLabel.delete().where(ManifestLabel.manifest == manifest_id, ManifestLabel.repository == context.repository).execute() # Delete any child manifest rows. ManifestChild.delete().where(ManifestChild.manifest == manifest_id, ManifestChild.repository == context.repository).execute() # Delete the manifest blobs for the manifest. ManifestBlob.delete().where(ManifestBlob.manifest == manifest_id, ManifestBlob.repository == context.repository).execute() # Delete the manifest legacy image row. if legacy_image_id: (ManifestLegacyImage .delete() .where(ManifestLegacyImage.manifest == manifest_id, ManifestLegacyImage.repository == context.repository) .execute()) # Delete the manifest. manifest.delete_instance() context.mark_manifest_removed(manifest) return True
def assert_gc_integrity(expect_storage_removed=True, check_oci_tags=True): """ Specialized assertion for ensuring that GC cleans up all dangling storages and labels, invokes the callback for images removed and doesn't invoke the callback for images *not* removed. """ # Add a callback for when images are removed. removed_image_storages = [] model.config.register_image_cleanup_callback(removed_image_storages.extend) # Store the number of dangling storages and labels. existing_storage_count = _get_dangling_storage_count() existing_label_count = _get_dangling_label_count() existing_manifest_count = _get_dangling_manifest_count() yield # Ensure the number of dangling storages, manifests and labels has not changed. updated_storage_count = _get_dangling_storage_count() assert updated_storage_count == existing_storage_count updated_label_count = _get_dangling_label_count() assert updated_label_count == existing_label_count, _get_dangling_labels() updated_manifest_count = _get_dangling_manifest_count() assert updated_manifest_count == existing_manifest_count # Ensure that for each call to the image+storage cleanup callback, the image and its # storage is not found *anywhere* in the database. for removed_image_and_storage in removed_image_storages: with pytest.raises(Image.DoesNotExist): Image.get(id=removed_image_and_storage.id) # Ensure that image storages are only removed if not shared. shared = Image.select().where( Image.storage == removed_image_and_storage.storage_id).count() if shared == 0: shared = (ManifestBlob.select().where( ManifestBlob.blob == removed_image_and_storage.storage_id).count()) if shared == 0: with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(id=removed_image_and_storage.storage_id) with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(uuid=removed_image_and_storage.storage.uuid) # Ensure all CAS storage is in the storage engine. preferred = storage.preferred_locations[0] for storage_row in ImageStorage.select(): if storage_row.cas_path: storage.get_content({preferred}, storage.blob_path( storage_row.content_checksum)) for blob_row in ApprBlob.select(): storage.get_content({preferred}, storage.blob_path(blob_row.digest)) # Ensure there are no danglings OCI tags. if check_oci_tags: oci_tags = {t.id for t in Tag.select()} referenced_oci_tags = {t.tag_id for t in TagToRepositoryTag.select()} assert not oci_tags - referenced_oci_tags # Ensure all tags have valid manifests. for manifest in {t.manifest for t in Tag.select()}: # Ensure that the manifest's blobs all exist. found_blobs = { b.blob.content_checksum for b in ManifestBlob.select().where( ManifestBlob.manifest == manifest) } parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(manifest.manifest_bytes), manifest.media_type.name) assert set(parsed.local_blob_digests) == found_blobs
def assert_gc_integrity(expect_storage_removed=True): """ Specialized assertion for ensuring that GC cleans up all dangling storages and labels, invokes the callback for images removed and doesn't invoke the callback for images *not* removed. """ # Add a callback for when images are removed. removed_image_storages = [] remove_callback = model.config.register_image_cleanup_callback( removed_image_storages.extend) # Store existing storages. We won't verify these for existence because they # were likely created as test data. existing_digests = set() for storage_row in ImageStorage.select(): if storage_row.cas_path: existing_digests.add(storage_row.content_checksum) for blob_row in ApprBlob.select(): existing_digests.add(blob_row.digest) # Store the number of dangling objects. existing_storage_count = _get_dangling_storage_count() existing_label_count = _get_dangling_label_count() existing_manifest_count = _get_dangling_manifest_count() # Yield to the GC test. with check_transitive_modifications(): try: yield finally: remove_callback() # Ensure the number of dangling storages, manifests and labels has not changed. updated_storage_count = _get_dangling_storage_count() assert updated_storage_count == existing_storage_count updated_label_count = _get_dangling_label_count() assert updated_label_count == existing_label_count, _get_dangling_labels() updated_manifest_count = _get_dangling_manifest_count() assert updated_manifest_count == existing_manifest_count # Ensure that for each call to the image+storage cleanup callback, the image and its # storage is not found *anywhere* in the database. for removed_image_and_storage in removed_image_storages: assert isinstance(removed_image_and_storage, Image) try: # NOTE: SQLite can and will reuse AUTOINCREMENT IDs occasionally, so if we find a row # with the same ID, make sure it does not have the same Docker Image ID. # See: https://www.sqlite.org/autoinc.html found_image = Image.get(id=removed_image_and_storage.id) assert (found_image.docker_image_id != removed_image_and_storage.docker_image_id ), "Found unexpected removed image %s under repo %s" % ( found_image.id, found_image.repository, ) except Image.DoesNotExist: pass # Ensure that image storages are only removed if not shared. shared = Image.select().where( Image.storage == removed_image_and_storage.storage_id).count() if shared == 0: shared = (ManifestBlob.select().where( ManifestBlob.blob == removed_image_and_storage.storage_id).count()) if shared == 0: shared = (UploadedBlob.select().where( UploadedBlob.blob == removed_image_and_storage.storage_id).count()) if shared == 0: with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(id=removed_image_and_storage.storage_id) with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(uuid=removed_image_and_storage.storage.uuid) # Ensure all CAS storage is in the storage engine. preferred = storage.preferred_locations[0] for storage_row in ImageStorage.select(): if storage_row.content_checksum in existing_digests: continue if storage_row.cas_path: storage.get_content({preferred}, storage.blob_path( storage_row.content_checksum)) for blob_row in ApprBlob.select(): if blob_row.digest in existing_digests: continue storage.get_content({preferred}, storage.blob_path(blob_row.digest)) # Ensure all tags have valid manifests. for manifest in {t.manifest for t in Tag.select()}: # Ensure that the manifest's blobs all exist. found_blobs = { b.blob.content_checksum for b in ManifestBlob.select().where( ManifestBlob.manifest == manifest) } parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(manifest.manifest_bytes), manifest.media_type.name) assert set(parsed.local_blob_digests) == found_blobs
def test_get_or_create_manifest(schema_version, initialized_db): repository = create_repository("devtable", "newrepo", None) expected_labels = { "Foo": "Bar", "Baz": "Meh", } layer_json = json.dumps({ "id": "somelegacyid", "config": { "Labels": expected_labels, }, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, ], }) # Create a legacy image. find_create_or_link_image("somelegacyid", repository, "devtable", {}, "local_us") # Add a blob containing the config. _, config_digest = _populate_blob(layer_json) # Add a blob of random data. random_data = "hello world" _, random_digest = _populate_blob(random_data) # Build the manifest. if schema_version == 1: builder = DockerSchema1ManifestBuilder("devtable", "simple", "anothertag") builder.add_layer(random_digest, layer_json) sample_manifest_instance = builder.build(docker_v2_signing_key) elif schema_version == 2: builder = DockerSchema2ManifestBuilder() builder.set_config_digest(config_digest, len(layer_json.encode("utf-8"))) builder.add_layer(random_digest, len(random_data.encode("utf-8"))) sample_manifest_instance = builder.build() assert sample_manifest_instance.layers_compressed_size is not None # Create a new manifest. created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage) created = created_manifest.manifest newly_created = created_manifest.newly_created assert newly_created assert created is not None assert created.media_type.name == sample_manifest_instance.media_type assert created.digest == sample_manifest_instance.digest assert created.manifest_bytes == sample_manifest_instance.bytes.as_encoded_str( ) assert created_manifest.labels_to_apply == expected_labels assert created.config_media_type == sample_manifest_instance.config_media_type assert created.layers_compressed_size == sample_manifest_instance.layers_compressed_size # Lookup the manifest and verify. found = lookup_manifest(repository, created.digest, allow_dead=True) assert found.digest == created.digest assert found.config_media_type == created.config_media_type assert found.layers_compressed_size == created.layers_compressed_size # Verify it has a temporary tag pointing to it. assert Tag.get(manifest=created, hidden=True).lifetime_end_ms # Verify the linked blobs. blob_digests = [ mb.blob.content_checksum for mb in ManifestBlob.select().where(ManifestBlob.manifest == created) ] assert random_digest in blob_digests if schema_version == 2: assert config_digest in blob_digests # Retrieve it again and ensure it is the same manifest. created_manifest2 = get_or_create_manifest(repository, sample_manifest_instance, storage) created2 = created_manifest2.manifest newly_created2 = created_manifest2.newly_created assert not newly_created2 assert created2 == created # Ensure it again has a temporary tag. assert Tag.get(manifest=created2, hidden=True).lifetime_end_ms # Ensure the labels were added. labels = list(list_manifest_labels(created)) assert len(labels) == 2 labels_dict = {label.key: label.value for label in labels} assert labels_dict == expected_labels
def test_create_placeholder_blobs_on_first_pull(self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] # no blob placeholders are created for manifest lists - we don't have # the sub-manifests at manifest list creation time, so there's no way # to know which blobs the sub-manifest has. if test_params["manifest_type"] in [ DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, OCI_IMAGE_INDEX_CONTENT_TYPE, ]: pytest.skip( "manifest list detected - skipping blob placeholder test") repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response(test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"]) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) manifest = Manifest.filter(Manifest.digest == parsed.digest).get() mdict = parsed.manifest_dict layers = mdict.get("layers", mdict.get("fsLayers")) mblobs = ManifestBlob.filter(ManifestBlob.manifest == manifest) expected_count = len(layers) # schema 2 manifests have an extra config blob which we need to take into # consideration in the total count config_digest = "" if parsed.schema_version == 2: config_digest = parsed.config.digest expected_count += 1 assert mblobs.count() == expected_count for mblob in mblobs: blob = None layer = None # don't assert if digest belongs to a config blob if mblob.blob.content_checksum == config_digest: continue for layer in layers: digest = layer.get("digest", layer.get("blobSum")) if mblob.blob.content_checksum == digest: blob = mblob.blob layer = layer break assert blob is not None assert blob.image_size == layer.get("size", None) # the absence of an image storage placement for a blob indicates that it's # a placeholder blob, not yet downloaded from the upstream registry. placements = ImageStoragePlacement.filter( ImageStoragePlacement.storage == blob) assert placements.count() == 0
def _purge_repository_contents(repo): """ Purges all the contents of a repository, removing all of its tags, manifests and images. """ logger.debug("Purging repository %s", repo) # Purge via all the tags. while True: found = False for tags in _chunk_iterate_for_deletion( Tag.select().where(Tag.repository == repo)): logger.debug("Found %s tags to GC under repository %s", len(tags), repo) found = True context = _GarbageCollectorContext(repo) for tag in tags: logger.debug("Deleting tag %s under repository %s", tag, repo) assert tag.repository_id == repo.id _purge_oci_tag(tag, context, allow_non_expired=True) _run_garbage_collection(context) if not found: break # Purge any uploaded blobs that have expired. while True: found = False for uploaded_blobs in _chunk_iterate_for_deletion( UploadedBlob.select().where(UploadedBlob.repository == repo)): logger.debug("Found %s uploaded blobs to GC under repository %s", len(uploaded_blobs), repo) found = True context = _GarbageCollectorContext(repo) for uploaded_blob in uploaded_blobs: logger.debug("Deleting uploaded blob %s under repository %s", uploaded_blob, repo) assert uploaded_blob.repository_id == repo.id _purge_uploaded_blob(uploaded_blob, context, allow_non_expired=True) _run_garbage_collection(context) if not found: break # TODO: remove this once we've removed the foreign key constraints from RepositoryTag # and Image. while True: found = False repo_tag_query = RepositoryTag.select().where( RepositoryTag.repository == repo) for tags in _chunk_iterate_for_deletion(repo_tag_query): logger.debug("Found %s tags to GC under repository %s", len(tags), repo) found = True context = _GarbageCollectorContext(repo) for tag in tags: logger.debug("Deleting tag %s under repository %s", tag, repo) assert tag.repository_id == repo.id _purge_pre_oci_tag(tag, context, allow_non_expired=True) _run_garbage_collection(context) if not found: break assert Tag.select().where(Tag.repository == repo).count() == 0 assert RepositoryTag.select().where( RepositoryTag.repository == repo).count() == 0 assert Manifest.select().where(Manifest.repository == repo).count() == 0 assert ManifestBlob.select().where( ManifestBlob.repository == repo).count() == 0 assert UploadedBlob.select().where( UploadedBlob.repository == repo).count() == 0 # Add all remaining images to a new context. We do this here to minimize the number of images # we need to load. while True: found_image = False image_context = _GarbageCollectorContext(repo) existing_count = Image.select().where(Image.repository == repo).count() if not existing_count: break for image in Image.select().where(Image.repository == repo): found_image = True logger.debug("Trying to delete image %s under repository %s", image, repo) assert image.repository_id == repo.id image_context.add_legacy_image_id(image.id) _run_garbage_collection(image_context) new_count = Image.select().where(Image.repository == repo).count() if new_count >= existing_count: raise Exception("GC purge bug! Please report this to support!")
def _garbage_collect_manifest(manifest_id, context): assert manifest_id is not None # Make sure the manifest isn't referenced. if _check_manifest_used(manifest_id): return False # Add the manifest's blobs to the context to be GCed. for manifest_blob in ManifestBlob.select().where( ManifestBlob.manifest == manifest_id): context.add_blob_id(manifest_blob.blob_id) # Retrieve the manifest's associated image, if any. try: legacy_image_id = ManifestLegacyImage.get( manifest=manifest_id).image_id context.add_legacy_image_id(legacy_image_id) except ManifestLegacyImage.DoesNotExist: legacy_image_id = None # Add child manifests to be GCed. for connector in ManifestChild.select().where( ManifestChild.manifest == manifest_id): context.add_manifest_id(connector.child_manifest_id) # Add the labels to be GCed. for manifest_label in ManifestLabel.select().where( ManifestLabel.manifest == manifest_id): context.add_label_id(manifest_label.label_id) # Delete the manifest. with db_transaction(): try: manifest = Manifest.select().where( Manifest.id == manifest_id).get() except Manifest.DoesNotExist: return False assert manifest.id == manifest_id assert manifest.repository_id == context.repository.id if _check_manifest_used(manifest_id): return False # Delete any label mappings. deleted_tag_manifest_label_map = (TagManifestLabelMap.delete().where( TagManifestLabelMap.manifest == manifest_id).execute()) # Delete any mapping rows for the manifest. deleted_tag_manifest_to_manifest = ( TagManifestToManifest.delete().where( TagManifestToManifest.manifest == manifest_id).execute()) # Delete any label rows. deleted_manifest_label = (ManifestLabel.delete().where( ManifestLabel.manifest == manifest_id, ManifestLabel.repository == context.repository, ).execute()) # Delete any child manifest rows. deleted_manifest_child = (ManifestChild.delete().where( ManifestChild.manifest == manifest_id, ManifestChild.repository == context.repository, ).execute()) # Delete the manifest blobs for the manifest. deleted_manifest_blob = (ManifestBlob.delete().where( ManifestBlob.manifest == manifest_id, ManifestBlob.repository == context.repository).execute()) # Delete the security status for the manifest deleted_manifest_security = (ManifestSecurityStatus.delete().where( ManifestSecurityStatus.manifest == manifest_id, ManifestSecurityStatus.repository == context.repository, ).execute()) # Delete the manifest legacy image row. deleted_manifest_legacy_image = 0 if legacy_image_id: deleted_manifest_legacy_image = ( ManifestLegacyImage.delete().where( ManifestLegacyImage.manifest == manifest_id, ManifestLegacyImage.repository == context.repository, ).execute()) # Delete the manifest. manifest.delete_instance() context.mark_manifest_removed(manifest) gc_table_rows_deleted.labels( table="TagManifestLabelMap").inc(deleted_tag_manifest_label_map) gc_table_rows_deleted.labels( table="TagManifestToManifest").inc(deleted_tag_manifest_to_manifest) gc_table_rows_deleted.labels( table="ManifestLabel").inc(deleted_manifest_label) gc_table_rows_deleted.labels( table="ManifestChild").inc(deleted_manifest_child) gc_table_rows_deleted.labels( table="ManifestBlob").inc(deleted_manifest_blob) gc_table_rows_deleted.labels( table="ManifestSecurityStatus").inc(deleted_manifest_security) if deleted_manifest_legacy_image: gc_table_rows_deleted.labels( table="ManifestLegacyImage").inc(deleted_manifest_legacy_image) gc_table_rows_deleted.labels(table="Manifest").inc() return True
def _create_manifest( repository_id, manifest_interface_instance, storage, temp_tag_expiration_sec=TEMP_TAG_EXPIRATION_SEC, for_tagging=False, raise_on_error=False, retriever=None, ): # Validate the manifest. retriever = retriever or RepositoryContentRetriever.for_repository( repository_id, storage) try: manifest_interface_instance.validate(retriever) except (ManifestException, MalformedSchema2ManifestList, BlobDoesNotExist, IOError) as ex: logger.exception("Could not validate manifest `%s`", manifest_interface_instance.digest) if raise_on_error: raise CreateManifestException(str(ex)) return None # Load, parse and get/create the child manifests, if any. child_manifest_refs = manifest_interface_instance.child_manifests( retriever) child_manifest_rows = {} child_manifest_label_dicts = [] if child_manifest_refs is not None: for child_manifest_ref in child_manifest_refs: # Load and parse the child manifest. try: child_manifest = child_manifest_ref.manifest_obj except ( ManifestException, MalformedSchema2ManifestList, BlobDoesNotExist, IOError, ) as ex: logger.exception( "Could not load manifest list for manifest `%s`", manifest_interface_instance.digest, ) if raise_on_error: raise CreateManifestException(str(ex)) return None # Retrieve its labels. labels = child_manifest.get_manifest_labels(retriever) if labels is None: if raise_on_error: raise CreateManifestException( "Unable to retrieve manifest labels") logger.exception( "Could not load manifest labels for child manifest") return None # Get/create the child manifest in the database. child_manifest_info = get_or_create_manifest( repository_id, child_manifest, storage, raise_on_error=raise_on_error) if child_manifest_info is None: if raise_on_error: raise CreateManifestException( "Unable to retrieve child manifest") logger.error("Could not get/create child manifest") return None child_manifest_rows[child_manifest_info.manifest. digest] = child_manifest_info.manifest child_manifest_label_dicts.append(labels) # Build the map from required blob digests to the blob objects. blob_map = _build_blob_map( repository_id, manifest_interface_instance, retriever, storage, raise_on_error, require_empty_layer=False, ) if blob_map is None: return None # Create the manifest and its blobs. media_type = Manifest.media_type.get_id( manifest_interface_instance.media_type) storage_ids = {storage.id for storage in list(blob_map.values())} # Check for the manifest, in case it was created since we checked earlier. try: manifest = Manifest.get(repository=repository_id, digest=manifest_interface_instance.digest) return CreatedManifest(manifest=manifest, newly_created=False, labels_to_apply=None) except Manifest.DoesNotExist: pass try: with db_transaction(): # Create the manifest. try: manifest = Manifest.create( repository=repository_id, digest=manifest_interface_instance.digest, media_type=media_type, manifest_bytes=manifest_interface_instance.bytes. as_encoded_str(), config_media_type=manifest_interface_instance. config_media_type, layers_compressed_size=manifest_interface_instance. layers_compressed_size, ) except IntegrityError as ie: # NOTE: An IntegrityError means (barring a bug) that the manifest was created by # another caller while we were attempting to create it. Since we need to return # the manifest, we raise a specialized exception here to break out of the # transaction so we can retrieve it. raise _ManifestAlreadyExists(ie) # Insert the blobs. blobs_to_insert = [ dict(manifest=manifest, repository=repository_id, blob=storage_id) for storage_id in storage_ids ] if blobs_to_insert: try: ManifestBlob.insert_many(blobs_to_insert).execute() except IntegrityError as ie: raise _ManifestAlreadyExists(ie) # Insert the manifest child rows (if applicable). if child_manifest_rows: children_to_insert = [ dict(manifest=manifest, child_manifest=child_manifest, repository=repository_id) for child_manifest in list(child_manifest_rows.values()) ] try: ManifestChild.insert_many(children_to_insert).execute() except IntegrityError as ie: raise _ManifestAlreadyExists(ie) # If this manifest is being created not for immediate tagging, add a temporary tag to the # manifest to ensure it isn't being GCed. If the manifest *is* for tagging, then since we're # creating a new one here, it cannot be GCed (since it isn't referenced by anything yet), so # its safe to elide the temp tag operation. If we ever change GC code to collect *all* manifests # in a repository for GC, then we will have to reevaluate this optimization at that time. if not for_tagging: create_temporary_tag_if_necessary(manifest, temp_tag_expiration_sec) # Define the labels for the manifest (if any). # TODO: Once the old data model is gone, turn this into a batch operation and make the label # application to the manifest occur under the transaction. labels = manifest_interface_instance.get_manifest_labels(retriever) if labels: for key, value in labels.items(): # NOTE: There can technically be empty label keys via Dockerfile's. We ignore any # such `labels`, as they don't really mean anything. if not key: continue media_type = "application/json" if is_json( value) else "text/plain" create_manifest_label(manifest, key, value, "manifest", media_type) # Return the dictionary of labels to apply (i.e. those labels that cause an action to be taken # on the manifest or its resulting tags). We only return those labels either defined on # the manifest or shared amongst all the child manifests. We intersect amongst all child manifests # to ensure that any action performed is defined in all manifests. labels_to_apply = labels or {} if child_manifest_label_dicts: labels_to_apply = child_manifest_label_dicts[0].items() for child_manifest_label_dict in child_manifest_label_dicts[1:]: # Intersect the key+values of the labels to ensure we get the exact same result # for all the child manifests. labels_to_apply = labels_to_apply & child_manifest_label_dict.items( ) labels_to_apply = dict(labels_to_apply) return CreatedManifest(manifest=manifest, newly_created=True, labels_to_apply=labels_to_apply) except _ManifestAlreadyExists as mae: try: manifest = Manifest.get(repository=repository_id, digest=manifest_interface_instance.digest) except Manifest.DoesNotExist: # NOTE: If we've reached this point, then somehow we had an IntegrityError without it # being due to a duplicate manifest. We therefore log the error. logger.error( "Got integrity error when trying to create manifest: %s", mae.internal_exception) if raise_on_error: raise CreateManifestException( "Attempt to create an invalid manifest. Please report this issue." ) return None return CreatedManifest(manifest=manifest, newly_created=False, labels_to_apply=None)
def test_get_or_create_manifest_with_remote_layers(initialized_db): repository = create_repository("devtable", "newrepo", None) layer_json = json.dumps({ "config": {}, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, ], }) # Add a blob containing the config. _, config_digest = _populate_blob(layer_json) # Add a blob of random data. random_data = "hello world" _, random_digest = _populate_blob(random_data) remote_digest = sha256_digest(b"something") builder = DockerSchema2ManifestBuilder() builder.set_config_digest(config_digest, len(layer_json.encode("utf-8"))) builder.add_layer(remote_digest, 1234, urls=["http://hello/world"]) builder.add_layer(random_digest, len(random_data.encode("utf-8"))) manifest = builder.build() assert remote_digest in manifest.blob_digests assert remote_digest not in manifest.local_blob_digests assert manifest.has_remote_layer assert not manifest.has_legacy_image assert manifest.get_schema1_manifest("foo", "bar", "baz", None) is None # Write the manifest. created_tuple = get_or_create_manifest(repository, manifest, storage) assert created_tuple is not None created_manifest = created_tuple.manifest assert created_manifest assert created_manifest.media_type.name == manifest.media_type assert created_manifest.digest == manifest.digest assert created_manifest.config_media_type == manifest.config_media_type assert created_manifest.layers_compressed_size == manifest.layers_compressed_size # Verify the legacy image. legacy_image = get_legacy_image_for_manifest(created_manifest) assert legacy_image is None # Verify the linked blobs. blob_digests = { mb.blob.content_checksum for mb in ManifestBlob.select().where( ManifestBlob.manifest == created_manifest) } assert random_digest in blob_digests assert config_digest in blob_digests assert remote_digest not in blob_digests
def test_retriever(initialized_db): repository = create_repository("devtable", "newrepo", None) layer_json = json.dumps({ "config": {}, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, ], }) # Add a blob containing the config. _, config_digest = _populate_blob(layer_json) # Add a blob of random data. random_data = "hello world" _, random_digest = _populate_blob(random_data) # Add another blob of random data. other_random_data = "hi place" _, other_random_digest = _populate_blob(other_random_data) remote_digest = sha256_digest(b"something") builder = DockerSchema2ManifestBuilder() builder.set_config_digest(config_digest, len(layer_json.encode("utf-8"))) builder.add_layer(other_random_digest, len(other_random_data.encode("utf-8"))) builder.add_layer(random_digest, len(random_data.encode("utf-8"))) manifest = builder.build() assert config_digest in manifest.blob_digests assert random_digest in manifest.blob_digests assert other_random_digest in manifest.blob_digests assert config_digest in manifest.local_blob_digests assert random_digest in manifest.local_blob_digests assert other_random_digest in manifest.local_blob_digests # Write the manifest. created_tuple = get_or_create_manifest(repository, manifest, storage) assert created_tuple is not None created_manifest = created_tuple.manifest assert created_manifest assert created_manifest.media_type.name == manifest.media_type assert created_manifest.digest == manifest.digest # Verify the linked blobs. blob_digests = { mb.blob.content_checksum for mb in ManifestBlob.select().where( ManifestBlob.manifest == created_manifest) } assert random_digest in blob_digests assert other_random_digest in blob_digests assert config_digest in blob_digests # Delete any Image rows linking to the blobs from temp tags. for blob_digest in blob_digests: storage_row = ImageStorage.get(content_checksum=blob_digest) for image in list(Image.select().where(Image.storage == storage_row)): all_temp = all([ rt.hidden for rt in RepositoryTag.select().where( RepositoryTag.image == image) ]) if all_temp: RepositoryTag.delete().where( RepositoryTag.image == image).execute() image.delete_instance(recursive=True) # Verify the blobs in the retriever. retriever = RepositoryContentRetriever(repository, storage) assert (retriever.get_manifest_bytes_with_digest( created_manifest.digest) == manifest.bytes.as_encoded_str()) for blob_digest in blob_digests: assert retriever.get_blob_bytes_with_digest(blob_digest) is not None
def _create_manifest( repository_id, manifest_interface_instance, storage, temp_tag_expiration_sec=TEMP_TAG_EXPIRATION_SEC, for_tagging=False, raise_on_error=False, retriever=None, ): # Validate the manifest. retriever = retriever or RepositoryContentRetriever.for_repository( repository_id, storage) try: manifest_interface_instance.validate(retriever) except (ManifestException, MalformedSchema2ManifestList, BlobDoesNotExist, IOError) as ex: logger.exception("Could not validate manifest `%s`", manifest_interface_instance.digest) if raise_on_error: raise CreateManifestException(str(ex)) return None # Load, parse and get/create the child manifests, if any. child_manifest_refs = manifest_interface_instance.child_manifests( retriever) child_manifest_rows = {} child_manifest_label_dicts = [] if child_manifest_refs is not None: for child_manifest_ref in child_manifest_refs: # Load and parse the child manifest. try: child_manifest = child_manifest_ref.manifest_obj except ( ManifestException, MalformedSchema2ManifestList, BlobDoesNotExist, IOError, ) as ex: logger.exception( "Could not load manifest list for manifest `%s`", manifest_interface_instance.digest, ) if raise_on_error: raise CreateManifestException(str(ex)) return None # Retrieve its labels. labels = child_manifest.get_manifest_labels(retriever) if labels is None: if raise_on_error: raise CreateManifestException( "Unable to retrieve manifest labels") logger.exception( "Could not load manifest labels for child manifest") return None # Get/create the child manifest in the database. child_manifest_info = get_or_create_manifest( repository_id, child_manifest, storage, raise_on_error=raise_on_error) if child_manifest_info is None: if raise_on_error: raise CreateManifestException( "Unable to retrieve child manifest") logger.error("Could not get/create child manifest") return None child_manifest_rows[child_manifest_info.manifest. digest] = child_manifest_info.manifest child_manifest_label_dicts.append(labels) # Ensure all the blobs in the manifest exist. digests = set(manifest_interface_instance.local_blob_digests) blob_map = {} # If the special empty layer is required, simply load it directly. This is much faster # than trying to load it on a per repository basis, and that is unnecessary anyway since # this layer is predefined. if EMPTY_LAYER_BLOB_DIGEST in digests: digests.remove(EMPTY_LAYER_BLOB_DIGEST) blob_map[EMPTY_LAYER_BLOB_DIGEST] = get_shared_blob( EMPTY_LAYER_BLOB_DIGEST) if not blob_map[EMPTY_LAYER_BLOB_DIGEST]: if raise_on_error: raise CreateManifestException( "Unable to retrieve specialized empty blob") logger.warning("Could not find the special empty blob in storage") return None if digests: query = lookup_repo_storages_by_content_checksum( repository_id, digests) blob_map.update({s.content_checksum: s for s in query}) for digest_str in digests: if digest_str not in blob_map: logger.warning( "Unknown blob `%s` under manifest `%s` for repository `%s`", digest_str, manifest_interface_instance.digest, repository_id, ) if raise_on_error: raise CreateManifestException("Unknown blob `%s`" % digest_str) return None # Special check: If the empty layer blob is needed for this manifest, add it to the # blob map. This is necessary because Docker decided to elide sending of this special # empty layer in schema version 2, but we need to have it referenced for GC and schema version 1. if EMPTY_LAYER_BLOB_DIGEST not in blob_map: try: requires_empty_layer = manifest_interface_instance.get_requires_empty_layer_blob( retriever) except ManifestException as ex: if raise_on_error: raise CreateManifestException(str(ex)) return None if requires_empty_layer is None: if raise_on_error: raise CreateManifestException( "Could not load configuration blob") return None if requires_empty_layer: shared_blob = get_or_create_shared_blob(EMPTY_LAYER_BLOB_DIGEST, EMPTY_LAYER_BYTES, storage) assert not shared_blob.uploading assert shared_blob.content_checksum == EMPTY_LAYER_BLOB_DIGEST blob_map[EMPTY_LAYER_BLOB_DIGEST] = shared_blob # Determine and populate the legacy image if necessary. Manifest lists will not have a legacy # image. legacy_image = None if manifest_interface_instance.has_legacy_image: legacy_image_id = _populate_legacy_image(repository_id, manifest_interface_instance, blob_map, retriever, raise_on_error) if legacy_image_id is None: return None legacy_image = get_image(repository_id, legacy_image_id) if legacy_image is None: return None # Create the manifest and its blobs. media_type = Manifest.media_type.get_id( manifest_interface_instance.media_type) storage_ids = {storage.id for storage in blob_map.values()} with db_transaction(): # Check for the manifest. This is necessary because Postgres doesn't handle IntegrityErrors # well under transactions. try: manifest = Manifest.get(repository=repository_id, digest=manifest_interface_instance.digest) return CreatedManifest(manifest=manifest, newly_created=False, labels_to_apply=None) except Manifest.DoesNotExist: pass # Create the manifest. try: manifest = Manifest.create( repository=repository_id, digest=manifest_interface_instance.digest, media_type=media_type, manifest_bytes=manifest_interface_instance.bytes. as_encoded_str(), ) except IntegrityError as ie: try: manifest = Manifest.get( repository=repository_id, digest=manifest_interface_instance.digest) except Manifest.DoesNotExist: logger.error( "Got integrity error when trying to create manifest: %s", ie) if raise_on_error: raise CreateManifestException( "Attempt to create an invalid manifest. Please report this issue." ) return None return CreatedManifest(manifest=manifest, newly_created=False, labels_to_apply=None) # Insert the blobs. blobs_to_insert = [ dict(manifest=manifest, repository=repository_id, blob=storage_id) for storage_id in storage_ids ] if blobs_to_insert: ManifestBlob.insert_many(blobs_to_insert).execute() # Set the legacy image (if applicable). if legacy_image is not None: ManifestLegacyImage.create(repository=repository_id, image=legacy_image, manifest=manifest) # Insert the manifest child rows (if applicable). if child_manifest_rows: children_to_insert = [ dict(manifest=manifest, child_manifest=child_manifest, repository=repository_id) for child_manifest in child_manifest_rows.values() ] ManifestChild.insert_many(children_to_insert).execute() # If this manifest is being created not for immediate tagging, add a temporary tag to the # manifest to ensure it isn't being GCed. If the manifest *is* for tagging, then since we're # creating a new one here, it cannot be GCed (since it isn't referenced by anything yet), so # its safe to elide the temp tag operation. If we ever change GC code to collect *all* manifests # in a repository for GC, then we will have to reevaluate this optimization at that time. if not for_tagging: create_temporary_tag_if_necessary(manifest, temp_tag_expiration_sec) # Define the labels for the manifest (if any). # TODO: Once the old data model is gone, turn this into a batch operation and make the label # application to the manifest occur under the transaction. labels = manifest_interface_instance.get_manifest_labels(retriever) if labels: for key, value in labels.iteritems(): # NOTE: There can technically be empty label keys via Dockerfile's. We ignore any # such `labels`, as they don't really mean anything. if not key: continue media_type = "application/json" if is_json(value) else "text/plain" create_manifest_label(manifest, key, value, "manifest", media_type) # Return the dictionary of labels to apply (i.e. those labels that cause an action to be taken # on the manifest or its resulting tags). We only return those labels either defined on # the manifest or shared amongst all the child manifests. We intersect amongst all child manifests # to ensure that any action performed is defined in all manifests. labels_to_apply = labels or {} if child_manifest_label_dicts: labels_to_apply = child_manifest_label_dicts[0].viewitems() for child_manifest_label_dict in child_manifest_label_dicts[1:]: # Intersect the key+values of the labels to ensure we get the exact same result # for all the child manifests. labels_to_apply = labels_to_apply & child_manifest_label_dict.viewitems( ) labels_to_apply = dict(labels_to_apply) return CreatedManifest(manifest=manifest, newly_created=True, labels_to_apply=labels_to_apply)
def test_tagbackfillworker(clear_all_rows, initialized_db): # Remove the new-style rows so we can backfill. TagToRepositoryTag.delete().execute() Tag.delete().execute() if clear_all_rows: TagManifestLabelMap.delete().execute() ManifestLabel.delete().execute() ManifestBlob.delete().execute() ManifestLegacyImage.delete().execute() TagManifestToManifest.delete().execute() Manifest.delete().execute() found_dead_tag = False for repository_tag in list(RepositoryTag.select()): # Backfill the tag. assert backfill_tag(repository_tag) # Ensure if we try again, the backfill is skipped. assert not backfill_tag(repository_tag) # Ensure that we now have the expected tag rows. tag_to_repo_tag = TagToRepositoryTag.get(repository_tag=repository_tag) tag = tag_to_repo_tag.tag assert tag.name == repository_tag.name assert tag.repository == repository_tag.repository assert not tag.hidden assert tag.reversion == repository_tag.reversion if repository_tag.lifetime_start_ts is None: assert tag.lifetime_start_ms is None else: assert tag.lifetime_start_ms == (repository_tag.lifetime_start_ts * 1000) if repository_tag.lifetime_end_ts is None: assert tag.lifetime_end_ms is None else: assert tag.lifetime_end_ms == (repository_tag.lifetime_end_ts * 1000) found_dead_tag = True assert tag.manifest # Ensure that we now have the expected manifest rows. try: tag_manifest = TagManifest.get(tag=repository_tag) except TagManifest.DoesNotExist: continue map_row = TagManifestToManifest.get(tag_manifest=tag_manifest) assert not map_row.broken manifest_row = map_row.manifest assert manifest_row.manifest_bytes == tag_manifest.json_data assert manifest_row.digest == tag_manifest.digest assert manifest_row.repository == tag_manifest.tag.repository assert tag.manifest == map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert tag_manifest.tag.image == legacy_image expected_storages = {tag_manifest.tag.image.storage.id} for parent_image_id in tag_manifest.tag.image.ancestor_id_list(): expected_storages.add(Image.get(id=parent_image_id).storage_id) found_storages = { manifest_blob.blob_id for manifest_blob in ManifestBlob.select().where( ManifestBlob.manifest == manifest_row) } assert expected_storages == found_storages # Ensure the labels were copied over. tmls = list(TagManifestLabel.select().where( TagManifestLabel.annotated == tag_manifest)) expected_labels = {tml.label_id for tml in tmls} found_labels = { m.label_id for m in ManifestLabel.select().where( ManifestLabel.manifest == manifest_row) } assert found_labels == expected_labels # Verify at the repository level. for repository in list(Repository.select()): tags = RepositoryTag.select().where( RepositoryTag.repository == repository, RepositoryTag.hidden == False) oci_tags = Tag.select().where(Tag.repository == repository) assert len(tags) == len(oci_tags) assert {t.name for t in tags} == {t.name for t in oci_tags} for tag in tags: tag_manifest = TagManifest.get(tag=tag) ttr = TagToRepositoryTag.get(repository_tag=tag) manifest = ttr.tag.manifest assert tag_manifest.json_data == manifest.manifest_bytes assert tag_manifest.digest == manifest.digest assert tag.image == ManifestLegacyImage.get( manifest=manifest).image assert tag.lifetime_start_ts == (ttr.tag.lifetime_start_ms / 1000) if tag.lifetime_end_ts: assert tag.lifetime_end_ts == (ttr.tag.lifetime_end_ms / 1000) else: assert ttr.tag.lifetime_end_ms is None assert found_dead_tag