def _garbage_collect_legacy_image(legacy_image_id, context): assert legacy_image_id is not None # Check if the image is referenced. if _check_image_used(legacy_image_id): return False # We have an unreferenced image. We can now delete it. # Grab any derived storage for the image. for derived in DerivedStorageForImage.select().where( DerivedStorageForImage.source_image == legacy_image_id): context.add_blob_id(derived.derivative_id) try: image = Image.select().where(Image.id == legacy_image_id).get() except Image.DoesNotExist: return False assert image.repository_id == context.repository.id # Add the image's blob to be GCed. context.add_blob_id(image.storage_id) # If the image has a parent ID, add the parent for GC. if image.parent_id is not None: context.add_legacy_image_id(image.parent_id) # Delete the image. with db_transaction(): if _check_image_used(legacy_image_id): return False try: image = Image.select().where(Image.id == legacy_image_id).get() except Image.DoesNotExist: return False assert image.id == legacy_image_id assert image.repository_id == context.repository.id # Delete any derived storage for the image. deleted_derived_storage = (DerivedStorageForImage.delete().where( DerivedStorageForImage.source_image == legacy_image_id).execute()) # Delete the image itself. image.delete_instance() context.mark_legacy_image_removed(image) gc_table_rows_deleted.labels(table="Image").inc() gc_table_rows_deleted.labels( table="DerivedStorageForImage").inc(deleted_derived_storage) if config.image_cleanup_callbacks: for callback in config.image_cleanup_callbacks: callback([image]) return True
def test_derived_image(registry_model): # Clear all existing derived storage. DerivedStorageForImage.delete().execute() repository_ref = registry_model.lookup_repository('devtable', 'simple') tag = registry_model.get_repo_tag(repository_ref, 'latest') manifest = registry_model.get_manifest_for_tag(tag) # Ensure the squashed image doesn't exist. assert registry_model.lookup_derived_image(manifest, 'squash', storage, {}) is None # Create a new one. squashed = registry_model.lookup_or_create_derived_image( manifest, 'squash', 'local_us', storage, {}) assert registry_model.lookup_or_create_derived_image( manifest, 'squash', 'local_us', storage, {}) == squashed assert squashed.unique_id # Check and set the size. assert squashed.blob.compressed_size is None registry_model.set_derived_image_size(squashed, 1234) found = registry_model.lookup_derived_image(manifest, 'squash', storage, {}) assert found.blob.compressed_size == 1234 assert found.unique_id == squashed.unique_id # Ensure its returned now. assert found == squashed # Ensure different metadata results in a different derived image. found = registry_model.lookup_derived_image(manifest, 'squash', storage, {'foo': 'bar'}) assert found is None squashed_foo = registry_model.lookup_or_create_derived_image( manifest, 'squash', 'local_us', storage, {'foo': 'bar'}) assert squashed_foo != squashed found = registry_model.lookup_derived_image(manifest, 'squash', storage, {'foo': 'bar'}) assert found == squashed_foo assert squashed.unique_id != squashed_foo.unique_id # Lookup with placements. squashed = registry_model.lookup_or_create_derived_image( manifest, 'squash', 'local_us', storage, {}, include_placements=True) assert squashed.blob.placements # Delete the derived image. registry_model.delete_derived_image(squashed) assert registry_model.lookup_derived_image(manifest, 'squash', storage, {}) is None
def find_or_create_derived_storage(source_image, transformation_name, preferred_location, varying_metadata=None): existing = find_derived_storage_for_image(source_image, transformation_name, varying_metadata) if existing is not None: return existing uniqueness_hash = _get_uniqueness_hash(varying_metadata) trans = ImageStorageTransformation.get(name=transformation_name) new_storage = storage.create_v1_storage(preferred_location) try: derived = DerivedStorageForImage.create( source_image=source_image, derivative=new_storage, transformation=trans, uniqueness_hash=uniqueness_hash, ) except IntegrityError: # Storage was created while this method executed. Just return the existing. ImageStoragePlacement.delete().where( ImageStoragePlacement.storage == new_storage).execute() new_storage.delete_instance() return find_derived_storage_for_image(source_image, transformation_name, varying_metadata) return derived
def _get_dangling_storage_count(): storage_ids = set([current.id for current in ImageStorage.select()]) referenced_by_image = set([image.storage_id for image in Image.select()]) referenced_by_manifest = set([blob.blob_id for blob in ManifestBlob.select()]) referenced_by_derived = set( [derived.derivative_id for derived in DerivedStorageForImage.select()] ) return len(storage_ids - referenced_by_image - referenced_by_derived - referenced_by_manifest)
def find_derived_storage_for_image(source_image, transformation_name, varying_metadata=None): uniqueness_hash = _get_uniqueness_hash(varying_metadata) try: found = (DerivedStorageForImage.select( ImageStorage, DerivedStorageForImage).join(ImageStorage).switch( DerivedStorageForImage).join(ImageStorageTransformation).where( DerivedStorageForImage.source_image == source_image, ImageStorageTransformation.name == transformation_name, DerivedStorageForImage.uniqueness_hash == uniqueness_hash, ).get()) return found except DerivedStorageForImage.DoesNotExist: return None
def _namespace_from_kwargs(self, args_dict): if "namespace_name" in args_dict: return args_dict["namespace_name"] if "repository_ref" in args_dict: return args_dict["repository_ref"].namespace_name if "tag" in args_dict: return args_dict["tag"].repository.namespace_name if "manifest" in args_dict: manifest = args_dict["manifest"] if manifest._is_tag_manifest: return TagManifest.get( id=manifest._db_id).tag.repository.namespace_user.username else: return Manifest.get( id=manifest._db_id).repository.namespace_user.username if "manifest_or_legacy_image" in args_dict: manifest_or_legacy_image = args_dict["manifest_or_legacy_image"] if isinstance(manifest_or_legacy_image, LegacyImage): return Image.get(id=manifest_or_legacy_image._db_id ).repository.namespace_user.username else: manifest = manifest_or_legacy_image if manifest._is_tag_manifest: return TagManifest.get( id=manifest._db_id ).tag.repository.namespace_user.username else: return Manifest.get( id=manifest._db_id).repository.namespace_user.username if "derived_image" in args_dict: return DerivedStorageForImage.get( id=args_dict["derived_image"]._db_id ).source_image.repository.namespace_user.username if "blob" in args_dict: return "" # Blob functions are shared, so no need to do anything. if "blob_upload" in args_dict: return "" # Blob functions are shared, so no need to do anything. raise Exception("Unknown namespace for dict `%s`" % args_dict)
def test_derived_image_for_manifest_list(oci_model): # Clear all existing derived storage. DerivedStorageForImage.delete().execute() # Create a config blob for testing. config_json = json.dumps({ 'config': {}, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, ], }) app_config = {'TESTING': True} repository_ref = oci_model.lookup_repository('devtable', 'simple') with upload_blob(repository_ref, storage, BlobUploadSettings(500, 500, 500)) as upload: upload.upload_chunk(app_config, BytesIO(config_json)) blob = upload.commit_to_blob(app_config) # Create the manifest in the repo. builder = DockerSchema2ManifestBuilder() builder.set_config_digest(blob.digest, blob.compressed_size) builder.add_layer(blob.digest, blob.compressed_size) amd64_manifest = builder.build() oci_model.create_manifest_and_retarget_tag(repository_ref, amd64_manifest, 'submanifest', storage) # Create a manifest list, pointing to at least one amd64+linux manifest. builder = DockerSchema2ManifestListBuilder() builder.add_manifest(amd64_manifest, 'amd64', 'linux') manifestlist = builder.build() oci_model.create_manifest_and_retarget_tag(repository_ref, manifestlist, 'listtag', storage) manifest = oci_model.get_manifest_for_tag( oci_model.get_repo_tag(repository_ref, 'listtag')) assert manifest assert manifest.get_parsed_manifest().is_manifest_list # Ensure the squashed image doesn't exist. assert oci_model.lookup_derived_image(manifest, 'squash', storage, {}) is None # Create a new one. squashed = oci_model.lookup_or_create_derived_image( manifest, 'squash', 'local_us', storage, {}) assert squashed.unique_id assert oci_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', storage, {}) == squashed # Perform lookup. assert oci_model.lookup_derived_image(manifest, 'squash', storage, {}) == squashed
def clear_derived_cache(): DerivedStorageForImage.delete().execute() return "OK"
def test_derived_image_for_manifest_list(manifest_builder, list_builder, oci_model): # Clear all existing derived storage. DerivedStorageForImage.delete().execute() # Create a config blob for testing. config_json = json.dumps({ "config": {}, "architecture": "amd64", "os": "linux", "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, ], }) app_config = {"TESTING": True} repository_ref = oci_model.lookup_repository("devtable", "simple") with upload_blob(repository_ref, storage, BlobUploadSettings(500, 500)) as upload: upload.upload_chunk(app_config, BytesIO(config_json)) blob = upload.commit_to_blob(app_config) # Create the manifest in the repo. builder = manifest_builder() builder.set_config_digest(blob.digest, blob.compressed_size) builder.add_layer(blob.digest, blob.compressed_size) amd64_manifest = builder.build() oci_model.create_manifest_and_retarget_tag(repository_ref, amd64_manifest, "submanifest", storage, raise_on_error=True) # Create a manifest list, pointing to at least one amd64+linux manifest. builder = list_builder() builder.add_manifest(amd64_manifest, "amd64", "linux") manifestlist = builder.build() oci_model.create_manifest_and_retarget_tag(repository_ref, manifestlist, "listtag", storage, raise_on_error=True) manifest = oci_model.get_manifest_for_tag( oci_model.get_repo_tag(repository_ref, "listtag")) assert manifest assert manifest.get_parsed_manifest().is_manifest_list # Ensure the squashed image doesn't exist. assert oci_model.lookup_derived_image(manifest, "squash", storage, {}) is None # Create a new one. squashed = oci_model.lookup_or_create_derived_image( manifest, "squash", "local_us", storage, {}) assert squashed.unique_id assert (oci_model.lookup_or_create_derived_image(manifest, "squash", "local_us", storage, {}) == squashed) # Perform lookup. assert oci_model.lookup_derived_image(manifest, "squash", storage, {}) == squashed