def _garbage_collect_legacy_image(legacy_image_id, context):
assert legacy_image_id is not None
# Check if the image is referenced.
if _check_image_used(legacy_image_id):
return False
# We have an unreferenced image. We can now delete it.
# Grab any derived storage for the image.
for derived in DerivedStorageForImage.select().where(
DerivedStorageForImage.source_image == legacy_image_id):
context.add_blob_id(derived.derivative_id)
try:
image = Image.select().where(Image.id == legacy_image_id).get()
except Image.DoesNotExist:
return False
assert image.repository_id == context.repository.id
# Add the image's blob to be GCed.
context.add_blob_id(image.storage_id)
# If the image has a parent ID, add the parent for GC.
if image.parent_id is not None:
context.add_legacy_image_id(image.parent_id)
# Delete the image.
with db_transaction():
if _check_image_used(legacy_image_id):
return False
try:
image = Image.select().where(Image.id == legacy_image_id).get()
except Image.DoesNotExist:
return False
assert image.id == legacy_image_id
assert image.repository_id == context.repository.id
# Delete any derived storage for the image.
deleted_derived_storage = (DerivedStorageForImage.delete().where(
DerivedStorageForImage.source_image == legacy_image_id).execute())
# Delete the image itself.
image.delete_instance()
context.mark_legacy_image_removed(image)
gc_table_rows_deleted.labels(table="Image").inc()
gc_table_rows_deleted.labels(
table="DerivedStorageForImage").inc(deleted_derived_storage)
if config.image_cleanup_callbacks:
for callback in config.image_cleanup_callbacks:
callback([image])
return True
def test_derived_image(registry_model):
# Clear all existing derived storage.
DerivedStorageForImage.delete().execute()
repository_ref = registry_model.lookup_repository('devtable', 'simple')
tag = registry_model.get_repo_tag(repository_ref, 'latest')
manifest = registry_model.get_manifest_for_tag(tag)
# Ensure the squashed image doesn't exist.
assert registry_model.lookup_derived_image(manifest, 'squash', storage,
{}) is None
# Create a new one.
squashed = registry_model.lookup_or_create_derived_image(
manifest, 'squash', 'local_us', storage, {})
assert registry_model.lookup_or_create_derived_image(
manifest, 'squash', 'local_us', storage, {}) == squashed
assert squashed.unique_id
# Check and set the size.
assert squashed.blob.compressed_size is None
registry_model.set_derived_image_size(squashed, 1234)
found = registry_model.lookup_derived_image(manifest, 'squash', storage,
{})
assert found.blob.compressed_size == 1234
assert found.unique_id == squashed.unique_id
# Ensure its returned now.
assert found == squashed
# Ensure different metadata results in a different derived image.
found = registry_model.lookup_derived_image(manifest, 'squash', storage,
{'foo': 'bar'})
assert found is None
squashed_foo = registry_model.lookup_or_create_derived_image(
manifest, 'squash', 'local_us', storage, {'foo': 'bar'})
assert squashed_foo != squashed
found = registry_model.lookup_derived_image(manifest, 'squash', storage,
{'foo': 'bar'})
assert found == squashed_foo
assert squashed.unique_id != squashed_foo.unique_id
# Lookup with placements.
squashed = registry_model.lookup_or_create_derived_image(
manifest, 'squash', 'local_us', storage, {}, include_placements=True)
assert squashed.blob.placements
# Delete the derived image.
registry_model.delete_derived_image(squashed)
assert registry_model.lookup_derived_image(manifest, 'squash', storage,
{}) is None
def find_or_create_derived_storage(source_image,
transformation_name,
preferred_location,
varying_metadata=None):
existing = find_derived_storage_for_image(source_image,
transformation_name,
varying_metadata)
if existing is not None:
return existing
uniqueness_hash = _get_uniqueness_hash(varying_metadata)
trans = ImageStorageTransformation.get(name=transformation_name)
new_storage = storage.create_v1_storage(preferred_location)
try:
derived = DerivedStorageForImage.create(
source_image=source_image,
derivative=new_storage,
transformation=trans,
uniqueness_hash=uniqueness_hash,
)
except IntegrityError:
# Storage was created while this method executed. Just return the existing.
ImageStoragePlacement.delete().where(
ImageStoragePlacement.storage == new_storage).execute()
new_storage.delete_instance()
return find_derived_storage_for_image(source_image,
transformation_name,
varying_metadata)
return derived
def _get_dangling_storage_count():
storage_ids = set([current.id for current in ImageStorage.select()])
referenced_by_image = set([image.storage_id for image in Image.select()])
referenced_by_manifest = set([blob.blob_id for blob in ManifestBlob.select()])
referenced_by_derived = set(
[derived.derivative_id for derived in DerivedStorageForImage.select()]
)
return len(storage_ids - referenced_by_image - referenced_by_derived - referenced_by_manifest)
def find_derived_storage_for_image(source_image,
transformation_name,
varying_metadata=None):
uniqueness_hash = _get_uniqueness_hash(varying_metadata)
try:
found = (DerivedStorageForImage.select(
ImageStorage, DerivedStorageForImage).join(ImageStorage).switch(
DerivedStorageForImage).join(ImageStorageTransformation).where(
DerivedStorageForImage.source_image == source_image,
ImageStorageTransformation.name == transformation_name,
DerivedStorageForImage.uniqueness_hash == uniqueness_hash,
).get())
return found
except DerivedStorageForImage.DoesNotExist:
return None
def _namespace_from_kwargs(self, args_dict):
if "namespace_name" in args_dict:
return args_dict["namespace_name"]
if "repository_ref" in args_dict:
return args_dict["repository_ref"].namespace_name
if "tag" in args_dict:
return args_dict["tag"].repository.namespace_name
if "manifest" in args_dict:
manifest = args_dict["manifest"]
if manifest._is_tag_manifest:
return TagManifest.get(
id=manifest._db_id).tag.repository.namespace_user.username
else:
return Manifest.get(
id=manifest._db_id).repository.namespace_user.username
if "manifest_or_legacy_image" in args_dict:
manifest_or_legacy_image = args_dict["manifest_or_legacy_image"]
if isinstance(manifest_or_legacy_image, LegacyImage):
return Image.get(id=manifest_or_legacy_image._db_id
).repository.namespace_user.username
else:
manifest = manifest_or_legacy_image
if manifest._is_tag_manifest:
return TagManifest.get(
id=manifest._db_id
).tag.repository.namespace_user.username
else:
return Manifest.get(
id=manifest._db_id).repository.namespace_user.username
if "derived_image" in args_dict:
return DerivedStorageForImage.get(
id=args_dict["derived_image"]._db_id
).source_image.repository.namespace_user.username
if "blob" in args_dict:
return "" # Blob functions are shared, so no need to do anything.
if "blob_upload" in args_dict:
return "" # Blob functions are shared, so no need to do anything.
raise Exception("Unknown namespace for dict `%s`" % args_dict)
def test_derived_image_for_manifest_list(oci_model):
# Clear all existing derived storage.
DerivedStorageForImage.delete().execute()
# Create a config blob for testing.
config_json = json.dumps({
'config': {},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
],
})
app_config = {'TESTING': True}
repository_ref = oci_model.lookup_repository('devtable', 'simple')
with upload_blob(repository_ref, storage,
BlobUploadSettings(500, 500, 500)) as upload:
upload.upload_chunk(app_config, BytesIO(config_json))
blob = upload.commit_to_blob(app_config)
# Create the manifest in the repo.
builder = DockerSchema2ManifestBuilder()
builder.set_config_digest(blob.digest, blob.compressed_size)
builder.add_layer(blob.digest, blob.compressed_size)
amd64_manifest = builder.build()
oci_model.create_manifest_and_retarget_tag(repository_ref, amd64_manifest,
'submanifest', storage)
# Create a manifest list, pointing to at least one amd64+linux manifest.
builder = DockerSchema2ManifestListBuilder()
builder.add_manifest(amd64_manifest, 'amd64', 'linux')
manifestlist = builder.build()
oci_model.create_manifest_and_retarget_tag(repository_ref, manifestlist,
'listtag', storage)
manifest = oci_model.get_manifest_for_tag(
oci_model.get_repo_tag(repository_ref, 'listtag'))
assert manifest
assert manifest.get_parsed_manifest().is_manifest_list
# Ensure the squashed image doesn't exist.
assert oci_model.lookup_derived_image(manifest, 'squash', storage,
{}) is None
# Create a new one.
squashed = oci_model.lookup_or_create_derived_image(
manifest, 'squash', 'local_us', storage, {})
assert squashed.unique_id
assert oci_model.lookup_or_create_derived_image(manifest, 'squash',
'local_us', storage,
{}) == squashed
# Perform lookup.
assert oci_model.lookup_derived_image(manifest, 'squash', storage,
{}) == squashed
def clear_derived_cache():
DerivedStorageForImage.delete().execute()
return "OK"
def test_derived_image_for_manifest_list(manifest_builder, list_builder,
oci_model):
# Clear all existing derived storage.
DerivedStorageForImage.delete().execute()
# Create a config blob for testing.
config_json = json.dumps({
"config": {},
"architecture":
"amd64",
"os":
"linux",
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
],
})
app_config = {"TESTING": True}
repository_ref = oci_model.lookup_repository("devtable", "simple")
with upload_blob(repository_ref, storage,
BlobUploadSettings(500, 500)) as upload:
upload.upload_chunk(app_config, BytesIO(config_json))
blob = upload.commit_to_blob(app_config)
# Create the manifest in the repo.
builder = manifest_builder()
builder.set_config_digest(blob.digest, blob.compressed_size)
builder.add_layer(blob.digest, blob.compressed_size)
amd64_manifest = builder.build()
oci_model.create_manifest_and_retarget_tag(repository_ref,
amd64_manifest,
"submanifest",
storage,
raise_on_error=True)
# Create a manifest list, pointing to at least one amd64+linux manifest.
builder = list_builder()
builder.add_manifest(amd64_manifest, "amd64", "linux")
manifestlist = builder.build()
oci_model.create_manifest_and_retarget_tag(repository_ref,
manifestlist,
"listtag",
storage,
raise_on_error=True)
manifest = oci_model.get_manifest_for_tag(
oci_model.get_repo_tag(repository_ref, "listtag"))
assert manifest
assert manifest.get_parsed_manifest().is_manifest_list
# Ensure the squashed image doesn't exist.
assert oci_model.lookup_derived_image(manifest, "squash", storage,
{}) is None
# Create a new one.
squashed = oci_model.lookup_or_create_derived_image(
manifest, "squash", "local_us", storage, {})
assert squashed.unique_id
assert (oci_model.lookup_or_create_derived_image(manifest, "squash",
"local_us", storage,
{}) == squashed)
# Perform lookup.
assert oci_model.lookup_derived_image(manifest, "squash", storage,
{}) == squashed
