def test_images_shared_cas(default_tag_policy, initialized_db):
"""
A repository, each two tags, pointing to the same image, which has image storage with the same
*CAS path*, but *distinct records*.
Deleting the first tag should delete the first image, and its storage, but not the file in
storage, as it shares its CAS path.
"""
with assert_gc_integrity(expect_storage_removed=True):
repository = create_repository()
# Create two image storage records with the same content checksum.
content = "hello world"
digest = "sha256:" + hashlib.sha256(content).hexdigest()
preferred = storage.preferred_locations[0]
storage.put_content({preferred}, storage.blob_path(digest), content)
is1 = database.ImageStorage.create(content_checksum=digest, uploading=False)
is2 = database.ImageStorage.create(content_checksum=digest, uploading=False)
location = database.ImageStorageLocation.get(name=preferred)
database.ImageStoragePlacement.create(location=location, storage=is1)
database.ImageStoragePlacement.create(location=location, storage=is2)
# Ensure the CAS path exists.
assert storage.exists({preferred}, storage.blob_path(digest))
# Create two images in the repository, and two tags, each pointing to one of the storages.
first_image = Image.create(
docker_image_id="i1", repository=repository, storage=is1, ancestors="/"
)
second_image = Image.create(
docker_image_id="i2", repository=repository, storage=is2, ancestors="/"
)
store_tag_manifest(
repository.namespace_user.username,
repository.name,
"first",
first_image.docker_image_id,
)
store_tag_manifest(
repository.namespace_user.username,
repository.name,
"second",
second_image.docker_image_id,
)
assert_not_deleted(repository, "i1", "i2")
# Delete the first tag.
delete_tag(repository, "first")
assert_deleted(repository, "i1")
assert_not_deleted(repository, "i2")
# Ensure the CAS path still exists.
assert storage.exists({preferred}, storage.blob_path(digest))
def test_images_shared_storage(default_tag_policy, initialized_db):
""" Repository with two tags, both with the same shared storage. Deleting the first
tag should delete the first image, but *not* its storage.
"""
with assert_gc_integrity(expect_storage_removed=False):
repository = create_repository()
# Add two tags, each with their own image, but with the same storage.
image_storage = model.storage.create_v1_storage(
storage.preferred_locations[0])
first_image = Image.create(docker_image_id='i1',
repository=repository,
storage=image_storage,
ancestors='/')
second_image = Image.create(docker_image_id='i2',
repository=repository,
storage=image_storage,
ancestors='/')
store_tag_manifest(repository.namespace_user.username, repository.name,
'first', first_image.docker_image_id)
store_tag_manifest(repository.namespace_user.username, repository.name,
'second', second_image.docker_image_id)
# Delete the first tag.
delete_tag(repository, 'first')
assert_deleted(repository, 'i1')
assert_not_deleted(repository, 'i2')
def test_manifest_backfill_broken_tag(clear_rows, initialized_db):
"""
Tests backfilling a broken tag.
"""
# Delete existing tag manifest so we can reuse the tag.
TagManifestLabel.delete().execute()
TagManifest.delete().execute()
# Create a tag with an image referenced missing parent images.
repo = model.repository.get_repository("devtable", "gargantuan")
broken_image = Image.create(
docker_image_id="foo",
repository=repo,
ancestors="/348723847234/",
storage=ImageStorage.get(),
)
broken_image_tag = RepositoryTag.create(repository=repo,
image=broken_image,
name="broken")
# Backfill the tag.
assert backfill_tag(broken_image_tag)
# Ensure we backfilled, even though we reference a broken manifest.
tag_manifest = TagManifest.get(tag=broken_image_tag)
map_row = TagManifestToManifest.get(tag_manifest=tag_manifest)
manifest = map_row.manifest
assert manifest.manifest_bytes == tag_manifest.json_data
tag = TagToRepositoryTag.get(repository_tag=broken_image_tag).tag
assert tag.name == "broken"
assert tag.manifest == manifest
def find_create_or_link_image(
docker_image_id, repo_obj, username, translations, preferred_location
):
# First check for the image existing in the repository. If found, we simply return it.
repo_image = get_repo_image(repo_obj.namespace_user.username, repo_obj.name, docker_image_id)
if repo_image:
return repo_image
# We next check to see if there is an existing storage the new image can link to.
existing_image_query = (
Image.select(Image, ImageStorage)
.distinct()
.join(ImageStorage)
.switch(Image)
.join(Repository)
.join(RepositoryPermission, JOIN.LEFT_OUTER)
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Image.docker_image_id == docker_image_id)
)
existing_image_query = _basequery.filter_to_repos_for_user(
existing_image_query, _namespace_id_for_username(username)
)
# If there is an existing image, we try to translate its ancestry and copy its storage.
new_image = None
try:
logger.debug("Looking up existing image for ID: %s", docker_image_id)
existing_image = existing_image_query.get()
logger.debug("Existing image %s found for ID: %s", existing_image.id, docker_image_id)
new_image = _find_or_link_image(
existing_image, repo_obj, username, translations, preferred_location
)
if new_image:
return new_image
except Image.DoesNotExist:
logger.debug("No existing image found for ID: %s", docker_image_id)
# Otherwise, create a new storage directly.
with db_transaction():
# Final check for an existing image, under the transaction.
repo_image = get_repo_image(
repo_obj.namespace_user.username, repo_obj.name, docker_image_id
)
if repo_image:
return repo_image
logger.debug("Creating new storage for docker id: %s", docker_image_id)
new_storage = storage.create_v1_storage(preferred_location)
return Image.create(
docker_image_id=docker_image_id, repository=repo_obj, storage=new_storage, ancestors="/"
)
def _temp_link_blob(repository_id, storage, link_expiration_s):
""" Note: Should *always* be called by a parent under a transaction. """
random_image_name = str(uuid4())
# Create a temporary link into the repository, to be replaced by the v1 metadata later
# and create a temporary tag to reference it
image = Image.create(storage=storage,
docker_image_id=random_image_name,
repository=repository_id)
tag.create_temporary_hidden_tag(repository_id, image, link_expiration_s)
def _find_or_link_image(existing_image, repo_obj, username, translations,
preferred_location):
with db_transaction():
# Check for an existing image, under the transaction, to make sure it doesn't already exist.
repo_image = get_repo_image(repo_obj.namespace_user.username,
repo_obj.name,
existing_image.docker_image_id)
if repo_image:
return repo_image
# Make sure the existing base image still exists.
try:
to_copy = Image.select().join(ImageStorage).where(
Image.id == existing_image.id).get()
msg = "Linking image to existing storage with docker id: %s and uuid: %s"
logger.debug(msg, existing_image.docker_image_id,
to_copy.storage.uuid)
new_image_ancestry = __translate_ancestry(to_copy.ancestors,
translations, repo_obj,
username,
preferred_location)
copied_storage = to_copy.storage
translated_parent_id = None
if new_image_ancestry != "/":
translated_parent_id = int(new_image_ancestry.split("/")[-2])
new_image = Image.create(
docker_image_id=existing_image.docker_image_id,
repository=repo_obj,
storage=copied_storage,
ancestors=new_image_ancestry,
command=existing_image.command,
created=existing_image.created,
comment=existing_image.comment,
v1_json_metadata=existing_image.v1_json_metadata,
aggregate_size=existing_image.aggregate_size,
parent=translated_parent_id,
v1_checksum=existing_image.v1_checksum,
)
logger.debug("Storing translation %s -> %s", existing_image.id,
new_image.id)
translations[existing_image.id] = new_image.id
return new_image
except Image.DoesNotExist:
return None
def test_images_shared_cas_with_new_blob_table(default_tag_policy,
initialized_db):
""" A repository with a tag and image that shares its CAS path with a record in the new Blob
table. Deleting the first tag should delete the first image, and its storage, but not the
file in storage, as it shares its CAS path with the blob row.
"""
with assert_gc_integrity(expect_storage_removed=True):
repository = create_repository()
# Create two image storage records with the same content checksum.
content = "hello world"
digest = "sha256:" + hashlib.sha256(content).hexdigest()
preferred = storage.preferred_locations[0]
storage.put_content({preferred}, storage.blob_path(digest), content)
media_type = database.MediaType.get(name="text/plain")
is1 = database.ImageStorage.create(content_checksum=digest,
uploading=False)
database.ApprBlob.create(digest=digest, size=0, media_type=media_type)
location = database.ImageStorageLocation.get(name=preferred)
database.ImageStoragePlacement.create(location=location, storage=is1)
# Ensure the CAS path exists.
assert storage.exists({preferred}, storage.blob_path(digest))
# Create the image in the repository, and the tag.
first_image = Image.create(docker_image_id="i1",
repository=repository,
storage=is1,
ancestors="/")
store_tag_manifest(
repository.namespace_user.username,
repository.name,
"first",
first_image.docker_image_id,
)
assert_not_deleted(repository, "i1")
# Delete the tag.
delete_tag(repository, "first")
assert_deleted(repository, "i1")
# Ensure the CAS path still exists, as it is referenced by the Blob table
assert storage.exists({preferred}, storage.blob_path(digest))
def synthesize_v1_image(
repo,
image_storage_id,
storage_image_size,
docker_image_id,
created_date_str,
comment,
command,
v1_json_metadata,
parent_image=None,
):
"""
Find an existing image with this docker image id, and if none exists, write one with the
specified metadata.
"""
ancestors = "/"
if parent_image is not None:
ancestors = "{0}{1}/".format(parent_image.ancestors, parent_image.id)
created = None
if created_date_str is not None:
try:
created = dateutil.parser.parse(created_date_str).replace(tzinfo=None)
except:
# parse raises different exceptions, so we cannot use a specific kind of handler here.
pass
# Get the aggregate size for the image.
aggregate_size = _basequery.calculate_image_aggregate_size(
ancestors, storage_image_size, parent_image
)
try:
return Image.create(
docker_image_id=docker_image_id,
ancestors=ancestors,
comment=comment,
command=command,
v1_json_metadata=v1_json_metadata,
created=created,
storage=image_storage_id,
repository=repo,
parent=parent_image,
aggregate_size=aggregate_size,
)
except IntegrityError:
return Image.get(docker_image_id=docker_image_id, repository=repo)
def test_image_with_cas(default_tag_policy, initialized_db):
""" A repository with a tag pointing to an image backed by CAS. Deleting and GCing the tag
should result in the storage and its CAS data being removed.
"""
with assert_gc_integrity(expect_storage_removed=True):
repository = create_repository()
# Create an image storage record under CAS.
content = "hello world"
digest = "sha256:" + hashlib.sha256(content).hexdigest()
preferred = storage.preferred_locations[0]
storage.put_content({preferred}, storage.blob_path(digest), content)
image_storage = database.ImageStorage.create(content_checksum=digest,
uploading=False)
location = database.ImageStorageLocation.get(name=preferred)
database.ImageStoragePlacement.create(location=location,
storage=image_storage)
# Ensure the CAS path exists.
assert storage.exists({preferred}, storage.blob_path(digest))
# Create the image and the tag.
first_image = Image.create(docker_image_id="i1",
repository=repository,
storage=image_storage,
ancestors="/")
store_tag_manifest(
repository.namespace_user.username,
repository.name,
"first",
first_image.docker_image_id,
)
assert_not_deleted(repository, "i1")
# Delete the tag.
delete_tag(repository, "first")
assert_deleted(repository, "i1")
# Ensure the CAS path is gone.
assert not storage.exists({preferred}, storage.blob_path(digest))