def _is_storage_orphaned(candidate_id):
"""
Returns the whether the given candidate storage ID is orphaned. Must be executed
under a transaction.
"""
with ensure_under_transaction():
try:
ManifestBlob.get(blob=candidate_id)
return False
except ManifestBlob.DoesNotExist:
pass
try:
Image.get(storage=candidate_id)
return False
except Image.DoesNotExist:
pass
try:
UploadedBlob.get(blob=candidate_id)
return False
except UploadedBlob.DoesNotExist:
pass
return True
def _populate_manifest_and_blobs(repository,
manifest,
storage_id_map,
leaf_layer_id=None):
leaf_layer_id = leaf_layer_id or manifest.leaf_layer_v1_image_id
try:
legacy_image = Image.get(Image.docker_image_id == leaf_layer_id,
Image.repository == repository)
except Image.DoesNotExist:
raise DataModelException("Invalid image with id: %s" % leaf_layer_id)
storage_ids = set()
for blob_digest in manifest.local_blob_digests:
image_storage_id = storage_id_map.get(blob_digest)
if image_storage_id is None:
logger.error("Missing blob for manifest `%s` in: %s", blob_digest,
storage_id_map)
raise DataModelException("Missing blob for manifest `%s`" %
blob_digest)
if image_storage_id in storage_ids:
continue
storage_ids.add(image_storage_id)
return populate_manifest(repository, manifest, legacy_image, storage_ids)
def test_create_temp_tag_deleted_repo(initialized_db):
repo = model.repository.get_repository("devtable", "simple")
repo.state = RepositoryState.MARKED_FOR_DELETION
repo.save()
image = Image.get(repository=repo)
assert model.tag.create_temporary_hidden_tag(repo, image, 10000000) is None
def test_load_security_information_api_responses(secscan_api_response,
initialized_db):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref,
"latest",
include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag,
backfill_if_necessary=True,
include_legacy_image=True)
set_secscan_status(Image.get(id=manifest.legacy_image._db_id), True, 3)
secscan = V2SecurityScanner(app, instance_keys, storage)
secscan._legacy_secscan_api = mock.Mock()
secscan._legacy_secscan_api.get_layer_data.return_value = secscan_api_response
security_information = secscan.load_security_information(
manifest).security_information
assert isinstance(security_information, SecurityInformation)
assert security_information.Layer.Name == secscan_api_response[
"Layer"].get("Name", "")
assert security_information.Layer.ParentName == secscan_api_response[
"Layer"].get("ParentName", "")
assert security_information.Layer.IndexedByVersion == secscan_api_response[
"Layer"].get("IndexedByVersion", None)
assert len(security_information.Layer.Features) == len(
secscan_api_response["Layer"].get("Features", []))
def synthesize_v1_image(
repo,
image_storage_id,
storage_image_size,
docker_image_id,
created_date_str,
comment,
command,
v1_json_metadata,
parent_image=None,
):
"""
Find an existing image with this docker image id, and if none exists, write one with the
specified metadata.
"""
ancestors = "/"
if parent_image is not None:
ancestors = "{0}{1}/".format(parent_image.ancestors, parent_image.id)
created = None
if created_date_str is not None:
try:
created = dateutil.parser.parse(created_date_str).replace(tzinfo=None)
except:
# parse raises different exceptions, so we cannot use a specific kind of handler here.
pass
# Get the aggregate size for the image.
aggregate_size = _basequery.calculate_image_aggregate_size(
ancestors, storage_image_size, parent_image
)
try:
return Image.create(
docker_image_id=docker_image_id,
ancestors=ancestors,
comment=comment,
command=command,
v1_json_metadata=v1_json_metadata,
created=created,
storage=image_storage_id,
repository=repo,
parent=parent_image,
aggregate_size=aggregate_size,
)
except IntegrityError:
return Image.get(docker_image_id=docker_image_id, repository=repo)
def _namespace_from_kwargs(self, args_dict):
if "namespace_name" in args_dict:
return args_dict["namespace_name"]
if "repository_ref" in args_dict:
return args_dict["repository_ref"].namespace_name
if "tag" in args_dict:
return args_dict["tag"].repository.namespace_name
if "manifest" in args_dict:
manifest = args_dict["manifest"]
if manifest._is_tag_manifest:
return TagManifest.get(
id=manifest._db_id).tag.repository.namespace_user.username
else:
return Manifest.get(
id=manifest._db_id).repository.namespace_user.username
if "manifest_or_legacy_image" in args_dict:
manifest_or_legacy_image = args_dict["manifest_or_legacy_image"]
if isinstance(manifest_or_legacy_image, LegacyImage):
return Image.get(id=manifest_or_legacy_image._db_id
).repository.namespace_user.username
else:
manifest = manifest_or_legacy_image
if manifest._is_tag_manifest:
return TagManifest.get(
id=manifest._db_id
).tag.repository.namespace_user.username
else:
return Manifest.get(
id=manifest._db_id).repository.namespace_user.username
if "derived_image" in args_dict:
return DerivedStorageForImage.get(
id=args_dict["derived_image"]._db_id
).source_image.repository.namespace_user.username
if "blob" in args_dict:
return "" # Blob functions are shared, so no need to do anything.
if "blob_upload" in args_dict:
return "" # Blob functions are shared, so no need to do anything.
raise Exception("Unknown namespace for dict `%s`" % args_dict)
def compute_layer_id(layer):
""" Returns the ID for the layer in the security scanner. """
# NOTE: this is temporary until we switch to Clair V3.
if isinstance(layer, ManifestDataType):
if layer._is_tag_manifest:
layer = TagManifest.get(id=layer._db_id).tag.image
else:
manifest = Manifest.get(id=layer._db_id)
try:
layer = ManifestLegacyImage.get(manifest=manifest).image
except ManifestLegacyImage.DoesNotExist:
return None
elif isinstance(layer, LegacyImage):
layer = Image.get(id=layer._db_id)
assert layer.docker_image_id
assert layer.storage.uuid
return '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
def _namespace_from_kwargs(self, args_dict):
if 'namespace_name' in args_dict:
return args_dict['namespace_name']
if 'repository_ref' in args_dict:
return args_dict['repository_ref'].namespace_name
if 'tag' in args_dict:
return args_dict['tag'].repository.namespace_name
if 'manifest' in args_dict:
manifest = args_dict['manifest']
if manifest._is_tag_manifest:
return TagManifest.get(id=manifest._db_id).tag.repository.namespace_user.username
else:
return Manifest.get(id=manifest._db_id).repository.namespace_user.username
if 'manifest_or_legacy_image' in args_dict:
manifest_or_legacy_image = args_dict['manifest_or_legacy_image']
if isinstance(manifest_or_legacy_image, LegacyImage):
return Image.get(id=manifest_or_legacy_image._db_id).repository.namespace_user.username
else:
manifest = manifest_or_legacy_image
if manifest._is_tag_manifest:
return TagManifest.get(id=manifest._db_id).tag.repository.namespace_user.username
else:
return Manifest.get(id=manifest._db_id).repository.namespace_user.username
if 'derived_image' in args_dict:
return (DerivedStorageForImage
.get(id=args_dict['derived_image']._db_id)
.source_image
.repository
.namespace_user
.username)
if 'blob' in args_dict:
return '' # Blob functions are shared, so no need to do anything.
if 'blob_upload' in args_dict:
return '' # Blob functions are shared, so no need to do anything.
raise Exception('Unknown namespace for dict `%s`' % args_dict)
def assert_gc_integrity(expect_storage_removed=True):
"""
Specialized assertion for ensuring that GC cleans up all dangling storages and labels, invokes
the callback for images removed and doesn't invoke the callback for images *not* removed.
"""
# Add a callback for when images are removed.
removed_image_storages = []
remove_callback = model.config.register_image_cleanup_callback(
removed_image_storages.extend)
# Store existing storages. We won't verify these for existence because they
# were likely created as test data.
existing_digests = set()
for storage_row in ImageStorage.select():
if storage_row.cas_path:
existing_digests.add(storage_row.content_checksum)
for blob_row in ApprBlob.select():
existing_digests.add(blob_row.digest)
# Store the number of dangling objects.
existing_storage_count = _get_dangling_storage_count()
existing_label_count = _get_dangling_label_count()
existing_manifest_count = _get_dangling_manifest_count()
# Yield to the GC test.
with check_transitive_modifications():
try:
yield
finally:
remove_callback()
# Ensure the number of dangling storages, manifests and labels has not changed.
updated_storage_count = _get_dangling_storage_count()
assert updated_storage_count == existing_storage_count
updated_label_count = _get_dangling_label_count()
assert updated_label_count == existing_label_count, _get_dangling_labels()
updated_manifest_count = _get_dangling_manifest_count()
assert updated_manifest_count == existing_manifest_count
# Ensure that for each call to the image+storage cleanup callback, the image and its
# storage is not found *anywhere* in the database.
for removed_image_and_storage in removed_image_storages:
assert isinstance(removed_image_and_storage, Image)
try:
# NOTE: SQLite can and will reuse AUTOINCREMENT IDs occasionally, so if we find a row
# with the same ID, make sure it does not have the same Docker Image ID.
# See: https://www.sqlite.org/autoinc.html
found_image = Image.get(id=removed_image_and_storage.id)
assert (found_image.docker_image_id !=
removed_image_and_storage.docker_image_id
), "Found unexpected removed image %s under repo %s" % (
found_image.id,
found_image.repository,
)
except Image.DoesNotExist:
pass
# Ensure that image storages are only removed if not shared.
shared = Image.select().where(
Image.storage == removed_image_and_storage.storage_id).count()
if shared == 0:
shared = (ManifestBlob.select().where(
ManifestBlob.blob ==
removed_image_and_storage.storage_id).count())
if shared == 0:
shared = (UploadedBlob.select().where(
UploadedBlob.blob ==
removed_image_and_storage.storage_id).count())
if shared == 0:
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(id=removed_image_and_storage.storage_id)
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(uuid=removed_image_and_storage.storage.uuid)
# Ensure all CAS storage is in the storage engine.
preferred = storage.preferred_locations[0]
for storage_row in ImageStorage.select():
if storage_row.content_checksum in existing_digests:
continue
if storage_row.cas_path:
storage.get_content({preferred},
storage.blob_path(
storage_row.content_checksum))
for blob_row in ApprBlob.select():
if blob_row.digest in existing_digests:
continue
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
# Ensure all tags have valid manifests.
for manifest in {t.manifest for t in Tag.select()}:
# Ensure that the manifest's blobs all exist.
found_blobs = {
b.blob.content_checksum
for b in ManifestBlob.select().where(
ManifestBlob.manifest == manifest)
}
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(manifest.manifest_bytes),
manifest.media_type.name)
assert set(parsed.local_blob_digests) == found_blobs
def assert_gc_integrity(expect_storage_removed=True, check_oci_tags=True):
""" Specialized assertion for ensuring that GC cleans up all dangling storages
and labels, invokes the callback for images removed and doesn't invoke the
callback for images *not* removed.
"""
# Add a callback for when images are removed.
removed_image_storages = []
model.config.register_image_cleanup_callback(removed_image_storages.extend)
# Store the number of dangling storages and labels.
existing_storage_count = _get_dangling_storage_count()
existing_label_count = _get_dangling_label_count()
existing_manifest_count = _get_dangling_manifest_count()
yield
# Ensure the number of dangling storages, manifests and labels has not changed.
updated_storage_count = _get_dangling_storage_count()
assert updated_storage_count == existing_storage_count
updated_label_count = _get_dangling_label_count()
assert updated_label_count == existing_label_count, _get_dangling_labels()
updated_manifest_count = _get_dangling_manifest_count()
assert updated_manifest_count == existing_manifest_count
# Ensure that for each call to the image+storage cleanup callback, the image and its
# storage is not found *anywhere* in the database.
for removed_image_and_storage in removed_image_storages:
with pytest.raises(Image.DoesNotExist):
Image.get(id=removed_image_and_storage.id)
# Ensure that image storages are only removed if not shared.
shared = Image.select().where(
Image.storage == removed_image_and_storage.storage_id).count()
if shared == 0:
shared = (ManifestBlob.select().where(
ManifestBlob.blob ==
removed_image_and_storage.storage_id).count())
if shared == 0:
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(id=removed_image_and_storage.storage_id)
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(uuid=removed_image_and_storage.storage.uuid)
# Ensure all CAS storage is in the storage engine.
preferred = storage.preferred_locations[0]
for storage_row in ImageStorage.select():
if storage_row.cas_path:
storage.get_content({preferred},
storage.blob_path(
storage_row.content_checksum))
for blob_row in ApprBlob.select():
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
# Ensure there are no danglings OCI tags.
if check_oci_tags:
oci_tags = {t.id for t in Tag.select()}
referenced_oci_tags = {t.tag_id for t in TagToRepositoryTag.select()}
assert not oci_tags - referenced_oci_tags
# Ensure all tags have valid manifests.
for manifest in {t.manifest for t in Tag.select()}:
# Ensure that the manifest's blobs all exist.
found_blobs = {
b.blob.content_checksum
for b in ManifestBlob.select().where(
ManifestBlob.manifest == manifest)
}
parsed = parse_manifest_from_bytes(
Bytes.for_string_or_unicode(manifest.manifest_bytes),
manifest.media_type.name)
assert set(parsed.local_blob_digests) == found_blobs
def _get_legacy_image(namespace, repo, tag, include_storage=True):
repo_ref = registry_model.lookup_repository(namespace, repo)
repo_tag = registry_model.get_repo_tag(repo_ref,
tag,
include_legacy_image=True)
return Image.get(id=repo_tag.legacy_image._db_id)
def create_or_update_tag_for_repo(repository_id,
tag_name,
tag_docker_image_id,
reversion=False,
oci_manifest=None,
now_ms=None):
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
try:
tag = db_for_update(
_tag_alive(
RepositoryTag.select().where(
RepositoryTag.repository == repository_id,
RepositoryTag.name == tag_name),
now_ts,
)).get()
tag.lifetime_end_ts = now_ts
tag.save()
# Check for an OCI tag.
try:
oci_tag = db_for_update(
Tag.select().join(TagToRepositoryTag).where(
TagToRepositoryTag.repository_tag == tag)).get()
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except Tag.DoesNotExist:
pass
except RepositoryTag.DoesNotExist:
pass
except IntegrityError:
msg = "Tag with name %s was stale when we tried to update it; Please retry the push"
raise StaleTagException(msg % tag_name)
try:
image_obj = Image.get(Image.docker_image_id == tag_docker_image_id,
Image.repository == repository_id)
except Image.DoesNotExist:
raise DataModelException("Invalid image with id: %s" %
tag_docker_image_id)
try:
created = RepositoryTag.create(
repository=repository_id,
image=image_obj,
name=tag_name,
lifetime_start_ts=now_ts,
reversion=reversion,
)
if oci_manifest:
# Create the OCI tag as well.
oci_tag = Tag.create(
repository=repository_id,
manifest=oci_manifest,
name=tag_name,
lifetime_start_ms=now_ms,
reversion=reversion,
tag_kind=Tag.tag_kind.get_id("tag"),
)
TagToRepositoryTag.create(tag=oci_tag,
repository_tag=created,
repository=repository_id)
return created
except IntegrityError:
msg = "Tag with name %s and lifetime start %s already exists"
raise TagAlreadyCreatedException(msg % (tag_name, now_ts))
def test_tagbackfillworker(clear_all_rows, initialized_db):
# Remove the new-style rows so we can backfill.
TagToRepositoryTag.delete().execute()
Tag.delete().execute()
if clear_all_rows:
TagManifestLabelMap.delete().execute()
ManifestLabel.delete().execute()
ManifestBlob.delete().execute()
ManifestLegacyImage.delete().execute()
TagManifestToManifest.delete().execute()
Manifest.delete().execute()
found_dead_tag = False
for repository_tag in list(RepositoryTag.select()):
# Backfill the tag.
assert backfill_tag(repository_tag)
# Ensure if we try again, the backfill is skipped.
assert not backfill_tag(repository_tag)
# Ensure that we now have the expected tag rows.
tag_to_repo_tag = TagToRepositoryTag.get(repository_tag=repository_tag)
tag = tag_to_repo_tag.tag
assert tag.name == repository_tag.name
assert tag.repository == repository_tag.repository
assert not tag.hidden
assert tag.reversion == repository_tag.reversion
if repository_tag.lifetime_start_ts is None:
assert tag.lifetime_start_ms is None
else:
assert tag.lifetime_start_ms == (repository_tag.lifetime_start_ts *
1000)
if repository_tag.lifetime_end_ts is None:
assert tag.lifetime_end_ms is None
else:
assert tag.lifetime_end_ms == (repository_tag.lifetime_end_ts *
1000)
found_dead_tag = True
assert tag.manifest
# Ensure that we now have the expected manifest rows.
try:
tag_manifest = TagManifest.get(tag=repository_tag)
except TagManifest.DoesNotExist:
continue
map_row = TagManifestToManifest.get(tag_manifest=tag_manifest)
assert not map_row.broken
manifest_row = map_row.manifest
assert manifest_row.manifest_bytes == tag_manifest.json_data
assert manifest_row.digest == tag_manifest.digest
assert manifest_row.repository == tag_manifest.tag.repository
assert tag.manifest == map_row.manifest
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
assert tag_manifest.tag.image == legacy_image
expected_storages = {tag_manifest.tag.image.storage.id}
for parent_image_id in tag_manifest.tag.image.ancestor_id_list():
expected_storages.add(Image.get(id=parent_image_id).storage_id)
found_storages = {
manifest_blob.blob_id
for manifest_blob in ManifestBlob.select().where(
ManifestBlob.manifest == manifest_row)
}
assert expected_storages == found_storages
# Ensure the labels were copied over.
tmls = list(TagManifestLabel.select().where(
TagManifestLabel.annotated == tag_manifest))
expected_labels = {tml.label_id for tml in tmls}
found_labels = {
m.label_id
for m in ManifestLabel.select().where(
ManifestLabel.manifest == manifest_row)
}
assert found_labels == expected_labels
# Verify at the repository level.
for repository in list(Repository.select()):
tags = RepositoryTag.select().where(
RepositoryTag.repository == repository,
RepositoryTag.hidden == False)
oci_tags = Tag.select().where(Tag.repository == repository)
assert len(tags) == len(oci_tags)
assert {t.name for t in tags} == {t.name for t in oci_tags}
for tag in tags:
tag_manifest = TagManifest.get(tag=tag)
ttr = TagToRepositoryTag.get(repository_tag=tag)
manifest = ttr.tag.manifest
assert tag_manifest.json_data == manifest.manifest_bytes
assert tag_manifest.digest == manifest.digest
assert tag.image == ManifestLegacyImage.get(
manifest=manifest).image
assert tag.lifetime_start_ts == (ttr.tag.lifetime_start_ms / 1000)
if tag.lifetime_end_ts:
assert tag.lifetime_end_ts == (ttr.tag.lifetime_end_ms / 1000)
else:
assert ttr.tag.lifetime_end_ms is None
assert found_dead_tag
def get_storage_replication_entry(image_id):
image = Image.get(docker_image_id=image_id)
QueueItem.select().where(
QueueItem.queue_name**("%" + image.storage.uuid + "%")).get()
return "OK"
def delete_image(image_id):
image = Image.get(docker_image_id=image_id)
image.docker_image_id = "DELETED"
image.save()
return "OK"
def get_image_by_db_id(id):
try:
return Image.get(id=id)
except Image.DoesNotExist:
return None
def test_create_temp_tag(initialized_db):
repo = model.repository.get_repository("devtable", "simple")
image = Image.get(repository=repo)
assert model.tag.create_temporary_hidden_tag(repo, image,
10000000) is not None
def delete_image(image_id): image = Image.get(docker_image_id=image_id) image.docker_image_id = 'DELETED' image.save() return 'OK'