def test_manifestbackfillworker_repeat_digest(clear_rows, initialized_db):
"""
Tests that a manifest with a shared digest will be properly linked.
"""
# Delete existing tag manifest so we can reuse the tag.
TagManifestLabel.delete().execute()
TagManifest.delete().execute()
repo = model.repository.get_repository("devtable", "gargantuan")
tag_v30 = model.tag.get_active_tag("devtable", "gargantuan", "v3.0")
tag_v50 = model.tag.get_active_tag("devtable", "gargantuan", "v5.0")
# Build a manifest and assign it to both tags (this is allowed in the old model).
builder = DockerSchema1ManifestBuilder("devtable", "gargantuan", "sometag")
builder.add_layer("sha256:deadbeef", '{"id": "foo"}')
manifest = builder.build(docker_v2_signing_key)
manifest_1 = TagManifest.create(json_data=manifest.bytes.as_encoded_str(),
digest=manifest.digest,
tag=tag_v30)
manifest_2 = TagManifest.create(json_data=manifest.bytes.as_encoded_str(),
digest=manifest.digest,
tag=tag_v50)
# Backfill "both" manifests and ensure both are pointed to by a single resulting row.
assert _backfill_manifest(manifest_1)
assert _backfill_manifest(manifest_2)
map_row1 = TagManifestToManifest.get(tag_manifest=manifest_1)
map_row2 = TagManifestToManifest.get(tag_manifest=manifest_2)
assert map_row1.manifest == map_row2.manifest
def test_manifestbackfillworker_mislinked_invalid_manifest(clear_rows, initialized_db):
""" Tests that a manifest whose image is mislinked will attempt to have its storages relinked
properly. """
# Delete existing tag manifest so we can reuse the tag.
TagManifestLabel.delete().execute()
TagManifest.delete().execute()
repo = model.repository.get_repository("devtable", "complex")
tag_v50 = model.tag.get_active_tag("devtable", "gargantuan", "v5.0")
# Add a mislinked manifest, by having its layer point to an invalid blob but its image
# be the v5.0 image.
builder = DockerSchema1ManifestBuilder("devtable", "gargantuan", "sometag")
builder.add_layer("sha256:deadbeef", '{"id": "foo"}')
manifest = builder.build(docker_v2_signing_key)
broken_manifest = TagManifest.create(
json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50
)
# Backfill the manifest and ensure it is marked as broken.
assert _backfill_manifest(broken_manifest)
map_row = TagManifestToManifest.get(tag_manifest=broken_manifest)
assert map_row.broken
manifest_row = map_row.manifest
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
assert legacy_image == tag_v50.image
manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row))
assert len(manifest_blobs) == 0
def test_manifestbackfillworker_mislinked_manifest(clear_rows, initialized_db):
""" Tests that a manifest whose image is mislinked will have its storages relinked properly. """
# Delete existing tag manifest so we can reuse the tag.
TagManifestLabel.delete().execute()
TagManifest.delete().execute()
repo = model.repository.get_repository('devtable', 'complex')
tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0')
tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0')
# Add a mislinked manifest, by having its layer point to a blob in v3.0 but its image
# be the v5.0 image.
builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag')
builder.add_layer(tag_v30.image.storage.content_checksum, '{"id": "foo"}')
manifest = builder.build(docker_v2_signing_key)
mislinked_manifest = TagManifest.create(json_data=manifest.bytes.as_encoded_str(),
digest=manifest.digest,
tag=tag_v50)
# Backfill the manifest and ensure its proper content checksum was linked.
assert _backfill_manifest(mislinked_manifest)
map_row = TagManifestToManifest.get(tag_manifest=mislinked_manifest)
assert not map_row.broken
manifest_row = map_row.manifest
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
assert legacy_image == tag_v50.image
manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row))
assert len(manifest_blobs) == 1
assert manifest_blobs[0].blob.content_checksum == tag_v30.image.storage.content_checksum
def _associate_manifest(tag, oci_manifest):
with db_transaction():
tag_manifest = TagManifest.create(
tag=tag,
digest=oci_manifest.digest,
json_data=oci_manifest.manifest_bytes)
TagManifestToManifest.create(tag_manifest=tag_manifest,
manifest=oci_manifest)
return tag_manifest
def _get_manifest_id(repositorytag):
repository_tag_datatype = TagDataType.for_repository_tag(repositorytag)
# Retrieve the TagManifest for the RepositoryTag, backfilling if necessary.
with db_transaction():
manifest_datatype = None
try:
manifest_datatype = pre_oci_model.get_manifest_for_tag(
repository_tag_datatype, backfill_if_necessary=True)
except MalformedSchema1Manifest:
logger.exception('Error backfilling manifest for tag `%s`',
repositorytag.id)
if manifest_datatype is None:
logger.error('Could not load or backfill manifest for tag `%s`',
repositorytag.id)
# Create a broken manifest for the tag.
tag_manifest = TagManifest.create(tag=repositorytag,
digest='BROKEN-%s' %
repositorytag.id,
json_data='{}')
else:
# Retrieve the new-style Manifest for the TagManifest, if any.
try:
tag_manifest = TagManifest.get(id=manifest_datatype._db_id)
except TagManifest.DoesNotExist:
logger.exception('Could not find tag manifest')
return None
try:
found = TagManifestToManifest.get(tag_manifest=tag_manifest).manifest
# Verify that the new-style manifest has the same contents as the old-style manifest.
# If not, update and then return. This is an extra check put in place to ensure unicode
# manifests have been correctly copied.
if found.manifest_bytes != tag_manifest.json_data:
logger.warning('Fixing manifest `%s`', found.id)
found.manifest_bytes = tag_manifest.json_data
found.save()
return found.id
except TagManifestToManifest.DoesNotExist:
# Could not find the new style manifest, so backfill.
_backfill_manifest(tag_manifest)
# Try to retrieve the manifest again, since we've performed a backfill.
try:
return TagManifestToManifest.get(tag_manifest=tag_manifest).manifest_id
except TagManifestToManifest.DoesNotExist:
return None
def test_manifestbackfillworker_broken_manifest(clear_rows, initialized_db):
# Delete existing tag manifest so we can reuse the tag.
TagManifestLabel.delete().execute()
TagManifest.delete().execute()
# Add a broken manifest.
broken_manifest = TagManifest.create(json_data='wat?', digest='sha256:foobar',
tag=RepositoryTag.get())
# Ensure the backfill works.
assert _backfill_manifest(broken_manifest)
# Ensure the mapping is marked as broken.
map_row = TagManifestToManifest.get(tag_manifest=broken_manifest)
assert map_row.broken
manifest_row = map_row.manifest
assert manifest_row.manifest_bytes == broken_manifest.json_data
assert manifest_row.digest == broken_manifest.digest
assert manifest_row.repository == broken_manifest.tag.repository
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
assert broken_manifest.tag.image == legacy_image
def retarget_tag(tag_name,
manifest_id,
is_reversion=False,
now_ms=None,
adjust_old_model=True):
"""
Creates or updates a tag with the specified name to point to the given manifest under its
repository.
If this action is a reversion to a previous manifest, is_reversion should be set to True.
Returns the newly created tag row or None on error.
"""
try:
manifest = (Manifest.select(
Manifest,
MediaType).join(MediaType).where(Manifest.id == manifest_id).get())
except Manifest.DoesNotExist:
return None
# CHECK: Make sure that we are not mistargeting a schema 1 manifest to a tag with a different
# name.
if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES:
try:
parsed = DockerSchema1Manifest(Bytes.for_string_or_unicode(
manifest.manifest_bytes),
validate=False)
if parsed.tag != tag_name:
logger.error(
"Tried to re-target schema1 manifest with tag `%s` to tag `%s",
parsed.tag,
tag_name,
)
return None
except MalformedSchema1Manifest:
logger.exception("Could not parse schema1 manifest")
return None
legacy_image = get_legacy_image_for_manifest(manifest)
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
# Lookup an existing tag in the repository with the same name and, if present, mark it
# as expired.
existing_tag = get_tag(manifest.repository_id, tag_name)
if existing_tag is not None:
_, okay = set_tag_end_ms(existing_tag, now_ms)
# TODO: should we retry here and/or use a for-update?
if not okay:
return None
# Create a new tag pointing to the manifest with a lifetime start of now.
created = Tag.create(
name=tag_name,
repository=manifest.repository_id,
lifetime_start_ms=now_ms,
reversion=is_reversion,
manifest=manifest,
tag_kind=Tag.tag_kind.get_id("tag"),
)
# TODO: Remove the linkage code once RepositoryTag is gone.
# If this is a schema 1 manifest, then add a TagManifest linkage to it. Otherwise, it will only
# be pullable via the new OCI model.
if adjust_old_model:
if (manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES
and legacy_image is not None):
old_style_tag = RepositoryTag.create(
repository=manifest.repository_id,
image=legacy_image,
name=tag_name,
lifetime_start_ts=now_ts,
reversion=is_reversion,
)
TagToRepositoryTag.create(tag=created,
repository_tag=old_style_tag,
repository=manifest.repository_id)
tag_manifest = TagManifest.create(
tag=old_style_tag,
digest=manifest.digest,
json_data=manifest.manifest_bytes)
TagManifestToManifest.create(tag_manifest=tag_manifest,
manifest=manifest,
repository=manifest.repository_id)
return created
