def list_repository_tag_history(
self,
repository_ref,
page=1,
size=100,
specific_tag_name=None,
active_tags_only=False,
since_time_ms=None,
):
"""
Returns the history of all tags in the repository (unless filtered).
This includes tags that have been made in-active due to newer versions of those tags coming
into service.
"""
tags, has_more = oci.tag.list_repository_tag_history(
repository_ref._db_id, page, size, specific_tag_name, active_tags_only, since_time_ms
)
# TODO: do we need legacy images here?
legacy_images_map = oci.tag.get_legacy_images_for_tags(tags)
return (
[
Tag.for_tag(tag, LegacyImage.for_image(legacy_images_map.get(tag.id)))
for tag in tags
],
has_more,
)
def list_repository_tag_history(self,
repository_ref,
page=1,
size=100,
specific_tag_name=None,
active_tags_only=False,
since_time_ms=None):
"""
Returns the history of all tags in the repository (unless filtered). This includes tags that
have been made in-active due to newer versions of those tags coming into service.
"""
# Only available on OCI model
if since_time_ms is not None:
raise NotImplementedError
tags, manifest_map, has_more = model.tag.list_repository_tag_history(
repository_ref._db_id, page, size, specific_tag_name,
active_tags_only)
return [
Tag.for_repository_tag(tag,
manifest_map.get(tag.id),
legacy_image=LegacyImage.for_image(
tag.image)) for tag in tags
], has_more
def delete_tags_for_manifest(self, manifest):
"""
Deletes all tags pointing to the given manifest, making the manifest inaccessible for pulling.
Returns the tags deleted, if any. Returns None on error.
"""
deleted_tags = oci.tag.delete_tags_for_manifest(manifest._db_id)
return [Tag.for_tag(tag) for tag in deleted_tags]
def find_matching_tag(self, repository_ref, tag_names):
""" Finds an alive tag in the repository matching one of the given tag names and returns it
or None if none.
"""
found_tag = oci.tag.find_matching_tag(repository_ref._db_id, tag_names)
assert found_tag is None or not found_tag.hidden
return Tag.for_tag(found_tag)
def get_most_recent_tag(self, repository_ref):
""" Returns the most recently pushed alive tag in the repository, if any. If none, returns
None.
"""
found_tag = model.tag.get_most_recent_tag(repository_ref._db_id)
assert found_tag is None or not found_tag.hidden
return Tag.for_repository_tag(found_tag)
def create_manifest_and_retarget_tag(self,
repository_ref,
manifest_interface_instance,
tag_name,
storage,
raise_on_error=False):
"""
Creates a manifest in a repository, adding all of the necessary data in the model.
The `manifest_interface_instance` parameter must be an instance of the manifest
interface as returned by the image/docker package.
Note that all blobs referenced by the manifest must exist under the repository or this
method will fail and return None.
Returns a reference to the (created manifest, tag) or (None, None) on error, unless
raise_on_error is set to True, in which case a CreateManifestException may also be
raised.
"""
with db_disallow_replica_use():
# Get or create the manifest itself.
created_manifest = oci.manifest.get_or_create_manifest(
repository_ref._db_id,
manifest_interface_instance,
storage,
for_tagging=True,
raise_on_error=raise_on_error,
)
if created_manifest is None:
return (None, None)
# Re-target the tag to it.
tag = oci.tag.retarget_tag(
tag_name,
created_manifest.manifest,
raise_on_error=raise_on_error,
)
if tag is None:
return (None, None)
wrapped_manifest = Manifest.for_manifest(
created_manifest.manifest, self._legacy_image_id_handler)
# Apply any labels that should modify the created tag.
if created_manifest.labels_to_apply:
for key, value in created_manifest.labels_to_apply.items():
apply_label_to_manifest(dict(key=key, value=value),
wrapped_manifest, self)
# Reload the tag in case any updates were applied.
tag = database.Tag.get(id=tag.id)
return (
wrapped_manifest,
Tag.for_tag(tag,
self._legacy_image_id_handler,
manifest_row=created_manifest.manifest),
)
def get_most_recent_tag(self, repository_ref):
"""
Returns the most recently pushed alive tag in the repository, if any.
If none, returns None.
"""
found_tag = oci.tag.get_most_recent_tag(repository_ref._db_id)
assert found_tag is None or not found_tag.hidden
return Tag.for_tag(found_tag, self._legacy_image_id_handler)
def list_all_active_repository_tags(self, repository_ref):
"""
Returns a list of all the active tags in the repository.
Note that this is a *HEAVY* operation on repositories with a lot of tags, and should only be
used for testing or legacy operations.
"""
tags = list(oci.tag.list_alive_tags(repository_ref._db_id))
return [Tag.for_tag(tag, self._legacy_image_id_handler) for tag in tags]
def retarget_tag(
self,
repository_ref,
tag_name,
manifest_or_legacy_image,
storage,
legacy_manifest_key,
is_reversion=False,
):
"""
Creates, updates or moves a tag to a new entry in history, pointing to the manifest or
legacy image specified.
If is_reversion is set to True, this operation is considered a reversion over a previous tag
move operation. Returns the updated Tag or None on error.
"""
with db_disallow_replica_use():
assert legacy_manifest_key is not None
manifest = manifest_or_legacy_image.as_manifest()
manifest_id = manifest._db_id
# If the manifest is a schema 1 manifest and its tag name does not match that
# specified, then we need to create a new manifest, but with that tag name.
if manifest.media_type in DOCKER_SCHEMA1_CONTENT_TYPES:
try:
parsed = manifest.get_parsed_manifest()
except ManifestException:
logger.exception(
"Could not parse manifest `%s` in retarget_tag",
manifest._db_id,
)
return None
if parsed.tag != tag_name:
logger.debug(
"Rewriting manifest `%s` for tag named `%s`",
manifest._db_id,
tag_name,
)
repository_id = repository_ref._db_id
updated = parsed.with_tag_name(tag_name,
legacy_manifest_key)
assert updated.is_signed
created = oci.manifest.get_or_create_manifest(
repository_id, updated, storage)
if created is None:
return None
manifest_id = created.manifest.id
tag = oci.tag.retarget_tag(tag_name,
manifest_id,
is_reversion=is_reversion)
return Tag.for_tag(tag, self._legacy_image_id_handler)
def get_repo_tag(self, repository_ref, tag_name, raise_on_error=True):
"""
Returns the latest, *active* tag found in the repository, with the matching
name or None if none.
If both manifest and tag don't exist, fetches the manifest with the tag
from upstream, and creates them both.
If tag and manifest exists and the manifest is a placeholder, pull the
upstream manifest and save it locally.
"""
db_tag = oci.tag.get_current_tag(repository_ref.id, tag_name)
existing_tag = Tag.for_tag(db_tag, self._legacy_image_id_handler)
if existing_tag is None:
try:
_, tag = self._create_and_tag_manifest(
repository_ref, tag_name,
self._create_manifest_and_retarget_tag)
except (UpstreamRegistryError, ManifestDoesNotExist) as e:
raise TagDoesNotExist(str(e))
return tag
new_tag = False
try:
tag, new_tag = self._update_manifest_for_tag(
repository_ref,
existing_tag,
existing_tag.manifest,
tag_name,
self._create_manifest_and_retarget_tag,
)
except ManifestDoesNotExist as e:
raise TagDoesNotExist(str(e))
except UpstreamRegistryError:
# when the upstream fetch fails, we only return the tag if
# it isn't yet expired. note that we don't bump the tag's
# expiration here either - we only do this when we can ensure
# the tag exists upstream.
isplaceholder = existing_tag.manifest.internal_manifest_bytes.as_unicode(
) == ""
return existing_tag if not existing_tag.expired and not isplaceholder else None
# always bump tag expiration when retrieving tags that both are cached
# and exist upstream, as a means to auto-renew the cache.
if tag.expired or not new_tag:
with db_disallow_replica_use():
new_expiration = (get_epoch_timestamp_ms() +
self._config.expiration_s * 1000
if self._config.expiration_s else None)
oci.tag.set_tag_end_ms(db_tag, new_expiration)
return super().get_repo_tag(repository_ref,
tag_name,
raise_on_error=True)
return tag
def delete_tag(self, repository_ref, tag_name):
"""
Deletes the latest, *active* tag with the given name in the repository.
"""
repo = model.repository.lookup_repository(repository_ref._db_id)
if repo is None:
return None
deleted_tag = model.tag.delete_tag(repo.namespace_user.username,
repo.name, tag_name)
return Tag.for_repository_tag(deleted_tag)
def get_repo_tag(self, repository_ref, tag_name):
"""
Returns the latest, *active* tag found in the repository, with the matching name or None if
none.
"""
assert isinstance(tag_name, str)
tag = oci.tag.get_tag(repository_ref._db_id, tag_name)
if tag is None:
return None
return Tag.for_tag(tag, self._legacy_image_id_handler)
def delete_tag(self, repository_ref, tag_name):
"""
Deletes the latest, *active* tag with the given name in the repository.
"""
deleted_tag = oci.tag.delete_tag(repository_ref._db_id, tag_name)
if deleted_tag is None:
# TODO: This is only needed because preoci raises an exception. Remove and fix
# expected status codes once PreOCIModel is gone.
msg = "Invalid repository tag '%s' on repository" % tag_name
raise DataModelException(msg)
return Tag.for_tag(deleted_tag)
def list_all_active_repository_tags(self,
repository_ref,
include_legacy_images=False):
"""
Returns a list of all the active tags in the repository. Note that this is a *HEAVY*
operation on repositories with a lot of tags, and should only be used for testing or
where other more specific operations are not possible.
"""
if not include_legacy_images:
tags = model.tag.list_active_repo_tags(repository_ref._db_id,
include_images=False)
return [Tag.for_repository_tag(tag) for tag in tags]
tags = model.tag.list_active_repo_tags(repository_ref._db_id)
return [
Tag.for_repository_tag(
tag,
legacy_image=LegacyImage.for_image(tag.image),
manifest_digest=(tag.tagmanifest.digest if hasattr(
tag, 'tagmanifest') else None)) for tag in tags
]
def _get_manifest_id(repositorytag):
repository_tag_datatype = TagDataType.for_repository_tag(repositorytag)
# Retrieve the TagManifest for the RepositoryTag, backfilling if necessary.
with db_transaction():
manifest_datatype = None
try:
manifest_datatype = pre_oci_model.get_manifest_for_tag(
repository_tag_datatype, backfill_if_necessary=True)
except MalformedSchema1Manifest:
logger.exception('Error backfilling manifest for tag `%s`',
repositorytag.id)
if manifest_datatype is None:
logger.error('Could not load or backfill manifest for tag `%s`',
repositorytag.id)
# Create a broken manifest for the tag.
tag_manifest = TagManifest.create(tag=repositorytag,
digest='BROKEN-%s' %
repositorytag.id,
json_data='{}')
else:
# Retrieve the new-style Manifest for the TagManifest, if any.
try:
tag_manifest = TagManifest.get(id=manifest_datatype._db_id)
except TagManifest.DoesNotExist:
logger.exception('Could not find tag manifest')
return None
try:
found = TagManifestToManifest.get(tag_manifest=tag_manifest).manifest
# Verify that the new-style manifest has the same contents as the old-style manifest.
# If not, update and then return. This is an extra check put in place to ensure unicode
# manifests have been correctly copied.
if found.manifest_bytes != tag_manifest.json_data:
logger.warning('Fixing manifest `%s`', found.id)
found.manifest_bytes = tag_manifest.json_data
found.save()
return found.id
except TagManifestToManifest.DoesNotExist:
# Could not find the new style manifest, so backfill.
_backfill_manifest(tag_manifest)
# Try to retrieve the manifest again, since we've performed a backfill.
try:
return TagManifestToManifest.get(tag_manifest=tag_manifest).manifest_id
except TagManifestToManifest.DoesNotExist:
return None
def test_expired_with_tag_lifetime_end_none(self):
now_ms = get_epoch_timestamp_ms()
one_hour_ago_ms = now_ms - 3600 * 1000
tag = Tag(
name="latest",
reversion=False,
manifest_digest="abc123",
lifetime_start_ts=one_hour_ago_ms // 1000,
lifetime_start_ms=one_hour_ago_ms,
lifetime_end_ts=None,
lifetime_end_ms=None,
)
assert not tag.expired
def get_repo_tag(self, repository_ref, tag_name, raise_on_error=False):
"""
Returns the latest, *active* tag found in the repository, with the matching name or None if
none.
"""
assert isinstance(tag_name, str)
tag = oci.tag.get_tag(repository_ref._db_id, tag_name)
if tag is None:
if raise_on_error:
raise model.TagDoesNotExist()
return None
return Tag.for_tag(tag, self._legacy_image_id_handler)
def delete_tags_for_manifest(self, manifest):
"""
Deletes all tags pointing to the given manifest, making the manifest inaccessible for pulling.
Returns the tags deleted, if any. Returns None on error.
"""
try:
tagmanifest = database.TagManifest.get(id=manifest._db_id)
except database.TagManifest.DoesNotExist:
return None
namespace_name = tagmanifest.tag.repository.namespace_user.username
repo_name = tagmanifest.tag.repository.name
tags = model.tag.delete_manifest_by_digest(namespace_name, repo_name,
manifest.digest)
return [Tag.for_repository_tag(tag) for tag in tags]
def list_all_active_repository_tags(self, repository_ref, include_legacy_images=False):
"""
Returns a list of all the active tags in the repository. Note that this is a *HEAVY*
operation on repositories with a lot of tags, and should only be used for testing or
where other more specific operations are not possible.
"""
tags = list(oci.tag.list_alive_tags(repository_ref._db_id))
legacy_images_map = {}
if include_legacy_images:
legacy_images_map = oci.tag.get_legacy_images_for_tags(tags)
return [
Tag.for_tag(tag, legacy_image=LegacyImage.for_image(legacy_images_map.get(tag.id)))
for tag in tags
]
def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False):
"""
Returns the latest, *active* tag found in the repository, with the matching name or None if
none.
"""
assert isinstance(tag_name, basestring)
tag = oci.tag.get_tag(repository_ref._db_id, tag_name)
if tag is None:
return None
legacy_image = None
if include_legacy_image:
legacy_images = oci.tag.get_legacy_images_for_tags([tag])
legacy_image = legacy_images.get(tag.id)
return Tag.for_tag(tag, legacy_image=LegacyImage.for_image(legacy_image))
def _create_manifest_with_temp_tag(
self,
repository_ref: RepositoryReference,
manifest: ManifestInterface,
manifest_ref: str | None = None,
) -> tuple[Manifest | None, Tag | None]:
with db_disallow_replica_use():
with db_transaction():
db_manifest = oci.manifest.create_manifest(
repository_ref.id, manifest)
self._recalculate_repository_size(repository_ref)
expiration = self._config.expiration_s or None
tag = Tag.for_tag(
oci.tag.create_temporary_tag_if_necessary(
db_manifest, expiration),
self._legacy_image_id_handler,
)
wrapped_manifest = Manifest.for_manifest(
db_manifest, self._legacy_image_id_handler)
if not manifest.is_manifest_list:
self._create_placeholder_blobs(manifest, db_manifest.id,
repository_ref.id)
return wrapped_manifest, tag
manifests_to_connect = []
for child in manifest.child_manifests(content_retriever=None):
m = oci.manifest.lookup_manifest(repository_ref.id,
child.digest)
if m is None:
m = oci.manifest.create_manifest(
repository_ref.id, child)
manifests_to_connect.append(m)
oci.manifest.connect_manifests(manifests_to_connect,
db_manifest, repository_ref.id)
for db_manifest in manifests_to_connect:
oci.tag.create_temporary_tag_if_necessary(
db_manifest, expiration)
return wrapped_manifest, tag
def get_repo_tag(self,
repository_ref,
tag_name,
include_legacy_image=False):
"""
Returns the latest, *active* tag found in the repository, with the matching name
or None if none.
"""
assert isinstance(tag_name, basestring)
tag = model.tag.get_active_tag_for_repo(repository_ref._db_id,
tag_name)
if tag is None:
return None
legacy_image = LegacyImage.for_image(
tag.image) if include_legacy_image else None
tag_manifest = model.tag.get_tag_manifest(tag)
manifest_digest = tag_manifest.digest if tag_manifest else None
return Tag.for_repository_tag(tag,
legacy_image=legacy_image,
manifest_digest=manifest_digest)
def list_repository_tag_history(
self,
repository_ref,
page=1,
size=100,
specific_tag_name=None,
active_tags_only=False,
since_time_ms=None,
):
"""
Returns the history of all tags in the repository (unless filtered).
This includes tags that have been made in-active due to newer versions of those tags coming
into service.
"""
tags, has_more = oci.tag.list_repository_tag_history(
repository_ref._db_id, page, size, specific_tag_name,
active_tags_only, since_time_ms)
# TODO: Remove this once the layers compressed sizes have been fully backfilled.
tags_missing_sizes = [
tag for tag in tags if tag.manifest.layers_compressed_size is None
]
legacy_images_map = {}
if tags_missing_sizes:
legacy_images_map = oci.tag.get_legacy_images_for_tags(
tags_missing_sizes)
return (
[
Tag.for_tag(
tag,
self._legacy_image_id_handler,
legacy_image_row=legacy_images_map.get(tag.id),
) for tag in tags
],
has_more,
)
def create_manifest_and_retarget_tag(
self, repository_ref, manifest_interface_instance, tag_name, storage, raise_on_error=False
):
"""
Creates a manifest in a repository, adding all of the necessary data in the model.
The `manifest_interface_instance` parameter must be an instance of the manifest
interface as returned by the image/docker package.
Note that all blobs referenced by the manifest must exist under the repository or this
method will fail and return None.
Returns a reference to the (created manifest, tag) or (None, None) on error, unless
raise_on_error is set to True, in which case a CreateManifestException may also be
raised.
"""
with db_disallow_replica_use():
# Get or create the manifest itself.
created_manifest = oci.manifest.get_or_create_manifest(
repository_ref._db_id,
manifest_interface_instance,
storage,
for_tagging=True,
raise_on_error=raise_on_error,
)
if created_manifest is None:
return (None, None)
wrapped_manifest = Manifest.for_manifest(
created_manifest.manifest, self._legacy_image_id_handler
)
# Optional expiration label
# NOTE: Since there is currently only one special label on a manifest that has an effect on its tags (expiration),
# it is just simpler to set that value at tag creation time (plus it saves an additional query).
# If we were to define more of these "special" labels in the future, we should use the handlers from
# data/registry_model/label_handlers.py
if not created_manifest.newly_created:
label_dict = next(
(
label.asdict()
for label in self.list_manifest_labels(
wrapped_manifest,
key_prefix="quay",
)
if label.key == LABEL_EXPIRY_KEY
),
None,
)
else:
label_dict = next(
(
dict(key=label_key, value=label_value)
for label_key, label_value in created_manifest.labels_to_apply.items()
if label_key == LABEL_EXPIRY_KEY
),
None,
)
expiration_seconds = None
if label_dict is not None:
try:
expiration_td = convert_to_timedelta(label_dict["value"])
expiration_seconds = expiration_td.total_seconds()
except ValueError:
pass
# Re-target the tag to it.
tag = oci.tag.retarget_tag(
tag_name,
created_manifest.manifest,
raise_on_error=raise_on_error,
expiration_seconds=expiration_seconds,
)
if tag is None:
return (None, None)
return (
wrapped_manifest,
Tag.for_tag(
tag, self._legacy_image_id_handler, manifest_row=created_manifest.manifest
),
)
def create_manifest_and_retarget_tag(self,
repository_ref,
manifest_interface_instance,
tag_name,
storage,
raise_on_error=False):
""" Creates a manifest in a repository, adding all of the necessary data in the model.
The `manifest_interface_instance` parameter must be an instance of the manifest
interface as returned by the image/docker package.
Note that all blobs referenced by the manifest must exist under the repository or this
method will fail and return None.
Returns a reference to the (created manifest, tag) or (None, None) on error.
"""
# NOTE: Only Schema1 is supported by the pre_oci_model.
assert isinstance(manifest_interface_instance, DockerSchema1Manifest)
if not manifest_interface_instance.layers:
return None, None
# Ensure all the blobs in the manifest exist.
digests = manifest_interface_instance.checksums
query = self._lookup_repo_storages_by_content_checksum(
repository_ref._db_id, digests)
blob_map = {s.content_checksum: s for s in query}
for layer in manifest_interface_instance.layers:
digest_str = str(layer.digest)
if digest_str not in blob_map:
return None, None
# Lookup all the images and their parent images (if any) inside the manifest.
# This will let us know which v1 images we need to synthesize and which ones are invalid.
docker_image_ids = list(manifest_interface_instance.legacy_image_ids)
images_query = model.image.lookup_repository_images(
repository_ref._db_id, docker_image_ids)
image_storage_map = {
i.docker_image_id: i.storage
for i in images_query
}
# Rewrite any v1 image IDs that do not match the checksum in the database.
try:
rewritten_images = manifest_interface_instance.rewrite_invalid_image_ids(
image_storage_map)
rewritten_images = list(rewritten_images)
parent_image_map = {}
for rewritten_image in rewritten_images:
if not rewritten_image.image_id in image_storage_map:
parent_image = None
if rewritten_image.parent_image_id:
parent_image = parent_image_map.get(
rewritten_image.parent_image_id)
if parent_image is None:
parent_image = model.image.get_image(
repository_ref._db_id,
rewritten_image.parent_image_id)
if parent_image is None:
return None, None
synthesized = model.image.synthesize_v1_image(
repository_ref._db_id,
blob_map[rewritten_image.content_checksum].id,
blob_map[rewritten_image.content_checksum].image_size,
rewritten_image.image_id,
rewritten_image.created,
rewritten_image.comment,
rewritten_image.command,
rewritten_image.compat_json,
parent_image,
)
parent_image_map[rewritten_image.image_id] = synthesized
except ManifestException:
logger.exception("exception when rewriting v1 metadata")
return None, None
# Store the manifest pointing to the tag.
leaf_layer_id = rewritten_images[-1].image_id
tag_manifest, newly_created = model.tag.store_tag_manifest_for_repo(
repository_ref._db_id, tag_name, manifest_interface_instance,
leaf_layer_id, blob_map)
manifest = Manifest.for_tag_manifest(tag_manifest)
# Save the labels on the manifest.
repo_tag = tag_manifest.tag
if newly_created:
has_labels = False
with self.batch_create_manifest_labels(manifest) as add_label:
if add_label is None:
return None, None
for key, value in manifest_interface_instance.layers[
-1].v1_metadata.labels.iteritems():
media_type = 'application/json' if is_json(
value) else 'text/plain'
add_label(key, value, 'manifest', media_type)
has_labels = True
# Reload the tag in case any updates were applied.
if has_labels:
repo_tag = database.RepositoryTag.get(id=repo_tag.id)
return manifest, Tag.for_repository_tag(repo_tag)
def retarget_tag(
self,
repository_ref,
tag_name,
manifest_or_legacy_image,
storage,
legacy_manifest_key,
is_reversion=False,
):
"""
Creates, updates or moves a tag to a new entry in history, pointing to the manifest or
legacy image specified.
If is_reversion is set to True, this operation is considered a reversion over a previous tag
move operation. Returns the updated Tag or None on error.
"""
with db_disallow_replica_use():
assert legacy_manifest_key is not None
manifest = manifest_or_legacy_image.as_manifest()
manifest_id = manifest._db_id
# If the manifest is a schema 1 manifest and its tag name does not match that
# specified, then we need to create a new manifest, but with that tag name.
if manifest.media_type in DOCKER_SCHEMA1_CONTENT_TYPES:
try:
parsed = manifest.get_parsed_manifest()
except ManifestException:
logger.exception(
"Could not parse manifest `%s` in retarget_tag",
manifest._db_id,
)
return None
if parsed.tag != tag_name:
logger.debug(
"Rewriting manifest `%s` for tag named `%s`",
manifest._db_id,
tag_name,
)
repository_id = repository_ref._db_id
updated = parsed.with_tag_name(tag_name, legacy_manifest_key)
assert updated.is_signed
created = oci.manifest.get_or_create_manifest(repository_id, updated, storage)
if created is None:
return None
manifest_id = created.manifest.id
label_dict = next(
(
label.asdict()
for label in self.list_manifest_labels(
manifest,
key_prefix="quay",
)
if label.key == LABEL_EXPIRY_KEY
),
None,
)
expiration_seconds = None
if label_dict is not None:
try:
expiration_td = convert_to_timedelta(label_dict["value"])
expiration_seconds = expiration_td.total_seconds()
except ValueError:
pass
tag = oci.tag.retarget_tag(
tag_name,
manifest_id,
is_reversion=is_reversion,
expiration_seconds=expiration_seconds,
)
return Tag.for_tag(tag, self._legacy_image_id_handler)
def retarget_tag(
self,
repository_ref,
tag_name,
manifest_or_legacy_image,
storage,
legacy_manifest_key,
is_reversion=False,
):
"""
Creates, updates or moves a tag to a new entry in history, pointing to the manifest or
legacy image specified.
If is_reversion is set to True, this operation is considered a reversion over a previous tag
move operation. Returns the updated Tag or None on error.
"""
assert legacy_manifest_key is not None
manifest_id = manifest_or_legacy_image._db_id
if isinstance(manifest_or_legacy_image, LegacyImage):
# If a legacy image was required, build a new manifest for it and move the tag to that.
try:
image_row = database.Image.get(id=manifest_or_legacy_image._db_id)
except database.Image.DoesNotExist:
return None
manifest_instance = self._build_manifest_for_legacy_image(tag_name, image_row)
if manifest_instance is None:
return None
created = oci.manifest.get_or_create_manifest(
repository_ref._db_id, manifest_instance, storage
)
if created is None:
return None
manifest_id = created.manifest.id
else:
# If the manifest is a schema 1 manifest and its tag name does not match that
# specified, then we need to create a new manifest, but with that tag name.
if manifest_or_legacy_image.media_type in DOCKER_SCHEMA1_CONTENT_TYPES:
try:
parsed = manifest_or_legacy_image.get_parsed_manifest()
except ManifestException:
logger.exception(
"Could not parse manifest `%s` in retarget_tag",
manifest_or_legacy_image._db_id,
)
return None
if parsed.tag != tag_name:
logger.debug(
"Rewriting manifest `%s` for tag named `%s`",
manifest_or_legacy_image._db_id,
tag_name,
)
repository_id = repository_ref._db_id
updated = parsed.with_tag_name(tag_name, legacy_manifest_key)
assert updated.is_signed
created = oci.manifest.get_or_create_manifest(repository_id, updated, storage)
if created is None:
return None
manifest_id = created.manifest.id
tag = oci.tag.retarget_tag(tag_name, manifest_id, is_reversion=is_reversion)
legacy_image = LegacyImage.for_image(oci.shared.get_legacy_image_for_manifest(manifest_id))
return Tag.for_tag(tag, legacy_image)
def lookup_manifest_by_digest(
self,
repository_ref,
manifest_digest,
allow_dead=False,
require_available=False,
raise_on_error=True,
):
"""
Looks up the manifest with the given digest under the given repository and returns it or
None if none.
If a manifest with the digest does not exist, fetches the manifest upstream
and creates it with a temp tag.
"""
wrapped_manifest = super().lookup_manifest_by_digest(
repository_ref,
manifest_digest,
allow_dead=True,
require_available=False)
if wrapped_manifest is None:
try:
wrapped_manifest, _ = self._create_and_tag_manifest(
repository_ref, manifest_digest,
self._create_manifest_with_temp_tag)
except (UpstreamRegistryError, ManifestDoesNotExist) as e:
raise ManifestDoesNotExist(str(e))
return wrapped_manifest
db_tag = oci.tag.get_tag_by_manifest_id(repository_ref.id,
wrapped_manifest.id)
existing_tag = Tag.for_tag(db_tag,
self._legacy_image_id_handler,
manifest_row=db_tag.manifest)
new_tag = False
try:
tag, new_tag = self._update_manifest_for_tag(
repository_ref,
existing_tag,
existing_tag.manifest,
manifest_digest,
self._create_manifest_with_temp_tag,
)
except ManifestDoesNotExist as e:
raise e
except UpstreamRegistryError:
# when the upstream fetch fails, we only return the tag if
# it isn't yet expired. note that we don't bump the tag's
# expiration here either - we only do this when we can ensure
# the tag exists upstream.
isplaceholder = wrapped_manifest.internal_manifest_bytes.as_unicode(
) == ""
return wrapped_manifest if not existing_tag.expired and not isplaceholder else None
if tag.expired or not new_tag:
with db_disallow_replica_use():
new_expiration = (get_epoch_timestamp_ms() +
self._config.expiration_s * 1000
if self._config.expiration_s else None)
oci.tag.set_tag_end_ms(db_tag, new_expiration)
# if the manifest is a child of a manifest list in this repo, renew
# the parent manifest list tag too.
parent = ManifestChild.select(ManifestChild.manifest_id).where(
(ManifestChild.repository_id == repository_ref.id)
& (ManifestChild.child_manifest_id == wrapped_manifest.id))
parent_tag = oci.tag.get_tag_by_manifest_id(
repository_ref.id, parent)
if parent_tag is not None:
oci.tag.set_tag_end_ms(parent_tag, new_expiration)
return super().lookup_manifest_by_digest(
repository_ref,
manifest_digest,
allow_dead=True,
require_available=False,
raise_on_error=True,
)
def _update_manifest_for_tag(
self,
repo_ref: RepositoryReference,
tag: Tag,
manifest: Manifest,
manifest_ref: str,
create_manifest_fn,
) -> tuple[Tag, bool]:
"""
Updates a placeholder manifest with the given tag name.
If the manifest is stale, downloads it from the upstream registry
and creates a new manifest and retargets the tag.
A manifest is considered stale when the manifest's digest changed in
the upstream registry.
A manifest is considered a placeholder when its db entry exists, but
its manifest_bytes field is empty.
Raises UpstreamRegistryError if the upstream registry returns anything
other than 200.
Raises ManifestDoesNotExist if the given manifest was not found in the
database.
Returns a new tag if one was created, or the existing one with a manifest
freshly out of the database, and a boolean indicating whether the returned
tag was newly created or not.
"""
upstream_manifest = None
upstream_digest = self._proxy.manifest_exists(manifest_ref,
ACCEPTED_MEDIA_TYPES)
up_to_date = manifest.digest == upstream_digest
# manifest_exists will return an empty/None digest when the upstream
# registry omits the docker-content-digest header.
if not upstream_digest:
upstream_manifest = self._pull_upstream_manifest(
repo_ref.name, manifest_ref)
up_to_date = manifest.digest == upstream_manifest.digest
placeholder = manifest.internal_manifest_bytes.as_unicode() == ""
if up_to_date and not placeholder:
return tag, False
if upstream_manifest is None:
upstream_manifest = self._pull_upstream_manifest(
repo_ref.name, manifest_ref)
self._enforce_repository_quota(repo_ref)
if up_to_date and placeholder:
with db_disallow_replica_use():
with db_transaction():
q = ManifestTable.update(
manifest_bytes=upstream_manifest.bytes.as_unicode(),
layers_compressed_size=upstream_manifest.
layers_compressed_size,
).where(ManifestTable.id == manifest.id)
q.execute()
self._create_placeholder_blobs(upstream_manifest,
manifest.id, repo_ref.id)
db_tag = oci.tag.get_tag_by_manifest_id(
repo_ref.id, manifest.id)
self._recalculate_repository_size(repo_ref)
return Tag.for_tag(db_tag,
self._legacy_image_id_handler), False
# if we got here, the manifest is stale, so we both create a new manifest
# entry in the db, and retarget the tag.
_, tag = create_manifest_fn(repo_ref, upstream_manifest, manifest_ref)
return tag, True
def _create_manifest_and_retarget_tag(
self, repository_ref: RepositoryReference,
manifest: ManifestInterface,
tag_name: str) -> tuple[Manifest | None, Tag | None]:
"""
Creates a manifest in the given repository.
Also creates placeholders for the objects referenced by the manifest.
For manifest lists, creates placeholder sub-manifests. For regular
manifests, creates placeholder blobs.
Placeholder objects will be "filled" with the objects' contents on
upcoming client requests, as part of the flow described in the OCI
distribution specification.
Returns a reference to the (created manifest, tag) or (None, None) on error.
"""
with db_disallow_replica_use():
with db_transaction():
db_manifest = oci.manifest.lookup_manifest(repository_ref.id,
manifest.digest,
allow_dead=True)
if db_manifest is None:
db_manifest = oci.manifest.create_manifest(
repository_ref.id, manifest, raise_on_error=True)
self._recalculate_repository_size(repository_ref)
if db_manifest is None:
return None, None
# 0 means a tag never expires - if we get 0 as expiration,
# we set the tag expiration to None.
expiration = self._config.expiration_s or None
tag = oci.tag.retarget_tag(
tag_name,
db_manifest,
raise_on_error=True,
expiration_seconds=expiration,
)
if tag is None:
return None, None
wrapped_manifest = Manifest.for_manifest(
db_manifest, self._legacy_image_id_handler)
wrapped_tag = Tag.for_tag(tag,
self._legacy_image_id_handler,
manifest_row=db_manifest)
if not manifest.is_manifest_list:
self._create_placeholder_blobs(manifest, db_manifest.id,
repository_ref.id)
return wrapped_manifest, wrapped_tag
manifests_to_connect = []
for child in manifest.child_manifests(content_retriever=None):
m = oci.manifest.lookup_manifest(repository_ref.id,
child.digest,
allow_dead=True)
if m is None:
m = oci.manifest.create_manifest(
repository_ref.id, child)
oci.tag.create_temporary_tag_if_necessary(
m, self._config.expiration_s or None)
try:
ManifestChild.get(manifest=db_manifest.id,
child_manifest=m.id)
except ManifestChild.DoesNotExist:
manifests_to_connect.append(m)
oci.manifest.connect_manifests(manifests_to_connect,
db_manifest, repository_ref.id)
return wrapped_manifest, wrapped_tag
