def test_lookup_manifest_dead_tag(initialized_db):
dead_tag = Tag.select().where(
Tag.lifetime_end_ms <= get_epoch_timestamp_ms()).get()
assert dead_tag.lifetime_end_ms <= get_epoch_timestamp_ms()
assert lookup_manifest(dead_tag.repository,
dead_tag.manifest.digest) is None
assert (lookup_manifest(dead_tag.repository,
dead_tag.manifest.digest,
allow_dead=True) == dead_tag.manifest)
def find_repository_with_garbage(limit_to_gc_policy_s):
""" Returns a repository that has garbage (defined as an expired Tag that is past
the repo's namespace's expiration window) or None if none.
"""
expiration_timestamp = get_epoch_timestamp_ms() - (limit_to_gc_policy_s *
1000)
try:
candidates = (Tag.select(Tag.repository).join(Repository).join(
Namespace, on=(Repository.namespace_user == Namespace.id)).where(
~(Tag.lifetime_end_ms >> None),
(Tag.lifetime_end_ms <= expiration_timestamp),
(Namespace.removed_tag_expiration_s == limit_to_gc_policy_s),
(Namespace.enabled == True),
(Repository.state != RepositoryState.MARKED_FOR_DELETION),
).limit(GC_CANDIDATE_COUNT).distinct().alias("candidates"))
found = (Tag.select(
candidates.c.repository_id).from_(candidates).order_by(
db_random_func()).get())
if found is None:
return
return Repository.get(Repository.id == found.repository_id)
except Tag.DoesNotExist:
return None
except Repository.DoesNotExist:
return None
def test_returns_None_when_manifest_no_longer_exists_upstream_and_local_cache_is_expired(
self, create_repo, proxy_manifest_response):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
proxy_mock = proxy_manifest_response(
self.tag, UBI8_8_5_MANIFEST_SCHEMA2,
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
tag = proxy_model.get_repo_tag(repo_ref, self.tag)
assert tag is not None
# expire the tag by setting start and end time to the past
before_ms = get_epoch_timestamp_ms() - timedelta(
hours=24).total_seconds() * 1000
Tag.update(
lifetime_start_ms=before_ms,
lifetime_end_ms=before_ms + 5,
).where(Tag.id == tag.id).execute()
proxy_mock = proxy_manifest_response("not-existing-ref", "", "")
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
tag = proxy_model.get_repo_tag(repo_ref, self.tag)
assert tag is None
def cache_namespace_repository_sizes(namespace_name):
namespace = user.get_user_or_org(namespace_name)
now_ms = get_epoch_timestamp_ms()
subquery = (Tag.select(Tag.repository_id).where(
Tag.hidden == False).where((Tag.lifetime_end_ms >> None)
| (Tag.lifetime_end_ms > now_ms)).group_by(
Tag.repository_id).having(
fn.Count(Tag.name) > 0))
namespace_repo_sizes = (Manifest.select(
(Repository.id).alias("repository_id"),
(Repository.name).alias("repository_name"),
fn.sum(Manifest.layers_compressed_size).alias("repository_size"),
).join(Repository).join(
subquery, on=(subquery.c.repository_id == Repository.id)).where(
Repository.namespace_user == namespace.id).group_by(Repository.id))
insert_query = (namespace_repo_sizes.select(
Repository.id, fn.sum(Manifest.layers_compressed_size)).join_from(
Repository, RepositorySize,
JOIN.LEFT_OUTER).where(RepositorySize.repository_id.is_null()))
RepositorySize.insert_from(
insert_query,
fields=[RepositorySize.repository_id, RepositorySize.size_bytes],
).execute()
def create_temporary_tag_if_necessary(manifest, expiration_sec):
"""
Creates a temporary tag pointing to the given manifest, with the given expiration in seconds,
unless there is an existing tag that will keep the manifest around.
"""
tag_name = "$temp-%s" % str(uuid.uuid4())
now_ms = get_epoch_timestamp_ms()
end_ms = now_ms + (expiration_sec * 1000)
# Check if there is an existing tag on the manifest that won't expire within the
# timeframe. If so, no need for a temporary tag.
with db_transaction():
try:
(Tag.select().where(
Tag.manifest == manifest,
(Tag.lifetime_end_ms >> None) |
(Tag.lifetime_end_ms >= end_ms),
).get())
return None
except Tag.DoesNotExist:
pass
return Tag.create(
name=tag_name,
repository=manifest.repository_id,
lifetime_start_ms=now_ms,
lifetime_end_ms=end_ms,
reversion=False,
hidden=True,
manifest=manifest,
tag_kind=Tag.tag_kind.get_id("tag"),
)
def delete_tag(repo, tag_name, models_ref, tag_kind="release"):
Tag = models_ref.Tag
tag_kind_id = Tag.tag_kind.get_id(tag_kind)
tag = tag_is_alive(
Tag.select().where(Tag.repository == repo, Tag.name == tag_name,
Tag.tag_kind == tag_kind_id), Tag).get()
tag.lifetime_end = get_epoch_timestamp_ms()
tag.save()
return tag
def delete_tag(repository_id, tag_name):
""" Deletes the alive tag with the given name in the specified repository and returns the deleted
tag. If the tag did not exist, returns None.
"""
tag = get_tag(repository_id, tag_name)
if tag is None:
return None
return _delete_tag(tag, get_epoch_timestamp_ms())
def lookup_unrecoverable_tags(repo):
""" Returns the tags in a repository that are expired and past their time machine recovery
period. """
expired_clause = get_epoch_timestamp_ms() - (
Namespace.removed_tag_expiration_s * 1000)
return (Tag.select().join(Repository).join(
Namespace, on=(Repository.namespace_user == Namespace.id)).where(
Tag.repository == repo).where(
~(Tag.lifetime_end_ms >> None),
Tag.lifetime_end_ms <= expired_clause))
def get_expired_tag(repository_id, tag_name):
""" Returns a tag with the given name that is expired in the repository or None if none.
"""
try:
return (Tag.select().where(
Tag.name == tag_name, Tag.repository == repository_id).where(
~(Tag.lifetime_end_ms >> None)).where(
Tag.lifetime_end_ms <= get_epoch_timestamp_ms()).get())
except Tag.DoesNotExist:
return None
def get_repo_tag(self, repository_ref, tag_name, raise_on_error=True):
"""
Returns the latest, *active* tag found in the repository, with the matching
name or None if none.
If both manifest and tag don't exist, fetches the manifest with the tag
from upstream, and creates them both.
If tag and manifest exists and the manifest is a placeholder, pull the
upstream manifest and save it locally.
"""
db_tag = oci.tag.get_current_tag(repository_ref.id, tag_name)
existing_tag = Tag.for_tag(db_tag, self._legacy_image_id_handler)
if existing_tag is None:
try:
_, tag = self._create_and_tag_manifest(
repository_ref, tag_name,
self._create_manifest_and_retarget_tag)
except (UpstreamRegistryError, ManifestDoesNotExist) as e:
raise TagDoesNotExist(str(e))
return tag
new_tag = False
try:
tag, new_tag = self._update_manifest_for_tag(
repository_ref,
existing_tag,
existing_tag.manifest,
tag_name,
self._create_manifest_and_retarget_tag,
)
except ManifestDoesNotExist as e:
raise TagDoesNotExist(str(e))
except UpstreamRegistryError:
# when the upstream fetch fails, we only return the tag if
# it isn't yet expired. note that we don't bump the tag's
# expiration here either - we only do this when we can ensure
# the tag exists upstream.
isplaceholder = existing_tag.manifest.internal_manifest_bytes.as_unicode(
) == ""
return existing_tag if not existing_tag.expired and not isplaceholder else None
# always bump tag expiration when retrieving tags that both are cached
# and exist upstream, as a means to auto-renew the cache.
if tag.expired or not new_tag:
with db_disallow_replica_use():
new_expiration = (get_epoch_timestamp_ms() +
self._config.expiration_s * 1000
if self._config.expiration_s else None)
oci.tag.set_tag_end_ms(db_tag, new_expiration)
return super().get_repo_tag(repository_ref,
tag_name,
raise_on_error=True)
return tag
def filter_to_alive_tags(query, now_ms=None, model=Tag):
""" Adjusts the specified Tag query to only return those tags alive. If now_ms is specified,
the given timestamp (in MS) is used in place of the current timestamp for determining wherther
a tag is alive.
"""
if now_ms is None:
now_ms = get_epoch_timestamp_ms()
return (query.where((model.lifetime_end_ms >> None)
| (model.lifetime_end_ms > now_ms)).where(
model.hidden == False))
def test_renews_expired_tag_when_manifest_is_up_to_date_with_upstream(
self, create_repo, proxy_manifest_response):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
proxy_mock = proxy_manifest_response(
self.tag, UBI8_8_5_MANIFEST_SCHEMA2,
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
tag = proxy_model.get_repo_tag(repo_ref, self.tag)
assert tag is not None
assert tag.name == self.tag
# expire the tag by setting start and end time to the past
before_ms = get_epoch_timestamp_ms() - timedelta(
hours=24).total_seconds() * 1000
Tag.update(
lifetime_start_ms=before_ms,
lifetime_end_ms=before_ms + 5,
).where(Tag.id == tag.id).execute()
expired_tag = tag
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
tag = proxy_model.get_repo_tag(repo_ref, self.tag)
assert tag is not None
assert expired_tag.id == tag.id
assert expired_tag.manifest.id == tag.manifest.id
assert not tag.expired
new_expiration_ms = get_epoch_timestamp_ms(
) + self.config.expiration_s * 1000
# subtract a some milliseconds so the test doesn't flake
assert tag.lifetime_end_ms >= new_expiration_ms - 500
def test_expired_with_tag_lifetime_end_none(self):
now_ms = get_epoch_timestamp_ms()
one_hour_ago_ms = now_ms - 3600 * 1000
tag = Tag(
name="latest",
reversion=False,
manifest_digest="abc123",
lifetime_start_ts=one_hour_ago_ms // 1000,
lifetime_start_ms=one_hour_ago_ms,
lifetime_end_ts=None,
lifetime_end_ms=None,
)
assert not tag.expired
def test_renew_tag_when_cache_is_expired_and_manifest_is_up_to_date_with_upstream(
self, create_repo, proxy_manifest_response):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
proxy_mock = proxy_manifest_response(
UBI8_8_4_DIGEST, UBI8_8_4_MANIFEST_SCHEMA2,
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest = proxy_model.lookup_manifest_by_digest(
repo_ref, UBI8_8_4_DIGEST)
assert manifest is not None
before_ms = get_epoch_timestamp_ms() - timedelta(
hours=24).total_seconds() * 1000
Tag.update(
lifetime_start_ms=before_ms,
lifetime_end_ms=before_ms + 5,
).where(Tag.manifest == manifest.id).execute()
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest = proxy_model.lookup_manifest_by_digest(
repo_ref, UBI8_8_4_DIGEST)
assert manifest is not None
tag = Tag.get(manifest_id=manifest.id)
now_ms = get_epoch_timestamp_ms()
assert tag.lifetime_end_ms > now_ms
def delete_manifest_by_digest(namespace, repo_name, digest):
tag_manifests = list(
_load_repo_manifests(namespace,
repo_name).where(TagManifest.digest == digest))
now_ms = get_epoch_timestamp_ms()
for tag_manifest in tag_manifests:
try:
tag = _tag_alive(RepositoryTag.select().where(
RepositoryTag.id == tag_manifest.tag_id)).get()
delete_tag(namespace, repo_name, tag_manifest.tag.name, now_ms)
except RepositoryTag.DoesNotExist:
pass
return [tag_manifest.tag for tag_manifest in tag_manifests]
def delete_tags_for_manifest(manifest):
""" Deletes all tags pointing to the given manifest. Returns the list of tags
deleted.
"""
query = Tag.select().where(Tag.manifest == manifest)
query = filter_to_alive_tags(query)
query = filter_to_visible_tags(query)
tags = list(query)
now_ms = get_epoch_timestamp_ms()
with db_transaction():
for tag in tags:
_delete_tag(tag, now_ms)
return tags
def get_namespace_size(namespace_name):
namespace = user.get_user_or_org(namespace_name)
now_ms = get_epoch_timestamp_ms()
subquery = (Tag.select(Tag.repository_id).where(
Tag.hidden == False).where((Tag.lifetime_end_ms >> None)
| (Tag.lifetime_end_ms > now_ms)).group_by(
Tag.repository_id).having(
fn.Count(Tag.name) > 0))
namespace_size = (Manifest.select(fn.sum(
Manifest.layers_compressed_size)).join(Repository).join(
subquery, on=(subquery.c.repository_id == Repository.id)).where(
Repository.namespace_user == namespace.id)).scalar()
return namespace_size or 0
def create_or_update_tag(repo,
tag_name,
models_ref,
manifest_list=None,
linked_tag=None,
tag_kind="release"):
Tag = models_ref.Tag
now_ts = get_epoch_timestamp_ms()
tag_kind_id = Tag.tag_kind.get_id(tag_kind)
with db_transaction():
try:
tag = db_for_update(
tag_is_alive(
Tag.select().where(Tag.repository == repo,
Tag.name == tag_name,
Tag.tag_kind == tag_kind_id),
Tag,
now_ts,
)).get()
if tag.manifest_list == manifest_list and tag.linked_tag == linked_tag:
return tag
tag.lifetime_end = now_ts
tag.save()
except Tag.DoesNotExist:
pass
try:
return Tag.create(
repository=repo,
manifest_list=manifest_list,
linked_tag=linked_tag,
name=tag_name,
lifetime_start=now_ts,
lifetime_end=None,
tag_kind=tag_kind_id,
)
except IntegrityError:
msg = "Tag with name %s and lifetime start %s under repository %s/%s already exists"
raise TagAlreadyCreatedException(
msg % (tag_name, now_ts, repo.namespace_user, repo.name))
def delete_tag(namespace_name, repository_name, tag_name, now_ms=None):
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
try:
query = _tag_alive(
RepositoryTag.select(
RepositoryTag, Repository).join(Repository).join(
Namespace,
on=(Repository.namespace_user == Namespace.id)).where(
Repository.name == repository_name,
Namespace.username == namespace_name,
RepositoryTag.name == tag_name,
),
now_ts,
)
found = db_for_update(query).get()
except RepositoryTag.DoesNotExist:
msg = "Invalid repository tag '%s' on repository '%s/%s'" % (
tag_name,
namespace_name,
repository_name,
)
raise DataModelException(msg)
found.lifetime_end_ts = now_ts
found.save()
try:
oci_tag_query = TagToRepositoryTag.select().where(
TagToRepositoryTag.repository_tag == found)
oci_tag = db_for_update(oci_tag_query).get().tag
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except TagToRepositoryTag.DoesNotExist:
pass
return found
def __create_manifest_and_tags(repo,
structure,
creator_username,
tag_map,
current_level=0,
builder=None,
last_leaf_id=None):
num_layers, subtrees, tag_names = structure
num_layers = num_layers or 1
tag_names = tag_names or []
tag_names = [tag_names] if not isinstance(tag_names, list) else tag_names
repo_ref = RepositoryReference.for_repo_obj(repo)
builder = (builder if builder else DockerSchema1ManifestBuilder(
repo.namespace_user.username, repo.name, ""))
# TODO: Change this to a mixture of Schema1 and Schema2 manifest once we no longer need to
# read from storage for Schema2.
# Populate layers. Note, we do this in reverse order using insert_layer, as it is easier to
# add the leaf last (even though Schema1 has it listed first).
parent_id = last_leaf_id
leaf_id = None
for layer_index in range(0, num_layers):
content = "layer-%s-%s-%s" % (layer_index, current_level,
get_epoch_timestamp_ms())
_, digest = _populate_blob(repo, content.encode("ascii"))
current_id = "abcdef%s%s%s" % (layer_index, current_level,
get_epoch_timestamp_ms())
if layer_index == num_layers - 1:
leaf_id = current_id
config = {
"id": current_id,
"Size": len(content),
}
if parent_id:
config["parent"] = parent_id
builder.insert_layer(digest, json.dumps(config))
parent_id = current_id
for tag_name in tag_names:
adjusted_tag_name = tag_name
now = datetime.utcnow()
if tag_name[0] == "#":
adjusted_tag_name = tag_name[1:]
now = now - timedelta(seconds=1)
manifest = builder.clone(adjusted_tag_name).build()
with freeze_time(now):
created_tag, _ = registry_model.create_manifest_and_retarget_tag(
repo_ref,
manifest,
adjusted_tag_name,
store,
raise_on_error=True)
assert created_tag
tag_map[adjusted_tag_name] = created_tag
for subtree in subtrees:
__create_manifest_and_tags(
repo,
subtree,
creator_username,
tag_map,
current_level=current_level + 1,
builder=builder,
last_leaf_id=leaf_id,
)
def create_or_update_tag_for_repo(repository_id,
tag_name,
tag_docker_image_id,
reversion=False,
oci_manifest=None,
now_ms=None):
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
try:
tag = db_for_update(
_tag_alive(
RepositoryTag.select().where(
RepositoryTag.repository == repository_id,
RepositoryTag.name == tag_name),
now_ts,
)).get()
tag.lifetime_end_ts = now_ts
tag.save()
# Check for an OCI tag.
try:
oci_tag = db_for_update(
Tag.select().join(TagToRepositoryTag).where(
TagToRepositoryTag.repository_tag == tag)).get()
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except Tag.DoesNotExist:
pass
except RepositoryTag.DoesNotExist:
pass
except IntegrityError:
msg = "Tag with name %s was stale when we tried to update it; Please retry the push"
raise StaleTagException(msg % tag_name)
try:
image_obj = Image.get(Image.docker_image_id == tag_docker_image_id,
Image.repository == repository_id)
except Image.DoesNotExist:
raise DataModelException("Invalid image with id: %s" %
tag_docker_image_id)
try:
created = RepositoryTag.create(
repository=repository_id,
image=image_obj,
name=tag_name,
lifetime_start_ts=now_ts,
reversion=reversion,
)
if oci_manifest:
# Create the OCI tag as well.
oci_tag = Tag.create(
repository=repository_id,
manifest=oci_manifest,
name=tag_name,
lifetime_start_ms=now_ms,
reversion=reversion,
tag_kind=Tag.tag_kind.get_id("tag"),
)
TagToRepositoryTag.create(tag=oci_tag,
repository_tag=created,
repository=repository_id)
return created
except IntegrityError:
msg = "Tag with name %s and lifetime start %s already exists"
raise TagAlreadyCreatedException(msg % (tag_name, now_ts))
def __create_subtree(with_storage, repo, structure, creator_username, parent,
tag_map):
num_nodes, subtrees, last_node_tags = structure
# create the nodes
for model_num in range(num_nodes):
image_num = next(global_image_num)
docker_image_id = __gen_image_id(repo, image_num)
logger.debug("new docker id: %s", docker_image_id)
checksum = __gen_checksum(docker_image_id)
new_image = model.image.find_create_or_link_image(
docker_image_id, repo, None, {}, "local_us")
new_image.storage.uuid = __gen_image_uuid(repo, image_num)
new_image.storage.uploading = False
new_image.storage.save()
# Write out a fake torrentinfo
model.storage.save_torrent_info(new_image.storage, 1, "deadbeef")
# Write some data for the storage.
if with_storage or os.environ.get("WRITE_STORAGE_FILES"):
storage_paths = StoragePaths()
paths = [storage_paths.v1_image_layer_path]
for path_builder in paths:
path = path_builder(new_image.storage.uuid)
store.put_content("local_us", path, checksum)
new_image.security_indexed = False
new_image.security_indexed_engine = -1
new_image.save()
creation_time = REFERENCE_DATE + timedelta(
weeks=image_num) + timedelta(days=model_num)
command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)]
command = json.dumps(command_list) if command_list else None
v1_metadata = {
"id": docker_image_id,
}
if parent is not None:
v1_metadata["parent"] = parent.docker_image_id
new_image = model.image.set_image_metadata(
docker_image_id,
repo.namespace_user.username,
repo.name,
str(creation_time),
"no comment",
command,
json.dumps(v1_metadata),
parent,
)
new_image.storage.content_checksum = checksum
new_image.storage.save()
compressed_size = random.randrange(1, 1024 * 1024 * 1024)
model.storage.set_image_storage_metadata(
docker_image_id,
repo.namespace_user.username,
repo.name,
compressed_size,
int(compressed_size * 1.4),
)
parent = new_image
if last_node_tags:
if not isinstance(last_node_tags, list):
last_node_tags = [last_node_tags]
repo_ref = registry_model.lookup_repository(
repo.namespace_user.username, repo.name)
for tag_name in last_node_tags:
adjusted_tag_name = tag_name
now_ms = None
if tag_name[0] == "#":
adjusted_tag_name = tag_name[1:]
now_ms = get_epoch_timestamp_ms() - 1000
new_tag = model.tag.create_or_update_tag(
repo.namespace_user.username,
repo.name,
adjusted_tag_name,
new_image.docker_image_id,
now_ms=now_ms,
)
derived = model.image.find_or_create_derived_storage(
new_tag, "squash", "local_us")
model.storage.find_or_create_storage_signature(derived, "gpg2")
tag = pre_oci_model.get_repo_tag(repo_ref, adjusted_tag_name)
assert tag._db_id == new_tag.id
assert pre_oci_model.backfill_manifest_for_tag(tag)
tag_map[tag_name] = new_tag
for subtree in subtrees:
__create_subtree(with_storage, repo, subtree, creator_username,
new_image, tag_map)
def retarget_tag(tag_name,
manifest_id,
is_reversion=False,
now_ms=None,
adjust_old_model=True):
"""
Creates or updates a tag with the specified name to point to the given manifest under its
repository.
If this action is a reversion to a previous manifest, is_reversion should be set to True.
Returns the newly created tag row or None on error.
"""
try:
manifest = (Manifest.select(
Manifest,
MediaType).join(MediaType).where(Manifest.id == manifest_id).get())
except Manifest.DoesNotExist:
return None
# CHECK: Make sure that we are not mistargeting a schema 1 manifest to a tag with a different
# name.
if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES:
try:
parsed = DockerSchema1Manifest(Bytes.for_string_or_unicode(
manifest.manifest_bytes),
validate=False)
if parsed.tag != tag_name:
logger.error(
"Tried to re-target schema1 manifest with tag `%s` to tag `%s",
parsed.tag,
tag_name,
)
return None
except MalformedSchema1Manifest:
logger.exception("Could not parse schema1 manifest")
return None
legacy_image = get_legacy_image_for_manifest(manifest)
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
# Lookup an existing tag in the repository with the same name and, if present, mark it
# as expired.
existing_tag = get_tag(manifest.repository_id, tag_name)
if existing_tag is not None:
_, okay = set_tag_end_ms(existing_tag, now_ms)
# TODO: should we retry here and/or use a for-update?
if not okay:
return None
# Create a new tag pointing to the manifest with a lifetime start of now.
created = Tag.create(
name=tag_name,
repository=manifest.repository_id,
lifetime_start_ms=now_ms,
reversion=is_reversion,
manifest=manifest,
tag_kind=Tag.tag_kind.get_id("tag"),
)
# TODO: Remove the linkage code once RepositoryTag is gone.
# If this is a schema 1 manifest, then add a TagManifest linkage to it. Otherwise, it will only
# be pullable via the new OCI model.
if adjust_old_model:
if (manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES
and legacy_image is not None):
old_style_tag = RepositoryTag.create(
repository=manifest.repository_id,
image=legacy_image,
name=tag_name,
lifetime_start_ts=now_ts,
reversion=is_reversion,
)
TagToRepositoryTag.create(tag=created,
repository_tag=old_style_tag,
repository=manifest.repository_id)
tag_manifest = TagManifest.create(
tag=old_style_tag,
digest=manifest.digest,
json_data=manifest.manifest_bytes)
TagManifestToManifest.create(tag_manifest=tag_manifest,
manifest=manifest,
repository=manifest.repository_id)
return created
def perform_mirror(skopeo, mirror):
"""
Run mirror on all matching tags of remote repository.
"""
if os.getenv("DEBUGLOG", "false").lower() == "true":
verbose_logs = True
else:
verbose_logs = False
mirror = claim_mirror(mirror)
if mirror == None:
raise PreemptedException
emit_log(
mirror,
"repo_mirror_sync_started",
"start",
"'%s' with tag pattern '%s'"
% (mirror.external_reference, ",".join(mirror.root_rule.rule_value)),
)
# Fetch the tags to mirror, being careful to handle exceptions. The 'Exception' is safety net only, allowing
# easy communication by user through bug report.
tags = []
try:
tags = tags_to_mirror(skopeo, mirror)
except RepoMirrorSkopeoException as e:
emit_log(
mirror,
"repo_mirror_sync_failed",
"end",
"'%s' with tag pattern '%s': %s"
% (mirror.external_reference, ",".join(mirror.root_rule.rule_value), str(e)),
tags=", ".join(tags),
stdout=e.stdout,
stderr=e.stderr,
)
release_mirror(mirror, RepoMirrorStatus.FAIL)
return
except Exception as e:
emit_log(
mirror,
"repo_mirror_sync_failed",
"end",
"'%s' with tag pattern '%s': INTERNAL ERROR"
% (mirror.external_reference, ",".join(mirror.root_rule.rule_value)),
tags=", ".join(tags),
stdout="Not applicable",
stderr=traceback.format_exc(e),
)
release_mirror(mirror, RepoMirrorStatus.FAIL)
return
if tags == []:
emit_log(
mirror,
"repo_mirror_sync_success",
"end",
"'%s' with tag pattern '%s'"
% (mirror.external_reference, ",".join(mirror.root_rule.rule_value)),
tags="No tags matched",
)
release_mirror(mirror, RepoMirrorStatus.SUCCESS)
return
# Sync tags
now_ms = database.get_epoch_timestamp_ms()
overall_status = RepoMirrorStatus.SUCCESS
try:
delete_obsolete_tags(mirror, tags)
try:
username = (
mirror.external_registry_username.decrypt()
if mirror.external_registry_username
else None
)
password = (
mirror.external_registry_password.decrypt()
if mirror.external_registry_password
else None
)
except DecryptionFailureException:
logger.exception(
"Failed to decrypt username or password for mirroring %s", mirror.repository
)
raise
dest_server = (
app.config.get("REPO_MIRROR_SERVER_HOSTNAME", None) or app.config["SERVER_HOSTNAME"]
)
for tag in tags:
src_image = "docker://%s:%s" % (mirror.external_reference, tag)
dest_image = "docker://%s/%s/%s:%s" % (
dest_server,
mirror.repository.namespace_user.username,
mirror.repository.name,
tag,
)
with database.CloseForLongOperation(app.config):
result = skopeo.copy(
src_image,
dest_image,
src_tls_verify=mirror.external_registry_config.get("verify_tls", True),
dest_tls_verify=app.config.get(
"REPO_MIRROR_TLS_VERIFY", True
), # TODO: is this a config choice or something else?
src_username=username,
src_password=password,
dest_username=mirror.internal_robot.username,
dest_password=retrieve_robot_token(mirror.internal_robot),
proxy=mirror.external_registry_config.get("proxy", {}),
verbose_logs=verbose_logs,
)
if not result.success:
overall_status = RepoMirrorStatus.FAIL
emit_log(
mirror,
"repo_mirror_sync_tag_failed",
"finish",
"Source '%s' failed to sync" % src_image,
tag=tag,
stdout=result.stdout,
stderr=result.stderr,
)
logger.info("Source '%s' failed to sync." % src_image)
else:
emit_log(
mirror,
"repo_mirror_sync_tag_success",
"finish",
"Source '%s' successful sync" % src_image,
tag=tag,
stdout=result.stdout,
stderr=result.stderr,
)
logger.info("Source '%s' successful sync." % src_image)
mirror = claim_mirror(mirror)
if mirror is None:
emit_log(
mirror,
"repo_mirror_sync_failed",
"lost",
"'%s' with tag pattern '%s'"
% (mirror.external_reference, ",".join(mirror.root_rule.rule_value)),
)
except Exception as e:
overall_status = RepoMirrorStatus.FAIL
emit_log(
mirror,
"repo_mirror_sync_failed",
"end",
"'%s' with tag pattern '%s': INTERNAL ERROR"
% (mirror.external_reference, ",".join(mirror.root_rule.rule_value)),
tags=", ".join(tags),
stdout="Not applicable",
stderr=traceback.format_exc(e),
)
release_mirror(mirror, overall_status)
return
finally:
if overall_status == RepoMirrorStatus.FAIL:
emit_log(
mirror,
"repo_mirror_sync_failed",
"lost",
"'%s' with tag pattern '%s'"
% (mirror.external_reference, ",".join(mirror.root_rule.rule_value)),
)
rollback(mirror, now_ms)
else:
emit_log(
mirror,
"repo_mirror_sync_success",
"end",
"'%s' with tag pattern '%s'"
% (mirror.external_reference, ",".join(mirror.root_rule.rule_value)),
tags=", ".join(tags),
)
release_mirror(mirror, overall_status)
return overall_status
def lookup_manifest_by_digest(
self,
repository_ref,
manifest_digest,
allow_dead=False,
require_available=False,
raise_on_error=True,
):
"""
Looks up the manifest with the given digest under the given repository and returns it or
None if none.
If a manifest with the digest does not exist, fetches the manifest upstream
and creates it with a temp tag.
"""
wrapped_manifest = super().lookup_manifest_by_digest(
repository_ref,
manifest_digest,
allow_dead=True,
require_available=False)
if wrapped_manifest is None:
try:
wrapped_manifest, _ = self._create_and_tag_manifest(
repository_ref, manifest_digest,
self._create_manifest_with_temp_tag)
except (UpstreamRegistryError, ManifestDoesNotExist) as e:
raise ManifestDoesNotExist(str(e))
return wrapped_manifest
db_tag = oci.tag.get_tag_by_manifest_id(repository_ref.id,
wrapped_manifest.id)
existing_tag = Tag.for_tag(db_tag,
self._legacy_image_id_handler,
manifest_row=db_tag.manifest)
new_tag = False
try:
tag, new_tag = self._update_manifest_for_tag(
repository_ref,
existing_tag,
existing_tag.manifest,
manifest_digest,
self._create_manifest_with_temp_tag,
)
except ManifestDoesNotExist as e:
raise e
except UpstreamRegistryError:
# when the upstream fetch fails, we only return the tag if
# it isn't yet expired. note that we don't bump the tag's
# expiration here either - we only do this when we can ensure
# the tag exists upstream.
isplaceholder = wrapped_manifest.internal_manifest_bytes.as_unicode(
) == ""
return wrapped_manifest if not existing_tag.expired and not isplaceholder else None
if tag.expired or not new_tag:
with db_disallow_replica_use():
new_expiration = (get_epoch_timestamp_ms() +
self._config.expiration_s * 1000
if self._config.expiration_s else None)
oci.tag.set_tag_end_ms(db_tag, new_expiration)
# if the manifest is a child of a manifest list in this repo, renew
# the parent manifest list tag too.
parent = ManifestChild.select(ManifestChild.manifest_id).where(
(ManifestChild.repository_id == repository_ref.id)
& (ManifestChild.child_manifest_id == wrapped_manifest.id))
parent_tag = oci.tag.get_tag_by_manifest_id(
repository_ref.id, parent)
if parent_tag is not None:
oci.tag.set_tag_end_ms(parent_tag, new_expiration)
return super().lookup_manifest_by_digest(
repository_ref,
manifest_digest,
allow_dead=True,
require_available=False,
raise_on_error=True,
)
def retarget_tag(
tag_name,
manifest_id,
is_reversion=False,
now_ms=None,
raise_on_error=False,
):
"""
Creates or updates a tag with the specified name to point to the given manifest under its
repository.
If this action is a reversion to a previous manifest, is_reversion should be set to True.
Returns the newly created tag row or None on error.
"""
try:
manifest = (Manifest.select(
Manifest,
MediaType).join(MediaType).where(Manifest.id == manifest_id).get())
except Manifest.DoesNotExist:
if raise_on_error:
raise RetargetTagException("Manifest requested no longer exists")
return None
# CHECK: Make sure that we are not mistargeting a schema 1 manifest to a tag with a different
# name.
if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES:
try:
parsed = DockerSchema1Manifest(Bytes.for_string_or_unicode(
manifest.manifest_bytes),
validate=False)
if parsed.tag != tag_name:
logger.error(
"Tried to re-target schema1 manifest with tag `%s` to tag `%s",
parsed.tag,
tag_name,
)
return None
except MalformedSchema1Manifest as msme:
logger.exception("Could not parse schema1 manifest")
if raise_on_error:
raise RetargetTagException(msme)
return None
legacy_image = get_legacy_image_for_manifest(manifest)
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms // 1000)
with db_transaction():
# Lookup an existing tag in the repository with the same name and, if present, mark it
# as expired.
existing_tag = get_tag(manifest.repository_id, tag_name)
if existing_tag is not None:
_, okay = set_tag_end_ms(existing_tag, now_ms)
# TODO: should we retry here and/or use a for-update?
if not okay:
return None
# Create a new tag pointing to the manifest with a lifetime start of now.
created = Tag.create(
name=tag_name,
repository=manifest.repository_id,
lifetime_start_ms=now_ms,
reversion=is_reversion,
manifest=manifest,
tag_kind=Tag.tag_kind.get_id("tag"),
)
return created