def move_tag(repository, tag, image_ids, expect_gc=True):
namespace = repository.namespace_user.username
name = repository.name
repo_ref = RepositoryReference.for_repo_obj(repository)
builder = DockerSchema1ManifestBuilder(namespace, name, tag)
# NOTE: Building root to leaf.
parent_id = None
for image_id in image_ids:
config = {"id": image_id, "config": {"Labels": {"foo": "bar", "meh": "grah",}}}
if parent_id:
config["parent"] = parent_id
# Create a storage row for the layer blob.
_, layer_blob_digest = _populate_blob(repository, image_id.encode("ascii"))
builder.insert_layer(layer_blob_digest, json.dumps(config))
parent_id = image_id
# Store the manifest.
manifest = builder.build(docker_v2_signing_key)
registry_model.create_manifest_and_retarget_tag(
repo_ref, manifest, tag, storage, raise_on_error=True
)
if expect_gc:
assert model.gc.garbage_collect_repo(repository) == expect_gc
def test_image_with_cas(default_tag_policy, initialized_db):
"""
A repository with a tag pointing to an image backed by CAS.
Deleting and GCing the tag should result in the storage and its CAS data being removed.
"""
with assert_gc_integrity(expect_storage_removed=True):
repository = create_repository()
# Create an image storage record under CAS.
content = b"hello world"
digest = "sha256:" + hashlib.sha256(content).hexdigest()
preferred = storage.preferred_locations[0]
storage.put_content({preferred}, storage.blob_path(digest), content)
image_storage = database.ImageStorage.create(content_checksum=digest)
location = database.ImageStorageLocation.get(name=preferred)
database.ImageStoragePlacement.create(location=location, storage=image_storage)
# Temp link so its available.
model.blob.store_blob_record_and_temp_link_in_repo(
repository, digest, location, len(content), 120
)
# Ensure the CAS path exists.
assert storage.exists({preferred}, storage.blob_path(digest))
# Store a manifest pointing to that path.
builder = DockerSchema1ManifestBuilder(
repository.namespace_user.username, repository.name, "first"
)
builder.insert_layer(
digest,
json.dumps(
{
"id": "i1",
}
),
)
# Store the manifest.
manifest = builder.build(docker_v2_signing_key)
repo_ref = RepositoryReference.for_repo_obj(repository)
registry_model.create_manifest_and_retarget_tag(
repo_ref, manifest, "first", storage, raise_on_error=True
)
# Delete the temp reference.
_delete_temp_links(repository)
# Delete the tag.
delete_tag(repository, "first")
assert_deleted(repository, "i1")
# Ensure the CAS path is gone.
assert not storage.exists({preferred}, storage.blob_path(digest))
def _create_tag(repo, name):
repo_ref = RepositoryReference.for_repo_obj(repo)
with upload_blob(repo_ref, storage, BlobUploadSettings(500,
500)) as upload:
app_config = {"TESTING": True}
config_json = json.dumps({
"config": {
"author": "Repo Mirror",
},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [
{
"created": "2019-07-30T18:37:09.284840891Z",
"created_by": "base",
"author": "Repo Mirror",
},
],
})
upload.upload_chunk(app_config, BytesIO(config_json.encode("utf-8")))
blob = upload.commit_to_blob(app_config)
builder = DockerSchema2ManifestBuilder()
builder.set_config_digest(blob.digest, blob.compressed_size)
builder.add_layer("sha256:abcd", 1234, urls=["http://hello/world"])
manifest = builder.build()
manifest, tag = registry_model.create_manifest_and_retarget_tag(
repo_ref, manifest, name, storage)
def _write_manifest(
namespace_name, repo_name, tag_name, manifest_impl, registry_model=registry_model
):
# Ensure that the repository exists.
repository_ref = registry_model.lookup_repository(namespace_name, repo_name)
if repository_ref is None:
raise NameUnknown("repository not found")
# Create the manifest(s) and retarget the tag to point to it.
try:
manifest, tag = registry_model.create_manifest_and_retarget_tag(
repository_ref, manifest_impl, tag_name, storage, raise_on_error=True
)
except CreateManifestException as cme:
raise ManifestInvalid(detail={"message": str(cme)})
except RetargetTagException as rte:
raise ManifestInvalid(detail={"message": str(rte)})
if manifest is None:
raise ManifestInvalid()
if app.config.get("FEATURE_QUOTA_MANAGEMENT", False):
quota = namespacequota.verify_namespace_quota_force_cache(repository_ref)
if quota["severity_level"] == "Warning":
namespacequota.notify_organization_admins(repository_ref, "quota_warning")
elif quota["severity_level"] == "Reject":
namespacequota.notify_organization_admins(repository_ref, "quota_error")
raise QuotaExceeded()
return repository_ref, manifest, tag
def _write_manifest(namespace_name, repo_name, tag_name, manifest_impl):
# NOTE: These extra checks are needed for schema version 1 because the manifests
# contain the repo namespace, name and tag name.
if manifest_impl.schema_version == 1:
if (manifest_impl.namespace == "" and features.LIBRARY_SUPPORT
and namespace_name == app.config["LIBRARY_NAMESPACE"]):
pass
elif manifest_impl.namespace != namespace_name:
raise NameInvalid(
message="namespace name does not match manifest",
detail={
"namespace name `%s` does not match `%s` in manifest" %
(namespace_name, manifest_impl.namespace)
},
)
if manifest_impl.repo_name != repo_name:
raise NameInvalid(
message="repository name does not match manifest",
detail={
"repository name `%s` does not match `%s` in manifest" %
(repo_name, manifest_impl.repo_name)
},
)
try:
if not manifest_impl.layers:
raise ManifestInvalid(
detail={
"message": "manifest does not reference any layers"
})
except ManifestException as me:
raise ManifestInvalid(detail={"message": str(me)})
# Ensure that the repository exists.
repository_ref = registry_model.lookup_repository(namespace_name,
repo_name)
if repository_ref is None:
raise NameUnknown()
# Create the manifest(s) and retarget the tag to point to it.
try:
manifest, tag = registry_model.create_manifest_and_retarget_tag(
repository_ref,
manifest_impl,
tag_name,
storage,
raise_on_error=True)
except CreateManifestException as cme:
raise ManifestInvalid(detail={"message": str(cme)})
except RetargetTagException as rte:
raise ManifestInvalid(detail={"message": str(rte)})
if manifest is None:
raise ManifestInvalid()
return repository_ref, manifest, tag
def commit_tag_and_manifest(self, tag_name, layer):
"""
Commits a new tag + manifest for that tag to the repository with the given name, pointing to
the given layer.
"""
# Lookup the top layer.
image_metadata = self._builder_state.image_metadata.get(layer.layer_id)
if image_metadata is None:
return None
# For each layer/image, add it to the manifest builder.
builder = DockerSchema1ManifestBuilder(
self._repository_ref.namespace_name, self._repository_ref.name,
tag_name)
current_layer_id = layer.layer_id
while True:
v1_metadata_string = self._builder_state.image_metadata.get(
current_layer_id)
if v1_metadata_string is None:
logger.warning("Missing metadata for layer %s",
current_layer_id)
return None
v1_metadata = json.loads(v1_metadata_string)
parent_id = v1_metadata.get("parent", None)
if parent_id is not None and parent_id not in self._builder_state.image_metadata:
logger.warning("Missing parent for layer %s", current_layer_id)
return None
blob_digest = self._builder_state.image_blobs.get(current_layer_id)
if blob_digest is None:
logger.warning("Missing blob for layer %s", current_layer_id)
return None
builder.add_layer(blob_digest, v1_metadata_string)
if not parent_id:
break
current_layer_id = parent_id
# Build the manifest.
manifest_instance = builder.build(self._legacy_signing_key)
# Target the tag at the manifest.
manifest, tag = registry_model.create_manifest_and_retarget_tag(
self._repository_ref, manifest_instance, tag_name, self._storage)
if tag is None:
return None
self._builder_state.tags[tag_name] = tag._db_id
self._save_to_session()
return tag
def __create_manifest_and_tags(repo,
structure,
creator_username,
tag_map,
current_level=0,
builder=None,
last_leaf_id=None):
num_layers, subtrees, tag_names = structure
num_layers = num_layers or 1
tag_names = tag_names or []
tag_names = [tag_names] if not isinstance(tag_names, list) else tag_names
repo_ref = RepositoryReference.for_repo_obj(repo)
builder = (builder if builder else DockerSchema1ManifestBuilder(
repo.namespace_user.username, repo.name, ""))
# TODO: Change this to a mixture of Schema1 and Schema2 manifest once we no longer need to
# read from storage for Schema2.
# Populate layers. Note, we do this in reverse order using insert_layer, as it is easier to
# add the leaf last (even though Schema1 has it listed first).
parent_id = last_leaf_id
leaf_id = None
for layer_index in range(0, num_layers):
content = "layer-%s-%s-%s" % (layer_index, current_level,
get_epoch_timestamp_ms())
_, digest = _populate_blob(repo, content.encode("ascii"))
current_id = "abcdef%s%s%s" % (layer_index, current_level,
get_epoch_timestamp_ms())
if layer_index == num_layers - 1:
leaf_id = current_id
config = {
"id": current_id,
"Size": len(content),
}
if parent_id:
config["parent"] = parent_id
builder.insert_layer(digest, json.dumps(config))
parent_id = current_id
for tag_name in tag_names:
adjusted_tag_name = tag_name
now = datetime.utcnow()
if tag_name[0] == "#":
adjusted_tag_name = tag_name[1:]
now = now - timedelta(seconds=1)
manifest = builder.clone(adjusted_tag_name).build()
with freeze_time(now):
created_tag, _ = registry_model.create_manifest_and_retarget_tag(
repo_ref,
manifest,
adjusted_tag_name,
store,
raise_on_error=True)
assert created_tag
tag_map[adjusted_tag_name] = created_tag
for subtree in subtrees:
__create_manifest_and_tags(
repo,
subtree,
creator_username,
tag_map,
current_level=current_level + 1,
builder=builder,
last_leaf_id=leaf_id,
)
def test_images_shared_cas(default_tag_policy, initialized_db):
"""
A repository, each two tags, pointing to the same image, which has image storage with the same
*CAS path*, but *distinct records*.
Deleting the first tag should delete the first image, and its storage, but not the file in
storage, as it shares its CAS path.
"""
with assert_gc_integrity(expect_storage_removed=True):
repository = create_repository()
# Create two image storage records with the same content checksum.
content = b"hello world"
digest = "sha256:" + hashlib.sha256(content).hexdigest()
preferred = storage.preferred_locations[0]
storage.put_content({preferred}, storage.blob_path(digest), content)
is1 = database.ImageStorage.create(content_checksum=digest)
is2 = database.ImageStorage.create(content_checksum=digest)
location = database.ImageStorageLocation.get(name=preferred)
database.ImageStoragePlacement.create(location=location, storage=is1)
database.ImageStoragePlacement.create(location=location, storage=is2)
# Temp link so its available.
model.blob.store_blob_record_and_temp_link_in_repo(
repository, digest, location, len(content), 120)
# Ensure the CAS path exists.
assert storage.exists({preferred}, storage.blob_path(digest))
repo_ref = RepositoryReference.for_repo_obj(repository)
# Store a manifest pointing to that path as `first`.
builder = DockerSchema1ManifestBuilder(
repository.namespace_user.username, repository.name, "first")
builder.insert_layer(
digest,
json.dumps({
"id": "i1",
}),
)
manifest = builder.build(docker_v2_signing_key)
registry_model.create_manifest_and_retarget_tag(repo_ref,
manifest,
"first",
storage,
raise_on_error=True)
tag_ref = registry_model.get_repo_tag(repo_ref, "first")
manifest_ref = registry_model.get_manifest_for_tag(tag_ref)
registry_model.populate_legacy_images_for_testing(
manifest_ref, storage)
# Store another as `second`.
builder = DockerSchema1ManifestBuilder(
repository.namespace_user.username, repository.name, "second")
builder.insert_layer(
digest,
json.dumps({
"id": "i2",
}),
)
manifest = builder.build(docker_v2_signing_key)
created, _ = registry_model.create_manifest_and_retarget_tag(
repo_ref, manifest, "second", storage, raise_on_error=True)
tag_ref = registry_model.get_repo_tag(repo_ref, "second")
manifest_ref = registry_model.get_manifest_for_tag(tag_ref)
registry_model.populate_legacy_images_for_testing(
manifest_ref, storage)
# Manually retarget the second manifest's blob to the second row.
try:
second_blob = ManifestBlob.get(manifest=created._db_id, blob=is1)
second_blob.blob = is2
second_blob.save()
except ManifestBlob.DoesNotExist:
second_blob = ManifestBlob.get(manifest=created._db_id, blob=is2)
second_blob.blob = is1
second_blob.save()
# Delete the temp reference.
_delete_temp_links(repository)
# Ensure the legacy images exist.
assert_not_deleted(repository, "i1", "i2")
# Delete the first tag.
delete_tag(repository, "first")
assert_deleted(repository, "i1")
assert_not_deleted(repository, "i2")
# Ensure the CAS path still exists.
assert storage.exists({preferred}, storage.blob_path(digest))
