def move_tag(repository, tag, image_ids, expect_gc=True): namespace = repository.namespace_user.username name = repository.name repo_ref = RepositoryReference.for_repo_obj(repository) builder = DockerSchema1ManifestBuilder(namespace, name, tag) # NOTE: Building root to leaf. parent_id = None for image_id in image_ids: config = {"id": image_id, "config": {"Labels": {"foo": "bar", "meh": "grah",}}} if parent_id: config["parent"] = parent_id # Create a storage row for the layer blob. _, layer_blob_digest = _populate_blob(repository, image_id.encode("ascii")) builder.insert_layer(layer_blob_digest, json.dumps(config)) parent_id = image_id # Store the manifest. manifest = builder.build(docker_v2_signing_key) registry_model.create_manifest_and_retarget_tag( repo_ref, manifest, tag, storage, raise_on_error=True ) if expect_gc: assert model.gc.garbage_collect_repo(repository) == expect_gc
def test_image_with_cas(default_tag_policy, initialized_db): """ A repository with a tag pointing to an image backed by CAS. Deleting and GCing the tag should result in the storage and its CAS data being removed. """ with assert_gc_integrity(expect_storage_removed=True): repository = create_repository() # Create an image storage record under CAS. content = b"hello world" digest = "sha256:" + hashlib.sha256(content).hexdigest() preferred = storage.preferred_locations[0] storage.put_content({preferred}, storage.blob_path(digest), content) image_storage = database.ImageStorage.create(content_checksum=digest) location = database.ImageStorageLocation.get(name=preferred) database.ImageStoragePlacement.create(location=location, storage=image_storage) # Temp link so its available. model.blob.store_blob_record_and_temp_link_in_repo( repository, digest, location, len(content), 120 ) # Ensure the CAS path exists. assert storage.exists({preferred}, storage.blob_path(digest)) # Store a manifest pointing to that path. builder = DockerSchema1ManifestBuilder( repository.namespace_user.username, repository.name, "first" ) builder.insert_layer( digest, json.dumps( { "id": "i1", } ), ) # Store the manifest. manifest = builder.build(docker_v2_signing_key) repo_ref = RepositoryReference.for_repo_obj(repository) registry_model.create_manifest_and_retarget_tag( repo_ref, manifest, "first", storage, raise_on_error=True ) # Delete the temp reference. _delete_temp_links(repository) # Delete the tag. delete_tag(repository, "first") assert_deleted(repository, "i1") # Ensure the CAS path is gone. assert not storage.exists({preferred}, storage.blob_path(digest))
def _create_tag(repo, name): repo_ref = RepositoryReference.for_repo_obj(repo) with upload_blob(repo_ref, storage, BlobUploadSettings(500, 500)) as upload: app_config = {"TESTING": True} config_json = json.dumps({ "config": { "author": "Repo Mirror", }, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2019-07-30T18:37:09.284840891Z", "created_by": "base", "author": "Repo Mirror", }, ], }) upload.upload_chunk(app_config, BytesIO(config_json.encode("utf-8"))) blob = upload.commit_to_blob(app_config) builder = DockerSchema2ManifestBuilder() builder.set_config_digest(blob.digest, blob.compressed_size) builder.add_layer("sha256:abcd", 1234, urls=["http://hello/world"]) manifest = builder.build() manifest, tag = registry_model.create_manifest_and_retarget_tag( repo_ref, manifest, name, storage)
def _write_manifest( namespace_name, repo_name, tag_name, manifest_impl, registry_model=registry_model ): # Ensure that the repository exists. repository_ref = registry_model.lookup_repository(namespace_name, repo_name) if repository_ref is None: raise NameUnknown("repository not found") # Create the manifest(s) and retarget the tag to point to it. try: manifest, tag = registry_model.create_manifest_and_retarget_tag( repository_ref, manifest_impl, tag_name, storage, raise_on_error=True ) except CreateManifestException as cme: raise ManifestInvalid(detail={"message": str(cme)}) except RetargetTagException as rte: raise ManifestInvalid(detail={"message": str(rte)}) if manifest is None: raise ManifestInvalid() if app.config.get("FEATURE_QUOTA_MANAGEMENT", False): quota = namespacequota.verify_namespace_quota_force_cache(repository_ref) if quota["severity_level"] == "Warning": namespacequota.notify_organization_admins(repository_ref, "quota_warning") elif quota["severity_level"] == "Reject": namespacequota.notify_organization_admins(repository_ref, "quota_error") raise QuotaExceeded() return repository_ref, manifest, tag
def _write_manifest(namespace_name, repo_name, tag_name, manifest_impl): # NOTE: These extra checks are needed for schema version 1 because the manifests # contain the repo namespace, name and tag name. if manifest_impl.schema_version == 1: if (manifest_impl.namespace == "" and features.LIBRARY_SUPPORT and namespace_name == app.config["LIBRARY_NAMESPACE"]): pass elif manifest_impl.namespace != namespace_name: raise NameInvalid( message="namespace name does not match manifest", detail={ "namespace name `%s` does not match `%s` in manifest" % (namespace_name, manifest_impl.namespace) }, ) if manifest_impl.repo_name != repo_name: raise NameInvalid( message="repository name does not match manifest", detail={ "repository name `%s` does not match `%s` in manifest" % (repo_name, manifest_impl.repo_name) }, ) try: if not manifest_impl.layers: raise ManifestInvalid( detail={ "message": "manifest does not reference any layers" }) except ManifestException as me: raise ManifestInvalid(detail={"message": str(me)}) # Ensure that the repository exists. repository_ref = registry_model.lookup_repository(namespace_name, repo_name) if repository_ref is None: raise NameUnknown() # Create the manifest(s) and retarget the tag to point to it. try: manifest, tag = registry_model.create_manifest_and_retarget_tag( repository_ref, manifest_impl, tag_name, storage, raise_on_error=True) except CreateManifestException as cme: raise ManifestInvalid(detail={"message": str(cme)}) except RetargetTagException as rte: raise ManifestInvalid(detail={"message": str(rte)}) if manifest is None: raise ManifestInvalid() return repository_ref, manifest, tag
def commit_tag_and_manifest(self, tag_name, layer): """ Commits a new tag + manifest for that tag to the repository with the given name, pointing to the given layer. """ # Lookup the top layer. image_metadata = self._builder_state.image_metadata.get(layer.layer_id) if image_metadata is None: return None # For each layer/image, add it to the manifest builder. builder = DockerSchema1ManifestBuilder( self._repository_ref.namespace_name, self._repository_ref.name, tag_name) current_layer_id = layer.layer_id while True: v1_metadata_string = self._builder_state.image_metadata.get( current_layer_id) if v1_metadata_string is None: logger.warning("Missing metadata for layer %s", current_layer_id) return None v1_metadata = json.loads(v1_metadata_string) parent_id = v1_metadata.get("parent", None) if parent_id is not None and parent_id not in self._builder_state.image_metadata: logger.warning("Missing parent for layer %s", current_layer_id) return None blob_digest = self._builder_state.image_blobs.get(current_layer_id) if blob_digest is None: logger.warning("Missing blob for layer %s", current_layer_id) return None builder.add_layer(blob_digest, v1_metadata_string) if not parent_id: break current_layer_id = parent_id # Build the manifest. manifest_instance = builder.build(self._legacy_signing_key) # Target the tag at the manifest. manifest, tag = registry_model.create_manifest_and_retarget_tag( self._repository_ref, manifest_instance, tag_name, self._storage) if tag is None: return None self._builder_state.tags[tag_name] = tag._db_id self._save_to_session() return tag
def __create_manifest_and_tags(repo, structure, creator_username, tag_map, current_level=0, builder=None, last_leaf_id=None): num_layers, subtrees, tag_names = structure num_layers = num_layers or 1 tag_names = tag_names or [] tag_names = [tag_names] if not isinstance(tag_names, list) else tag_names repo_ref = RepositoryReference.for_repo_obj(repo) builder = (builder if builder else DockerSchema1ManifestBuilder( repo.namespace_user.username, repo.name, "")) # TODO: Change this to a mixture of Schema1 and Schema2 manifest once we no longer need to # read from storage for Schema2. # Populate layers. Note, we do this in reverse order using insert_layer, as it is easier to # add the leaf last (even though Schema1 has it listed first). parent_id = last_leaf_id leaf_id = None for layer_index in range(0, num_layers): content = "layer-%s-%s-%s" % (layer_index, current_level, get_epoch_timestamp_ms()) _, digest = _populate_blob(repo, content.encode("ascii")) current_id = "abcdef%s%s%s" % (layer_index, current_level, get_epoch_timestamp_ms()) if layer_index == num_layers - 1: leaf_id = current_id config = { "id": current_id, "Size": len(content), } if parent_id: config["parent"] = parent_id builder.insert_layer(digest, json.dumps(config)) parent_id = current_id for tag_name in tag_names: adjusted_tag_name = tag_name now = datetime.utcnow() if tag_name[0] == "#": adjusted_tag_name = tag_name[1:] now = now - timedelta(seconds=1) manifest = builder.clone(adjusted_tag_name).build() with freeze_time(now): created_tag, _ = registry_model.create_manifest_and_retarget_tag( repo_ref, manifest, adjusted_tag_name, store, raise_on_error=True) assert created_tag tag_map[adjusted_tag_name] = created_tag for subtree in subtrees: __create_manifest_and_tags( repo, subtree, creator_username, tag_map, current_level=current_level + 1, builder=builder, last_leaf_id=leaf_id, )
def test_images_shared_cas(default_tag_policy, initialized_db): """ A repository, each two tags, pointing to the same image, which has image storage with the same *CAS path*, but *distinct records*. Deleting the first tag should delete the first image, and its storage, but not the file in storage, as it shares its CAS path. """ with assert_gc_integrity(expect_storage_removed=True): repository = create_repository() # Create two image storage records with the same content checksum. content = b"hello world" digest = "sha256:" + hashlib.sha256(content).hexdigest() preferred = storage.preferred_locations[0] storage.put_content({preferred}, storage.blob_path(digest), content) is1 = database.ImageStorage.create(content_checksum=digest) is2 = database.ImageStorage.create(content_checksum=digest) location = database.ImageStorageLocation.get(name=preferred) database.ImageStoragePlacement.create(location=location, storage=is1) database.ImageStoragePlacement.create(location=location, storage=is2) # Temp link so its available. model.blob.store_blob_record_and_temp_link_in_repo( repository, digest, location, len(content), 120) # Ensure the CAS path exists. assert storage.exists({preferred}, storage.blob_path(digest)) repo_ref = RepositoryReference.for_repo_obj(repository) # Store a manifest pointing to that path as `first`. builder = DockerSchema1ManifestBuilder( repository.namespace_user.username, repository.name, "first") builder.insert_layer( digest, json.dumps({ "id": "i1", }), ) manifest = builder.build(docker_v2_signing_key) registry_model.create_manifest_and_retarget_tag(repo_ref, manifest, "first", storage, raise_on_error=True) tag_ref = registry_model.get_repo_tag(repo_ref, "first") manifest_ref = registry_model.get_manifest_for_tag(tag_ref) registry_model.populate_legacy_images_for_testing( manifest_ref, storage) # Store another as `second`. builder = DockerSchema1ManifestBuilder( repository.namespace_user.username, repository.name, "second") builder.insert_layer( digest, json.dumps({ "id": "i2", }), ) manifest = builder.build(docker_v2_signing_key) created, _ = registry_model.create_manifest_and_retarget_tag( repo_ref, manifest, "second", storage, raise_on_error=True) tag_ref = registry_model.get_repo_tag(repo_ref, "second") manifest_ref = registry_model.get_manifest_for_tag(tag_ref) registry_model.populate_legacy_images_for_testing( manifest_ref, storage) # Manually retarget the second manifest's blob to the second row. try: second_blob = ManifestBlob.get(manifest=created._db_id, blob=is1) second_blob.blob = is2 second_blob.save() except ManifestBlob.DoesNotExist: second_blob = ManifestBlob.get(manifest=created._db_id, blob=is2) second_blob.blob = is1 second_blob.save() # Delete the temp reference. _delete_temp_links(repository) # Ensure the legacy images exist. assert_not_deleted(repository, "i1", "i2") # Delete the first tag. delete_tag(repository, "first") assert_deleted(repository, "i1") assert_not_deleted(repository, "i2") # Ensure the CAS path still exists. assert storage.exists({preferred}, storage.blob_path(digest))