def build(self, ensure_ascii=True): """ Builds and returns the DockerSchema2Manifest. """ assert self.config def _build_layer(layer): if layer.urls: return { DOCKER_SCHEMA2_MANIFEST_MEDIATYPE_KEY: DOCKER_SCHEMA2_REMOTE_LAYER_CONTENT_TYPE, DOCKER_SCHEMA2_MANIFEST_SIZE_KEY: layer.compressed_size, DOCKER_SCHEMA2_MANIFEST_DIGEST_KEY: str(layer.digest), DOCKER_SCHEMA2_MANIFEST_URLS_KEY: layer.urls, } return { DOCKER_SCHEMA2_MANIFEST_MEDIATYPE_KEY: DOCKER_SCHEMA2_LAYER_CONTENT_TYPE, DOCKER_SCHEMA2_MANIFEST_SIZE_KEY: layer.compressed_size, DOCKER_SCHEMA2_MANIFEST_DIGEST_KEY: str(layer.digest), } manifest_dict = { DOCKER_SCHEMA2_MANIFEST_VERSION_KEY: 2, DOCKER_SCHEMA2_MANIFEST_MEDIATYPE_KEY: DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE, # Config DOCKER_SCHEMA2_MANIFEST_CONFIG_KEY: { DOCKER_SCHEMA2_MANIFEST_MEDIATYPE_KEY: DOCKER_SCHEMA2_CONFIG_CONTENT_TYPE, DOCKER_SCHEMA2_MANIFEST_SIZE_KEY: self.config.size, DOCKER_SCHEMA2_MANIFEST_DIGEST_KEY: str(self.config.digest), }, # Layers DOCKER_SCHEMA2_MANIFEST_LAYERS_KEY: [ _build_layer(layer) for layer in self.filesystem_layers ], } json_str = json.dumps(manifest_dict, ensure_ascii=ensure_ascii, indent=3) return DockerSchema2Manifest(Bytes.for_string_or_unicode(json_str))
def test_create_manifest_config_blob(self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response( test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"], ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) resp = conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) if manifest.schema_version == 2 and not manifest.is_manifest_list: q = ImageStorage.filter( ImageStorage.content_checksum == manifest.config.digest) assert q.count() == 1
def for_manifest(cls, manifest, legacy_id_handler, legacy_image_row=None): if manifest is None: return None # NOTE: `manifest_bytes` will be None if not selected by certain join queries. manifest_bytes = ( Bytes.for_string_or_unicode(manifest.manifest_bytes) if manifest.manifest_bytes is not None else None ) return Manifest( db_id=manifest.id, digest=manifest.digest, internal_manifest_bytes=manifest_bytes, media_type=ManifestTable.media_type.get_name(manifest.media_type_id), _layers_compressed_size=manifest.layers_compressed_size, config_media_type=manifest.config_media_type, inputs=dict( legacy_id_handler=legacy_id_handler, legacy_image_row=legacy_image_row, repository=RepositoryReference.for_id(manifest.repository_id), ), )
def _validate(self): if not self._signatures: return payload_str = self._payload for signature in self._signatures: bytes_to_verify = b"%s.%s" % ( Bytes.for_string_or_unicode( signature["protected"]).as_encoded_str(), base64url_encode(payload_str), ) signer = SIGNER_ALGS[signature["header"]["alg"]] key = keyrep(signature["header"]["jwk"]) gk = key.get_key() sig = base64url_decode(signature["signature"].encode("utf-8")) try: verified = signer.verify(bytes_to_verify, sig, gk) except BadSignature: raise InvalidSchema1Signature() if not verified: raise InvalidSchema1Signature()
def build(self, ensure_ascii=True): """ Builds and returns the OCIManifest. """ assert self.filesystem_layers assert self.config def _build_layer(layer): if layer.urls: return { OCI_MANIFEST_MEDIATYPE_KEY: OCI_IMAGE_TAR_GZIP_NON_DISTRIBUTABLE_LAYER_CONTENT_TYPE, OCI_MANIFEST_SIZE_KEY: layer.compressed_size, OCI_MANIFEST_DIGEST_KEY: str(layer.digest), OCI_MANIFEST_URLS_KEY: layer.urls, } return { OCI_MANIFEST_MEDIATYPE_KEY: OCI_IMAGE_TAR_GZIP_LAYER_CONTENT_TYPE, OCI_MANIFEST_SIZE_KEY: layer.compressed_size, OCI_MANIFEST_DIGEST_KEY: str(layer.digest), } manifest_dict = { OCI_MANIFEST_VERSION_KEY: 2, OCI_MANIFEST_MEDIATYPE_KEY: OCI_IMAGE_MANIFEST_CONTENT_TYPE, # Config OCI_MANIFEST_CONFIG_KEY: { OCI_MANIFEST_MEDIATYPE_KEY: OCI_IMAGE_CONFIG_CONTENT_TYPE, OCI_MANIFEST_SIZE_KEY: self.config.size, OCI_MANIFEST_DIGEST_KEY: str(self.config.digest), }, # Layers OCI_MANIFEST_LAYERS_KEY: [_build_layer(layer) for layer in self.filesystem_layers], } json_str = json.dumps(manifest_dict, ensure_ascii=ensure_ascii, indent=3) return OCIManifest(Bytes.for_string_or_unicode(json_str))
def test_valid_manifest(): manifest = DockerSchema1Manifest(Bytes.for_string_or_unicode(MANIFEST_BYTES), validate=False) assert len(manifest.signatures) == 1 assert manifest.namespace == "" assert manifest.repo_name == "hello-world" assert manifest.tag == "latest" assert manifest.image_ids == {"someid", "anotherid"} assert manifest.parent_image_ids == {"anotherid"} assert len(manifest.layers) == 2 assert manifest.layers[0].v1_metadata.image_id == "anotherid" assert manifest.layers[0].v1_metadata.parent_image_id is None assert manifest.layers[1].v1_metadata.image_id == "someid" assert manifest.layers[1].v1_metadata.parent_image_id == "anotherid" assert manifest.layers[0].compressed_size is None assert manifest.layers[1].compressed_size is None assert manifest.leaf_layer == manifest.layers[1] assert manifest.created_datetime is None unsigned = manifest.unsigned() assert unsigned.namespace == manifest.namespace assert unsigned.repo_name == manifest.repo_name assert unsigned.tag == manifest.tag assert unsigned.layers == manifest.layers assert unsigned.blob_digests == manifest.blob_digests assert unsigned.digest != manifest.digest image_layers = list(manifest.get_layers(None)) assert len(image_layers) == 2 for index in range(0, 2): assert image_layers[index].layer_id == manifest.layers[index].v1_metadata.image_id assert image_layers[index].blob_digest == manifest.layers[index].digest assert image_layers[index].command == manifest.layers[index].v1_metadata.command
def test_validate_manifest_without_metadata(): test_dir = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(test_dir, "validated_manifest.json"), "r") as f: manifest_bytes = f.read() manifest = DockerSchema1Manifest(Bytes.for_string_or_unicode(manifest_bytes), validate=True) digest = manifest.digest assert digest == "sha256:b5dc4f63fdbd64f34f2314c0747ef81008f9fcddce4edfc3fd0e8ec8b358d571" assert manifest.created_datetime with_metadata_removed = manifest._unsigned_builder().with_metadata_removed().build() assert with_metadata_removed.leaf_layer_v1_image_id == manifest.leaf_layer_v1_image_id manifest_layers = list(manifest.get_layers(None)) with_metadata_removed_layers = list(with_metadata_removed.get_layers(None)) assert len(manifest_layers) == len(with_metadata_removed_layers) for index, built_layer in enumerate(manifest_layers): with_metadata_removed_layer = with_metadata_removed_layers[index] assert built_layer.layer_id == with_metadata_removed_layer.layer_id assert built_layer.compressed_size == with_metadata_removed_layer.compressed_size assert built_layer.command == with_metadata_removed_layer.command assert built_layer.comment == with_metadata_removed_layer.comment assert built_layer.author == with_metadata_removed_layer.author assert built_layer.blob_digest == with_metadata_removed_layer.blob_digest assert built_layer.created_datetime == with_metadata_removed_layer.created_datetime assert with_metadata_removed.digest != manifest.digest assert with_metadata_removed.namespace == manifest.namespace assert with_metadata_removed.repo_name == manifest.repo_name assert with_metadata_removed.tag == manifest.tag assert with_metadata_removed.created_datetime == manifest.created_datetime assert with_metadata_removed.checksums == manifest.checksums assert with_metadata_removed.image_ids == manifest.image_ids assert with_metadata_removed.parent_image_ids == manifest.parent_image_ids
def test_valid_manifestlist(): manifestlist = DockerSchema2ManifestList( Bytes.for_string_or_unicode(MANIFESTLIST_BYTES)) assert len(manifestlist.manifests(retriever)) == 2 assert manifestlist.media_type == "application/vnd.docker.distribution.manifest.list.v2+json" assert manifestlist.bytes.as_encoded_str() == MANIFESTLIST_BYTES assert manifestlist.manifest_dict == json.loads(MANIFESTLIST_BYTES) assert manifestlist.get_layers(retriever) is None assert not manifestlist.blob_digests for index, manifest in enumerate(manifestlist.manifests(retriever)): if index == 0: assert isinstance(manifest.manifest_obj, DockerSchema2Manifest) assert manifest.manifest_obj.schema_version == 2 else: assert isinstance(manifest.manifest_obj, DockerSchema1Manifest) assert manifest.manifest_obj.schema_version == 1 # Check retrieval of a schema 2 manifest. This should return None, because the schema 2 manifest # is not amd64-compatible. schema2_manifest = manifestlist.convert_manifest( [DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE], "foo", "bar", "baz", retriever) assert schema2_manifest is None # Check retrieval of a schema 1 manifest. compatible_manifest = manifestlist.get_schema1_manifest( "foo", "bar", "baz", retriever) assert compatible_manifest.schema_version == 1 schema1_manifest = manifestlist.convert_manifest( DOCKER_SCHEMA1_CONTENT_TYPES, "foo", "bar", "baz", retriever) assert schema1_manifest.schema_version == 1 assert schema1_manifest.digest == compatible_manifest.digest # Ensure it validates. manifestlist.validate(retriever)
def test_get_schema1_manifest(): retriever = ContentRetrieverForTesting.for_config( { "config": {"Labels": {"foo": "bar",},}, "rootfs": {"type": "layers", "diff_ids": []}, "history": [ {"created": "2018-04-03T18:37:09.284840891Z", "created_by": "foo"}, {"created": "2018-04-12T18:37:09.284840891Z", "created_by": "bar"}, {"created": "2018-04-03T18:37:09.284840891Z", "created_by": "foo"}, ], "architecture": "amd64", "os": "linux", }, "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", 7023, ) manifest = OCIManifest(Bytes.for_string_or_unicode(SAMPLE_MANIFEST)) assert manifest.get_manifest_labels(retriever) == { "com.example.key1": "value1", "com.example.key2": "value2", "foo": "bar", } schema1 = manifest.get_schema1_manifest("somenamespace", "somename", "sometag", retriever) assert schema1 is not None assert schema1.media_type == DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE assert set(schema1.local_blob_digests) == ( set(manifest.local_blob_digests) - {"sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"} ) assert len(schema1.layers) == 3 via_convert = manifest.convert_manifest( [schema1.media_type], "somenamespace", "somename", "sometag", retriever ) assert via_convert.digest == schema1.digest
def test_connect_existing_blobs_to_new_manifest(self, create_repo): repo_ref = create_repo(self.orgname, self.upstream_repository, self.user) input_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(UBI8_8_4_MANIFEST_SCHEMA2), DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE, ) layer = input_manifest.manifest_dict["layers"][0] blob = ImageStorage.create( image_size=layer["size"], uncompressed_size=layer["size"], content_checksum=layer["digest"], ) proxy_model = ProxyModel( self.orgname, self.upstream_repository, self.user, ) proxy_model._create_manifest_and_retarget_tag(repo_ref, input_manifest, self.tag) blob_count = (ImageStorage.select().where( ImageStorage.content_checksum == blob.content_checksum).count()) assert blob_count == 1
def access_valid_token(token_code): """ Looks up an unexpired app specific token with the given token code. If found, the token's last_accessed field is set to now and the token is returned. If not found, returns None. """ token_code = remove_unicode( Bytes.for_string_or_unicode(token_code).as_encoded_str()) prefix = token_code[:TOKEN_NAME_PREFIX_LENGTH] if len(prefix) != TOKEN_NAME_PREFIX_LENGTH: return None suffix = token_code[TOKEN_NAME_PREFIX_LENGTH:] # Lookup the token by its prefix. try: token = (AppSpecificAuthToken.select( AppSpecificAuthToken, User).join(User).where( AppSpecificAuthToken.token_name == prefix, ((AppSpecificAuthToken.expiration > datetime.now()) | (AppSpecificAuthToken.expiration >> None)), ).get()) if not token.token_secret.matches(suffix): return None assert len(prefix) == TOKEN_NAME_PREFIX_LENGTH assert len(suffix) >= MINIMUM_TOKEN_SUFFIX_LENGTH update_last_accessed(token) return token except AppSpecificAuthToken.DoesNotExist: pass return None
def test_build_unencoded_unicode_manifest(): config_json = json.dumps( { "config": { "author": "Sômé guy", }, "rootfs": {"type": "layers", "diff_ids": []}, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "base", "author": "Sômé guy", }, ], }, ensure_ascii=False, ) schema2_config = DockerSchema2Config(Bytes.for_string_or_unicode(config_json)) builder = DockerSchema2ManifestBuilder() builder.set_config(schema2_config) builder.add_layer("sha256:abc123", 123) builder.build()
def _load_manifest(self): digest = self._manifest_data[self._digest_key] size = self._manifest_data[self._size_key] manifest_bytes = self._content_retriever.get_manifest_bytes_with_digest( digest) if manifest_bytes is None: raise ManifestException( "Could not find child manifest with digest `%s`" % digest) if len(manifest_bytes) != size: raise ManifestException( "Size of manifest does not match that retrieved: %s vs %s", len(manifest_bytes), size, ) content_type = self._manifest_data[self._media_type_key] if content_type not in self._supported_types: raise ManifestException( "Unknown or unsupported manifest media type `%s`" % content_type) return self._supported_types[content_type]( Bytes.for_string_or_unicode(manifest_bytes), validate=False)
def build(self): """ Builds and returns the DockerSchema2ManifestList. """ assert self.manifests manifest_list_dict = { DOCKER_SCHEMA2_MANIFESTLIST_VERSION_KEY: 2, DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY: DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY: [{ DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY: manifest[2], DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY: manifest[0], DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY: manifest[1], DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY: manifest[3], } for manifest in self.manifests], } json_str = Bytes.for_string_or_unicode( json.dumps(manifest_list_dict, indent=3)) return DockerSchema2ManifestList(json_str)
def assert_gc_integrity(expect_storage_removed=True, check_oci_tags=True): """ Specialized assertion for ensuring that GC cleans up all dangling storages and labels, invokes the callback for images removed and doesn't invoke the callback for images *not* removed. """ # Add a callback for when images are removed. removed_image_storages = [] model.config.register_image_cleanup_callback(removed_image_storages.extend) # Store the number of dangling storages and labels. existing_storage_count = _get_dangling_storage_count() existing_label_count = _get_dangling_label_count() existing_manifest_count = _get_dangling_manifest_count() yield # Ensure the number of dangling storages, manifests and labels has not changed. updated_storage_count = _get_dangling_storage_count() assert updated_storage_count == existing_storage_count updated_label_count = _get_dangling_label_count() assert updated_label_count == existing_label_count, _get_dangling_labels() updated_manifest_count = _get_dangling_manifest_count() assert updated_manifest_count == existing_manifest_count # Ensure that for each call to the image+storage cleanup callback, the image and its # storage is not found *anywhere* in the database. for removed_image_and_storage in removed_image_storages: with pytest.raises(Image.DoesNotExist): Image.get(id=removed_image_and_storage.id) # Ensure that image storages are only removed if not shared. shared = Image.select().where( Image.storage == removed_image_and_storage.storage_id).count() if shared == 0: shared = (ManifestBlob.select().where( ManifestBlob.blob == removed_image_and_storage.storage_id).count()) if shared == 0: with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(id=removed_image_and_storage.storage_id) with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(uuid=removed_image_and_storage.storage.uuid) # Ensure all CAS storage is in the storage engine. preferred = storage.preferred_locations[0] for storage_row in ImageStorage.select(): if storage_row.cas_path: storage.get_content({preferred}, storage.blob_path( storage_row.content_checksum)) for blob_row in ApprBlob.select(): storage.get_content({preferred}, storage.blob_path(blob_row.digest)) # Ensure there are no danglings OCI tags. if check_oci_tags: oci_tags = {t.id for t in Tag.select()} referenced_oci_tags = {t.tag_id for t in TagToRepositoryTag.select()} assert not oci_tags - referenced_oci_tags # Ensure all tags have valid manifests. for manifest in {t.manifest for t in Tag.select()}: # Ensure that the manifest's blobs all exist. found_blobs = { b.blob.content_checksum for b in ManifestBlob.select().where( ManifestBlob.manifest == manifest) } parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(manifest.manifest_bytes), manifest.media_type.name) assert set(parsed.local_blob_digests) == found_blobs
def hash_password(password, salt=None): salt = salt or bcrypt.gensalt() salt = Bytes.for_string_or_unicode(salt).as_encoded_str() return bcrypt.hashpw(password.encode("utf-8"), salt)
def test_create_placeholder_blobs_on_first_pull(self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] # no blob placeholders are created for manifest lists - we don't have # the sub-manifests at manifest list creation time, so there's no way # to know which blobs the sub-manifest has. if test_params["manifest_type"] in [ DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, OCI_IMAGE_INDEX_CONTENT_TYPE, ]: pytest.skip( "manifest list detected - skipping blob placeholder test") repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response(test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"]) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) manifest = Manifest.filter(Manifest.digest == parsed.digest).get() mdict = parsed.manifest_dict layers = mdict.get("layers", mdict.get("fsLayers")) mblobs = ManifestBlob.filter(ManifestBlob.manifest == manifest) expected_count = len(layers) # schema 2 manifests have an extra config blob which we need to take into # consideration in the total count config_digest = "" if parsed.schema_version == 2: config_digest = parsed.config.digest expected_count += 1 assert mblobs.count() == expected_count for mblob in mblobs: blob = None layer = None # don't assert if digest belongs to a config blob if mblob.blob.content_checksum == config_digest: continue for layer in layers: digest = layer.get("digest", layer.get("blobSum")) if mblob.blob.content_checksum == digest: blob = mblob.blob layer = layer break assert blob is not None assert blob.image_size == layer.get("size", None) # the absence of an image storage placement for a blob indicates that it's # a placeholder blob, not yet downloaded from the upstream registry. placements = ImageStoragePlacement.filter( ImageStoragePlacement.storage == blob) assert placements.count() == 0
def pull( self, session, namespace, repo_name, tag_names, images, credentials=None, expected_failure=None, options=None, ): options = options or ProtocolOptions() scopes = options.scopes or ["repository:%s:pull" % self.repo_name(namespace, repo_name)] tag_names = [tag_names] if isinstance(tag_names, str) else tag_names # Ping! self.ping(session) # Perform auth and retrieve a token. token, _ = self.auth( session, credentials, namespace, repo_name, scopes=scopes, expected_failure=expected_failure, ) if token is None and not options.attempt_pull_without_token: return None headers = {} if token: headers = { "Authorization": "Bearer " + token, } if self.schema == "oci": headers["Accept"] = ",".join( options.accept_mimetypes if options.accept_mimetypes is not None else OCI_CONTENT_TYPES ) elif self.schema == "schema2": headers["Accept"] = ",".join( options.accept_mimetypes if options.accept_mimetypes is not None else DOCKER_SCHEMA2_CONTENT_TYPES ) manifests = {} image_ids = {} for tag_name in tag_names: # Retrieve the manifest for the tag or digest. response = self.conduct( session, "GET", "/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), tag_name), expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST), headers=headers, ) if response.status_code == 401: assert "WWW-Authenticate" in response.headers response.encoding = "utf-8" if expected_failure is not None: return None # Ensure the manifest returned by us is valid. ct = response.headers["Content-Type"] if self.schema == "schema1": assert ct in DOCKER_SCHEMA1_CONTENT_TYPES if options.require_matching_manifest_type: if self.schema == "schema1": assert ct in DOCKER_SCHEMA1_CONTENT_TYPES if self.schema == "schema2": assert ct in DOCKER_SCHEMA2_CONTENT_TYPES if self.schema == "oci": assert ct in OCI_CONTENT_TYPES manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct) manifests[tag_name] = manifest if manifest.schema_version == 1: image_ids[tag_name] = manifest.leaf_layer_v1_image_id # Verify the blobs. layer_index = 0 empty_count = 0 blob_digests = list(manifest.blob_digests) for image in images: if manifest.schema_version == 2 and image.is_empty: empty_count += 1 continue # If the layer is remote, then we expect the blob to *not* exist in the system. blob_digest = blob_digests[layer_index] expected_status = 404 if image.urls else 200 result = self.conduct( session, "GET", "/v2/%s/blobs/%s" % (self.repo_name(namespace, repo_name), blob_digest), expected_status=(expected_status, expected_failure, V2ProtocolSteps.GET_BLOB), headers=headers, options=options, ) if expected_status == 200: assert result.content == image.bytes layer_index += 1 assert (len(blob_digests) + empty_count) >= len( images ) # OCI/Schema 2 has 1 extra for config return PullResult(manifests=manifests, image_ids=image_ids)
def pull_list( self, session, namespace, repo_name, tag_names, manifestlist, credentials=None, expected_failure=None, options=None, ): options = options or ProtocolOptions() scopes = options.scopes or [ "repository:%s:push,pull" % self.repo_name(namespace, repo_name) ] tag_names = [tag_names] if isinstance(tag_names, str) else tag_names # Ping! self.ping(session) # Perform auth and retrieve a token. token, _ = self.auth( session, credentials, namespace, repo_name, scopes=scopes, expected_failure=expected_failure, ) if token is None: assert V2Protocol.FAILURE_CODES[V2ProtocolSteps.AUTH].get(expected_failure) return headers = { "Authorization": "Bearer " + token, "Accept": ",".join(MANIFEST_LIST_TYPES), } for tag_name in tag_names: # Retrieve the manifest for the tag or digest. response = self.conduct( session, "GET", "/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), tag_name), expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST_LIST), headers=headers, ) if expected_failure is not None: return None # Parse the returned manifest list and ensure it matches. ct = response.headers["Content-Type"] assert is_manifest_list_type(ct), "Expected list type, found: %s" % ct retrieved = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct) assert retrieved.schema_version == 2 assert retrieved.is_manifest_list assert retrieved.digest == manifestlist.digest # Pull each of the manifests inside and ensure they can be retrieved. for manifest_digest in retrieved.child_manifest_digests(): response = self.conduct( session, "GET", "/v2/%s/manifests/%s" % (self.repo_name(namespace, repo_name), manifest_digest), expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST), headers=headers, ) if expected_failure is not None: return None ct = response.headers["Content-Type"] manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct) assert not manifest.is_manifest_list assert manifest.digest == manifest_digest
def _get_test_file_contents(test_name, kind): filename = '%s.%s.json' % (test_name, kind) data_dir = os.path.dirname(__file__) with open(os.path.join(data_dir, 'conversion_data', filename), 'r') as f: return Bytes.for_string_or_unicode(f.read())
def test_parse_manifest_from_bytes(media_type, manifest_bytes): assert parse_manifest_from_bytes( Bytes.for_string_or_unicode(manifest_bytes), media_type, validate=False)
def python_value(self, value): if value is None: return None return Credential(Bytes.for_string_or_unicode(value).as_encoded_str())
def build_schema2(self, images, blobs, options): builder = DockerSchema2ManifestBuilder() for image in images: checksum = "sha256:" + hashlib.sha256(image.bytes).hexdigest() if image.urls is None: blobs[checksum] = image.bytes # If invalid blob references were requested, just make it up. if options.manifest_invalid_blob_references: checksum = "sha256:" + hashlib.sha256( "notarealthing").hexdigest() if not image.is_empty: builder.add_layer(checksum, len(image.bytes), urls=image.urls) def history_for_image(image): history = { "created": "2018-04-03T18:37:09.284840891Z", "created_by": (("/bin/sh -c #(nop) ENTRYPOINT %s" % image.config["Entrypoint"]) if image.config and image.config.get("Entrypoint") else "/bin/sh -c #(nop) %s" % image.id), } if image.is_empty: history["empty_layer"] = True return history config = { "os": "linux", "rootfs": { "type": "layers", "diff_ids": [] }, "history": [history_for_image(image) for image in images], } if options.with_broken_manifest_config: # NOTE: We are missing the history entry on purpose. config = { "os": "linux", "rootfs": { "type": "layers", "diff_ids": [] }, } if images and images[-1].config: config["config"] = images[-1].config config_json = json.dumps(config, ensure_ascii=options.ensure_ascii) schema2_config = DockerSchema2Config( Bytes.for_string_or_unicode(config_json), skip_validation_for_testing=options.with_broken_manifest_config, ) builder.set_config(schema2_config) blobs[schema2_config.digest] = schema2_config.bytes.as_encoded_str() return builder.build(ensure_ascii=options.ensure_ascii)
def retarget_tag( tag_name, manifest_id, is_reversion=False, now_ms=None, raise_on_error=False, ): """ Creates or updates a tag with the specified name to point to the given manifest under its repository. If this action is a reversion to a previous manifest, is_reversion should be set to True. Returns the newly created tag row or None on error. """ try: manifest = (Manifest.select( Manifest, MediaType).join(MediaType).where(Manifest.id == manifest_id).get()) except Manifest.DoesNotExist: if raise_on_error: raise RetargetTagException("Manifest requested no longer exists") return None # CHECK: Make sure that we are not mistargeting a schema 1 manifest to a tag with a different # name. if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES: try: parsed = DockerSchema1Manifest(Bytes.for_string_or_unicode( manifest.manifest_bytes), validate=False) if parsed.tag != tag_name: logger.error( "Tried to re-target schema1 manifest with tag `%s` to tag `%s", parsed.tag, tag_name, ) return None except MalformedSchema1Manifest as msme: logger.exception("Could not parse schema1 manifest") if raise_on_error: raise RetargetTagException(msme) return None legacy_image = get_legacy_image_for_manifest(manifest) now_ms = now_ms or get_epoch_timestamp_ms() now_ts = int(now_ms // 1000) with db_transaction(): # Lookup an existing tag in the repository with the same name and, if present, mark it # as expired. existing_tag = get_tag(manifest.repository_id, tag_name) if existing_tag is not None: _, okay = set_tag_end_ms(existing_tag, now_ms) # TODO: should we retry here and/or use a for-update? if not okay: return None # Create a new tag pointing to the manifest with a lifetime start of now. created = Tag.create( name=tag_name, repository=manifest.repository_id, lifetime_start_ms=now_ms, reversion=is_reversion, manifest=manifest, tag_kind=Tag.tag_kind.get_id("tag"), ) return created
def test_config_missing_required(): valid_index = json.loads(SAMPLE_INDEX) valid_index.pop("schemaVersion") with pytest.raises(MalformedIndex): OCIIndex(Bytes.for_string_or_unicode(json.dumps(valid_index)))
def test_malformed_manifest_lists(json_data): with pytest.raises(MalformedSchema2ManifestList): DockerSchema2ManifestList(Bytes.for_string_or_unicode(json_data))
def assert_gc_integrity(expect_storage_removed=True): """ Specialized assertion for ensuring that GC cleans up all dangling storages and labels, invokes the callback for images removed and doesn't invoke the callback for images *not* removed. """ # Add a callback for when images are removed. removed_image_storages = [] remove_callback = model.config.register_image_cleanup_callback( removed_image_storages.extend) # Store existing storages. We won't verify these for existence because they # were likely created as test data. existing_digests = set() for storage_row in ImageStorage.select(): if storage_row.cas_path: existing_digests.add(storage_row.content_checksum) for blob_row in ApprBlob.select(): existing_digests.add(blob_row.digest) # Store the number of dangling objects. existing_storage_count = _get_dangling_storage_count() existing_label_count = _get_dangling_label_count() existing_manifest_count = _get_dangling_manifest_count() # Yield to the GC test. with check_transitive_modifications(): try: yield finally: remove_callback() # Ensure the number of dangling storages, manifests and labels has not changed. updated_storage_count = _get_dangling_storage_count() assert updated_storage_count == existing_storage_count updated_label_count = _get_dangling_label_count() assert updated_label_count == existing_label_count, _get_dangling_labels() updated_manifest_count = _get_dangling_manifest_count() assert updated_manifest_count == existing_manifest_count # Ensure that for each call to the image+storage cleanup callback, the image and its # storage is not found *anywhere* in the database. for removed_image_and_storage in removed_image_storages: assert isinstance(removed_image_and_storage, Image) try: # NOTE: SQLite can and will reuse AUTOINCREMENT IDs occasionally, so if we find a row # with the same ID, make sure it does not have the same Docker Image ID. # See: https://www.sqlite.org/autoinc.html found_image = Image.get(id=removed_image_and_storage.id) assert (found_image.docker_image_id != removed_image_and_storage.docker_image_id ), "Found unexpected removed image %s under repo %s" % ( found_image.id, found_image.repository, ) except Image.DoesNotExist: pass # Ensure that image storages are only removed if not shared. shared = Image.select().where( Image.storage == removed_image_and_storage.storage_id).count() if shared == 0: shared = (ManifestBlob.select().where( ManifestBlob.blob == removed_image_and_storage.storage_id).count()) if shared == 0: shared = (UploadedBlob.select().where( UploadedBlob.blob == removed_image_and_storage.storage_id).count()) if shared == 0: with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(id=removed_image_and_storage.storage_id) with pytest.raises(ImageStorage.DoesNotExist): ImageStorage.get(uuid=removed_image_and_storage.storage.uuid) # Ensure all CAS storage is in the storage engine. preferred = storage.preferred_locations[0] for storage_row in ImageStorage.select(): if storage_row.content_checksum in existing_digests: continue if storage_row.cas_path: storage.get_content({preferred}, storage.blob_path( storage_row.content_checksum)) for blob_row in ApprBlob.select(): if blob_row.digest in existing_digests: continue storage.get_content({preferred}, storage.blob_path(blob_row.digest)) # Ensure all tags have valid manifests. for manifest in {t.manifest for t in Tag.select()}: # Ensure that the manifest's blobs all exist. found_blobs = { b.blob.content_checksum for b in ManifestBlob.select().where( ManifestBlob.manifest == manifest) } parsed = parse_manifest_from_bytes( Bytes.for_string_or_unicode(manifest.manifest_bytes), manifest.media_type.name) assert set(parsed.local_blob_digests) == found_blobs
def test_valid_remote_manifest(): manifest = DockerSchema2Manifest( Bytes.for_string_or_unicode(REMOTE_MANIFEST_BYTES)) assert manifest.config.size == 1885 assert ( str(manifest.config.digest) == "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" ) assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json" assert manifest.has_remote_layer assert len(manifest.filesystem_layers) == 4 assert manifest.filesystem_layers[0].compressed_size == 1234 assert ( str(manifest.filesystem_layers[0].digest) == "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736" ) assert manifest.filesystem_layers[0].is_remote assert manifest.filesystem_layers[0].urls == ["http://some/url"] assert manifest.leaf_filesystem_layer == manifest.filesystem_layers[3] assert not manifest.leaf_filesystem_layer.is_remote assert manifest.leaf_filesystem_layer.compressed_size == 73109 expected = set([str(layer.digest) for layer in manifest.filesystem_layers] + [manifest.config.digest]) blob_digests = set(manifest.blob_digests) local_digests = set(manifest.local_blob_digests) assert blob_digests == expected assert local_digests == (expected - {manifest.filesystem_layers[0].digest}) assert manifest.has_remote_layer assert manifest.get_leaf_layer_v1_image_id(None) is None assert manifest.get_legacy_image_ids(None) is None retriever = ContentRetrieverForTesting.for_config( { "config": { "Labels": {}, }, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "foo" }, { "created": "2018-04-12T18:37:09.284840891Z", "created_by": "bar" }, { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "foo" }, { "created": "2018-04-12T18:37:09.284840891Z", "created_by": "bar" }, ], }, "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", 1885, ) manifest_image_layers = list(manifest.get_layers(retriever)) assert len(manifest_image_layers) == len(list(manifest.filesystem_layers)) for index in range(0, 4): assert manifest_image_layers[index].blob_digest == str( manifest.filesystem_layers[index].digest)
def build(self, json_web_key=None, ensure_ascii=True): """ Builds a DockerSchema1Manifest object, with optional signature. NOTE: For backward compatibility, "JWS JSON Serialization" is used instead of "JWS Compact Serialization", since the latter **requires** that the "alg" headers be carried in the **protected** headers, which was never done before migrating to authlib (One shouldn't be using schema1 anyways) References: - https://tools.ietf.org/html/rfc7515#section-10.7 - https://docs.docker.com/registry/spec/manifest-v2-1/#signed-manifests """ payload = OrderedDict(self._base_payload) payload.update({ DOCKER_SCHEMA1_HISTORY_KEY: self._history, DOCKER_SCHEMA1_FS_LAYERS_KEY: self._fs_layer_digests, }) payload_str = json.dumps(payload, indent=3, ensure_ascii=ensure_ascii) if json_web_key is None: return DockerSchema1Manifest( Bytes.for_string_or_unicode(payload_str)) payload_str = Bytes.for_string_or_unicode(payload_str).as_encoded_str() split_point = payload_str.rfind(b"\n}") protected_payload = { DOCKER_SCHEMA1_FORMAT_TAIL_KEY: base64url_encode(payload_str[split_point:]).decode("ascii"), DOCKER_SCHEMA1_FORMAT_LENGTH_KEY: split_point, "time": datetime.utcnow().strftime(_ISO_DATETIME_FORMAT_ZULU), } # Flattened JSON serialization header jws = JsonWebSignature(algorithms=[_JWS_SIGNING_ALGORITHM]) headers = { "protected": protected_payload, "header": { "alg": _JWS_SIGNING_ALGORITHM }, } signed = jws.serialize_json(headers, payload_str, json_web_key.get_private_key()) protected = signed["protected"] signature = signed["signature"] logger.debug("Generated signature: %s", signature) logger.debug("Generated protected block: %s", protected) public_members = set(json_web_key.REQUIRED_JSON_FIELDS + json_web_key.ALLOWED_PARAMS) public_key = { comp: value for comp, value in list(json_web_key.as_dict().items()) if comp in public_members } public_key["kty"] = json_web_key.kty signature_block = { DOCKER_SCHEMA1_HEADER_KEY: { "jwk": public_key, "alg": _JWS_SIGNING_ALGORITHM }, DOCKER_SCHEMA1_SIGNATURE_KEY: signature, DOCKER_SCHEMA1_PROTECTED_KEY: protected, } logger.debug("Encoded signature block: %s", json.dumps(signature_block)) payload.update({DOCKER_SCHEMA1_SIGNATURES_KEY: [signature_block]}) json_str = json.dumps(payload, indent=3, ensure_ascii=ensure_ascii) return DockerSchema1Manifest(Bytes.for_string_or_unicode(json_str))
def test_invalid_index(): with pytest.raises(MalformedIndex): OCIIndex(Bytes.for_string_or_unicode("{}"))