def manifest_for(namespace, repository, tagname): repository_ref = registry_model.lookup_repository(namespace, repository) tag = registry_model.get_repo_tag(repository_ref, tagname, include_legacy_image=True) return registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
def get(self, namespace, repository, tag, parsed_args): """ List the images for the specified repository tag. """ repo_ref = registry_model.lookup_repository(namespace, repository) if repo_ref is None: raise NotFound() tag_ref = registry_model.get_repo_tag(repo_ref, tag, include_legacy_image=True) if tag_ref is None: raise NotFound() if tag_ref.legacy_image_if_present is None: return {"images": []} image_id = tag_ref.legacy_image.docker_image_id all_images = None if parsed_args["owned"]: # TODO: Remove the `owned` image concept once we are fully on V2_2. all_images = registry_model.get_legacy_images_owned_by_tag(tag_ref) else: image_with_parents = registry_model.get_legacy_image( repo_ref, image_id, include_parents=True) if image_with_parents is None: raise NotFound() all_images = [image_with_parents] + image_with_parents.parents return { "images": [image_dict(image) for image in all_images], }
def test_get_security_info_with_pull_secret(endpoint, anonymous_allowed, auth_headers, expected_code, client): with patch("features.ANONYMOUS_ACCESS", anonymous_allowed): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) params = { "repository": "devtable/simple", "imageid": tag.manifest.legacy_image_root_id, "manifestref": manifest.digest, } headers = {} if auth_headers is not None: headers["Authorization"] = auth_headers conduct_api_call(client, endpoint, "GET", params, None, headers=headers, expected_code=expected_code)
def test_load_security_information_api_responses(secscan_api_response, initialized_db): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) registry_model.populate_legacy_images_for_testing(manifest, storage) legacy_image_row = shared.get_legacy_image_for_manifest(manifest._db_id) assert legacy_image_row is not None set_secscan_status(legacy_image_row, True, 3) secscan = V2SecurityScanner(app, instance_keys, storage) secscan._legacy_secscan_api = mock.Mock() secscan._legacy_secscan_api.get_layer_data.return_value = secscan_api_response security_information = secscan.load_security_information( manifest).security_information assert isinstance(security_information, SecurityInformation) assert security_information.Layer.Name == secscan_api_response[ "Layer"].get("Name", "") assert security_information.Layer.ParentName == secscan_api_response[ "Layer"].get("ParentName", "") assert security_information.Layer.IndexedByVersion == secscan_api_response[ "Layer"].get("IndexedByVersion", None) assert len(security_information.Layer.Features) == len( secscan_api_response["Layer"].get("Features", []))
def test_generate_url(initialized_db): try: application.register_blueprint(v2_bp, url_prefix="/v2") except: # Already registered. pass repo_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repo_ref, "latest") manifest = tag.manifest blobs = registry_model.get_manifest_local_blobs(manifest) retriever = BlobURLRetriever(storage, instance_keys, application) headers = retriever.headers_for_download(repo_ref, blobs[0]) identity, _ = identity_from_bearer_token(headers["Authorization"][0]) assert len(identity.provides) == 1 provide = list(identity.provides)[0] assert provide.role == "read" assert provide.name == "simple" assert provide.namespace == "devtable" assert provide.type == "repository" assert retriever.url_for_download(repo_ref, blobs[0]).startswith( "http://localhost:5000/v2/devtable/simple/blobs/" )
def test_load_security_information_queued(initialized_db, set_secscan_config): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) secscan = V4SecurityScanner(app, instance_keys, storage) assert secscan.load_security_information(manifest).status == ScanLookupStatus.NOT_YET_INDEXED
def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref, registry_model): try: repository_ref = registry_model.lookup_repository( namespace_name, repo_name, raise_on_error=True, manifest_ref=manifest_ref ) except RepositoryDoesNotExist as e: image_pulls.labels("v2", "tag", 404).inc() raise NameUnknown("repository not found") try: tag = registry_model.get_repo_tag(repository_ref, manifest_ref, raise_on_error=True) except TagDoesNotExist as e: if registry_model.has_expired_tag(repository_ref, manifest_ref): logger.debug( "Found expired tag %s for repository %s/%s", manifest_ref, namespace_name, repo_name ) msg = ( "Tag %s was deleted or has expired. To pull, revive via time machine" % manifest_ref ) image_pulls.labels("v2", "tag", 404).inc() raise TagExpired(msg) image_pulls.labels("v2", "tag", 404).inc() raise ManifestUnknown(str(e)) manifest = registry_model.get_manifest_for_tag(tag) if manifest is None: # Something went wrong. image_pulls.labels("v2", "tag", 400).inc() raise ManifestInvalid() try: manifest_bytes, manifest_digest, manifest_media_type = _rewrite_schema_if_necessary( namespace_name, repo_name, manifest_ref, manifest, registry_model ) except (ManifestException, ManifestDoesNotExist) as e: image_pulls.labels("v2", "tag", 404).inc() raise ManifestUnknown(str(e)) if manifest_bytes is None: image_pulls.labels("v2", "tag", 404).inc() raise ManifestUnknown() track_and_log( "pull_repo", repository_ref, analytics_name="pull_repo_100x", analytics_sample=0.01, tag=manifest_ref, ) image_pulls.labels("v2", "tag", 200).inc() return Response( manifest_bytes.as_unicode(), status=200, headers={ "Content-Type": manifest_media_type, "Docker-Content-Digest": manifest_digest, }, )
def test_load_security_information_success(initialized_db, set_secscan_config): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) ManifestSecurityStatus.create( manifest=manifest._db_id, repository=repository_ref._db_id, error_json={}, index_status=IndexStatus.COMPLETED, indexer_hash="abc", indexer_version=IndexerVersion.V4, metadata_json={}, ) secscan = V4SecurityScanner(app, instance_keys, storage) secscan._secscan_api = mock.Mock() secscan._secscan_api.vulnerability_report.return_value = { "manifest_hash": manifest.digest, "state": "IndexFinished", "packages": {}, "distributions": {}, "repository": {}, "environments": {}, "package_vulnerabilities": {}, "success": True, "err": "", } result = secscan.load_security_information(manifest) assert result.status == ScanLookupStatus.SUCCESS assert result.security_information == SecurityInformation( Layer(manifest.digest, "", "", 4, []))
def test_get_layer(self): """ Test for basic retrieval of layers from the security scanner. """ repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, SIMPLE_REPO) repo_tag = registry_model.get_repo_tag(repo_ref, "latest") manifest = registry_model.get_manifest_for_tag(repo_tag) registry_model.populate_legacy_images_for_testing(manifest, storage) with fake_security_scanner() as security_scanner: # Ensure the layer doesn't exist yet. self.assertFalse( security_scanner.has_layer( security_scanner.layer_id(manifest))) self.assertIsNone(self.api.get_layer_data(manifest)) # Add the layer. security_scanner.add_layer(security_scanner.layer_id(manifest)) # Retrieve the results. result = self.api.get_layer_data(manifest, include_vulnerabilities=True) self.assertIsNotNone(result) self.assertEquals(result["Layer"]["Name"], security_scanner.layer_id(manifest))
def test_get_security_info_with_pull_secret(endpoint, client): repository_ref = registry_model.lookup_repository('devtable', 'simple') tag = registry_model.get_repo_tag(repository_ref, 'latest', include_legacy_image=True) manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True) params = { 'repository': 'devtable/simple', 'imageid': tag.legacy_image.docker_image_id, 'manifestref': manifest.digest, } headers = { 'Authorization': 'Basic %s' % base64.b64encode('devtable:password'), } conduct_api_call(client, endpoint, 'GET', params, None, headers=headers, expected_code=200)
def test_load_security_information_api_returns_none(initialized_db, set_secscan_config): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True) manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True) ManifestSecurityStatus.create( manifest=manifest._db_id, repository=repository_ref._db_id, error_json={}, index_status=IndexStatus.COMPLETED, indexer_hash="abc", indexer_version=IndexerVersion.V4, metadata_json={}, ) secscan = V4SecurityScanner(app, instance_keys, storage) secscan._secscan_api = mock.Mock() secscan._secscan_api.vulnerability_report.return_value = None assert secscan.load_security_information( manifest).status == ScanLookupStatus.NOT_YET_INDEXED
def test_load_security_information_api_request_failure(initialized_db, set_secscan_config): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True) manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True) mss = ManifestSecurityStatus.create( manifest=manifest._db_id, repository=repository_ref._db_id, error_json={}, index_status=IndexStatus.COMPLETED, indexer_hash="abc", indexer_version=IndexerVersion.V4, metadata_json={}, ) secscan = V4SecurityScanner(app, instance_keys, storage) secscan._secscan_api = mock.Mock() secscan._secscan_api.vulnerability_report.side_effect = APIRequestFailure() assert secscan.load_security_information( manifest).status == ScanLookupStatus.COULD_NOT_LOAD assert not ManifestSecurityStatus.select().where( ManifestSecurityStatus.id == mss.id).exists()
def test_get_security_info_with_pull_secret(endpoint, client): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True) manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True) params = { "repository": "devtable/simple", "imageid": tag.legacy_image.docker_image_id, "manifestref": manifest.digest, } headers = { "Authorization": "Basic %s" % base64.b64encode("devtable:password"), } conduct_api_call(client, endpoint, "GET", params, None, headers=headers, expected_code=200)
def test_load_security_information_api_responses(secscan_api_response, initialized_db): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True) manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True, include_legacy_image=True) set_secscan_status(Image.get(id=manifest.legacy_image._db_id), True, 3) secscan = V2SecurityScanner(app, instance_keys, storage) secscan._legacy_secscan_api = mock.Mock() secscan._legacy_secscan_api.get_layer_data.return_value = secscan_api_response security_information = secscan.load_security_information( manifest).security_information assert isinstance(security_information, SecurityInformation) assert security_information.Layer.Name == secscan_api_response[ "Layer"].get("Name", "") assert security_information.Layer.ParentName == secscan_api_response[ "Layer"].get("ParentName", "") assert security_information.Layer.IndexedByVersion == secscan_api_response[ "Layer"].get("IndexedByVersion", None) assert len(security_information.Layer.Features) == len( secscan_api_response["Layer"].get("Features", []))
def test_load_security_information_queued(initialized_db): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True) manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True) secscan = V4SecurityScanner(app, instance_keys, storage) assert secscan.load_security_information(manifest).status == ScanLookupStatus.NOT_YET_INDEXED
def test_e2e_query_count_manifest_norewrite(client, app): repo_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repo_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) params = { "repository": "devtable/simple", "manifest_ref": manifest.digest, } user = model.user.get_user("devtable") access = [{ "type": "repository", "name": "devtable/simple", "actions": ["pull", "push"], }] context, subject = build_context_and_subject( ValidatedAuthContext(user=user)) token = generate_bearer_token(realapp.config["SERVER_HOSTNAME"], subject, context, access, 600, instance_keys) headers = { "Authorization": "Bearer %s" % token.decode("ascii"), } # Conduct a call to prime the instance key and other caches. conduct_call( client, "v2.write_manifest_by_digest", url_for, "PUT", params, expected_code=201, headers=headers, raw_body=manifest.internal_manifest_bytes.as_encoded_str(), ) timecode = time.time() def get_time(): return timecode + 10 with patch("time.time", get_time): # Necessary in order to have the tag updates not occur in the same second, which is the # granularity supported currently. with count_queries() as counter: conduct_call( client, "v2.write_manifest_by_digest", url_for, "PUT", params, expected_code=201, headers=headers, raw_body=manifest.internal_manifest_bytes.as_encoded_str(), ) assert counter.count <= 27
def test_creates_manifest_on_first_pull(self, test_name, proxy_manifest_response): test_params = storage_test_cases[test_name] repo = f"{self.orgname}/{test_params['image_name']}" params = { "repository": repo, "manifest_ref": test_params["manifest_ref"], } proxy_mock = proxy_manifest_response( test_params["manifest_ref"], test_params["manifest_json"], test_params["manifest_type"], ) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): headers = _get_auth_headers(self.sub, self.ctx, repo) headers["Accept"] = ", ".join( DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union( DOCKER_SCHEMA1_CONTENT_TYPES)) conduct_call( self.client, test_params["view_name"], url_for, "GET", params, expected_code=200, headers=headers, ) repository_ref = registry_model.lookup_repository( self.orgname, test_params["image_name"]) assert repository_ref is not None tag = registry_model.get_repo_tag(repository_ref, test_params["manifest_ref"]) # when testing the fetch_manifest_by_digest view the tag created is temporary, # and it does not refer to the manifest digest (manifest_ref), so we need to # fetch it by its link to the repository instead. if test_params["ref_type"] == "digest": tag = Tag.filter(Tag.repository_id == repository_ref.id).get() # get_manifest_for_tag returns a tag of datatypes.Tag, so we convert # the one we have to that type. tag = datatypes.Tag.for_tag(tag, SyntheticIDHandler()) assert tag is not None manifest = registry_model.get_manifest_for_tag(tag) assert manifest is not None output_manifest = manifest.get_parsed_manifest() input_manifest = parse_manifest_from_bytes( Bytes.for_string_or_unicode(test_params["manifest_json"]), test_params["manifest_type"], sparse_manifest_support=True, ) assert output_manifest.schema_version == input_manifest.schema_version assert output_manifest.media_type == input_manifest.media_type assert output_manifest.is_manifest_list == input_manifest.is_manifest_list assert output_manifest.digest == input_manifest.digest assert output_manifest.manifest_dict == input_manifest.manifest_dict
def committed_tags(self): """ Returns the tags committed by this builder, if any. """ return [ registry_model.get_repo_tag(self._repository_ref, tag_name) for tag_name in self._builder_state.tags.keys() ]
def committed_tags(self): """ Returns the tags committed by this builder, if any. """ return [ registry_model.get_repo_tag(self._repository_ref, tag_name, include_legacy_image=True) for tag_name in list(self._builder_state.tags.keys()) ]
def test_change_tag_expiration(client, app): with client_with_identity("devtable", client) as cl: params = { "repository": "devtable/simple", "tag": "latest", } repo_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repo_ref, "latest") updated_expiration = tag.lifetime_start_ts + 60 * 60 * 24 request_body = { "expiration": updated_expiration, } conduct_api_call(cl, RepositoryTag, "put", params, request_body, 201) tag = registry_model.get_repo_tag(repo_ref, "latest") assert tag.lifetime_end_ts == updated_expiration
def test_load_security_information_unknown_manifest(initialized_db): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) registry_model.populate_legacy_images_for_testing(manifest, storage) # Delete the manifest. Manifest.get(id=manifest._db_id).delete_instance(recursive=True) secscan = V2SecurityScanner(app, instance_keys, storage) assert (secscan.load_security_information(manifest).status == ScanLookupStatus.UNSUPPORTED_FOR_INDEXING)
def test_perform_indexing_manifest_list(initialized_db, set_secscan_config): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) Manifest.update(media_type=MediaType.get( name=DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE)).execute() secscan = V4SecurityScanner(app, instance_keys, storage) secscan._secscan_api = mock.Mock() secscan.perform_indexing() assert ManifestSecurityStatus.select().count() == Manifest.select().count() for mss in ManifestSecurityStatus.select(): assert mss.index_status == IndexStatus.MANIFEST_UNSUPPORTED
def test_deprecated_route(client): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) with client_with_identity("devtable", client) as cl: resp = conduct_api_call( cl, RepositoryImageSecurity, "get", {"repository": "devtable/simple", "imageid": manifest.legacy_image_root_id}, expected_code=200, ) assert resp.headers["Deprecation"] == "true"
def verify_replication_for(namespace, repo_name, tag_name): repo_ref = registry_model.lookup_repository(namespace, repo_name) assert repo_ref tag = registry_model.get_repo_tag(repo_ref, tag_name) assert tag manifest = registry_model.get_manifest_for_tag(tag) assert manifest for layer in registry_model.list_manifest_layers(manifest, storage): if layer.blob.digest != EMPTY_LAYER_BLOB_DIGEST: QueueItem.select().where( QueueItem.queue_name**("%" + layer.blob.uuid + "%")).get() return "OK"
def test_deprecated_route(client): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True) manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True) image = shared.get_legacy_image_for_manifest(manifest._db_id) with client_with_identity("devtable", client) as cl: resp = conduct_api_call( cl, RepositoryImageSecurity, "get", {"repository": "devtable/simple", "imageid": image.docker_image_id}, expected_code=200, ) assert resp.headers["Deprecation"] == "true"
def test_get_security_info_with_pull_secret(endpoint, client): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) params = { "repository": "devtable/simple", "imageid": tag.manifest.legacy_image_root_id, "manifestref": manifest.digest, } headers = { "Authorization": gen_basic_auth("devtable", "password"), } conduct_api_call(client, endpoint, "GET", params, None, headers=headers, expected_code=200)
def test_load_security_information_failed_to_index(initialized_db): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) registry_model.populate_legacy_images_for_testing(manifest, storage) # Set the index status. image = shared.get_legacy_image_for_manifest(manifest._db_id) image.security_indexed = False image.security_indexed_engine = 3 image.save() secscan = V2SecurityScanner(app, instance_keys, storage) assert secscan.load_security_information( manifest).status == ScanLookupStatus.FAILED_TO_INDEX
def test_load_security_information_failed_to_index(initialized_db): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True) manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True) ManifestSecurityStatus.create( manifest=manifest._db_id, repository=repository_ref._db_id, error_json='failed to fetch layers: encountered error while fetching a layer: fetcher: unknown content-type "binary/octet-stream"', index_status=IndexStatus.FAILED, indexer_hash="", indexer_version=IndexerVersion.V4, metadata_json={}, ) secscan = V4SecurityScanner(app, instance_keys, storage) assert secscan.load_security_information(manifest).status == ScanLookupStatus.FAILED_TO_INDEX
def get(self, namespace, repository, tag, parsed_args): """ List the images for the specified repository tag. """ repo_ref = registry_model.lookup_repository(namespace, repository) if repo_ref is None: raise NotFound() tag_ref = registry_model.get_repo_tag(repo_ref, tag) if tag_ref is None: raise NotFound() if parsed_args["owned"]: # NOTE: This is deprecated, so we just return empty now. return {"images": []} manifest = registry_model.get_manifest_for_tag(tag_ref) if manifest is None: raise NotFound() legacy_image = registry_model.get_legacy_image( repo_ref, manifest.legacy_image_root_id, storage ) if legacy_image is None: raise NotFound() # NOTE: This is replicating our older response for this endpoint, but # returns empty for the metadata fields. This is to ensure back-compat # for callers still using the deprecated API, while not having to load # all the manifests from storage. return { "images": [ { "id": image_id, "created": format_date(datetime.utcfromtimestamp(tag_ref.lifetime_start_ts)), "comment": "", "command": "", "size": 0, "uploading": False, "sort_index": 0, "ancestors": "", } for image_id in legacy_image.full_image_id_chain ] }
def move_tag(repository, tag, image_ids, expect_gc=True): namespace = repository.namespace_user.username name = repository.name repo_ref = RepositoryReference.for_repo_obj(repository) builder = DockerSchema1ManifestBuilder(namespace, name, tag) # NOTE: Building root to leaf. parent_id = None for image_id in image_ids: config = { "id": image_id, "config": { "Labels": { "foo": "bar", "meh": "grah", } }, } if parent_id: config["parent"] = parent_id # Create a storage row for the layer blob. _, layer_blob_digest = _populate_blob(repository, image_id.encode("ascii")) builder.insert_layer(layer_blob_digest, json.dumps(config)) parent_id = image_id # Store the manifest. manifest = builder.build(docker_v2_signing_key) registry_model.create_manifest_and_retarget_tag(repo_ref, manifest, tag, storage, raise_on_error=True) tag_ref = registry_model.get_repo_tag(repo_ref, tag) manifest_ref = registry_model.get_manifest_for_tag(tag_ref) registry_model.populate_legacy_images_for_testing(manifest_ref, storage) if expect_gc: assert gc_now(repository) == expect_gc