def test_repository_tag_history(namespace, name, expected_tag_count, has_expired, registry_model, with_size_fallback): # Pre-cache media type loads to ensure consistent query count. Manifest.media_type.get_name(1) # If size fallback is requested, delete the sizes on the manifest rows. if with_size_fallback: Manifest.update(layers_compressed_size=None).execute() repository_ref = registry_model.lookup_repository(namespace, name) with assert_query_count(2 if with_size_fallback else 1): history, has_more = registry_model.list_repository_tag_history( repository_ref) assert not has_more assert len(history) == expected_tag_count for tag in history: # Retrieve the manifest to ensure it doesn't issue extra queries. tag.manifest # Verify that looking up the size doesn't issue extra queries. tag.manifest_layers_size if has_expired: # Ensure the latest tag is marked expired, since there is an expired one. with assert_query_count(1): assert registry_model.has_expired_tag(repository_ref, "latest")
def test_perform_indexing_manifest_list(initialized_db, set_secscan_config): repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.get_repo_tag(repository_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) Manifest.update(media_type=MediaType.get( name=DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE)).execute() secscan = V4SecurityScanner(app, instance_keys, storage) secscan._secscan_api = mock.Mock() secscan.perform_indexing() assert ManifestSecurityStatus.select().count() == Manifest.select().count() for mss in ManifestSecurityStatus.select(): assert mss.index_status == IndexStatus.MANIFEST_UNSUPPORTED
def _backfill_manifests(self): try: Manifest.select().where( Manifest.layers_compressed_size >> None).get() except Manifest.DoesNotExist: logger.debug("Manifest backfill worker has completed; skipping") return False iterator = yield_random_entries( lambda: Manifest.select().where(Manifest.layers_compressed_size >> None), Manifest.id, 250, Manifest.select(fn.Max(Manifest.id)).scalar(), 1, ) for manifest_row, abt, _ in iterator: if manifest_row.layers_compressed_size is not None: logger.debug("Another worker preempted this worker") abt.set() continue logger.debug("Setting layers compressed size for manifest %s", manifest_row.id) layers_compressed_size = -1 config_media_type = None manifest_bytes = Bytes.for_string_or_unicode( manifest_row.manifest_bytes) try: parsed = parse_manifest_from_bytes( manifest_bytes, manifest_row.media_type.name, validate=False) layers_compressed_size = parsed.layers_compressed_size if layers_compressed_size is None: layers_compressed_size = 0 config_media_type = parsed.config_media_type or None except ManifestException as me: logger.warning( "Got exception when trying to parse manifest %s: %s", manifest_row.id, me) assert layers_compressed_size is not None updated = (Manifest.update( layers_compressed_size=layers_compressed_size, config_media_type=config_media_type, ).where(Manifest.id == manifest_row.id, Manifest.layers_compressed_size >> None).execute()) if updated != 1: logger.debug("Another worker preempted this worker") abt.set() continue return True
def _update_manifest_for_tag( self, repo_ref: RepositoryReference, tag: Tag, manifest: Manifest, manifest_ref: str, create_manifest_fn, ) -> tuple[Tag, bool]: """ Updates a placeholder manifest with the given tag name. If the manifest is stale, downloads it from the upstream registry and creates a new manifest and retargets the tag. A manifest is considered stale when the manifest's digest changed in the upstream registry. A manifest is considered a placeholder when its db entry exists, but its manifest_bytes field is empty. Raises UpstreamRegistryError if the upstream registry returns anything other than 200. Raises ManifestDoesNotExist if the given manifest was not found in the database. Returns a new tag if one was created, or the existing one with a manifest freshly out of the database, and a boolean indicating whether the returned tag was newly created or not. """ upstream_manifest = None upstream_digest = self._proxy.manifest_exists(manifest_ref, ACCEPTED_MEDIA_TYPES) up_to_date = manifest.digest == upstream_digest # manifest_exists will return an empty/None digest when the upstream # registry omits the docker-content-digest header. if not upstream_digest: upstream_manifest = self._pull_upstream_manifest( repo_ref.name, manifest_ref) up_to_date = manifest.digest == upstream_manifest.digest placeholder = manifest.internal_manifest_bytes.as_unicode() == "" if up_to_date and not placeholder: return tag, False if upstream_manifest is None: upstream_manifest = self._pull_upstream_manifest( repo_ref.name, manifest_ref) self._enforce_repository_quota(repo_ref) if up_to_date and placeholder: with db_disallow_replica_use(): with db_transaction(): q = ManifestTable.update( manifest_bytes=upstream_manifest.bytes.as_unicode(), layers_compressed_size=upstream_manifest. layers_compressed_size, ).where(ManifestTable.id == manifest.id) q.execute() self._create_placeholder_blobs(upstream_manifest, manifest.id, repo_ref.id) db_tag = oci.tag.get_tag_by_manifest_id( repo_ref.id, manifest.id) self._recalculate_repository_size(repo_ref) return Tag.for_tag(db_tag, self._legacy_image_id_handler), False # if we got here, the manifest is stale, so we both create a new manifest # entry in the db, and retarget the tag. _, tag = create_manifest_fn(repo_ref, upstream_manifest, manifest_ref) return tag, True