def _manifest_dict(manifest): image = None if manifest.legacy_image_if_present is not None: image = image_dict(manifest.legacy_image, with_history=True) layers = None if not manifest.is_manifest_list: layers = registry_model.list_manifest_layers(manifest, storage) if layers is None: logger.debug('Missing layers for manifest `%s`', manifest.digest) abort(404) return { 'digest': manifest.digest, 'is_manifest_list': manifest.is_manifest_list, 'manifest_data': manifest.internal_manifest_bytes.as_unicode(), 'image': image, 'layers': ([_layer_dict(lyr.layer_info, idx) for idx, lyr in enumerate(layers)] if layers else None), }
def _manifest_dict(manifest): layers = None if not manifest.is_manifest_list: layers = registry_model.list_manifest_layers(manifest, storage) if layers is None: logger.debug("Missing layers for manifest `%s`", manifest.digest) abort(404) image = None if manifest.legacy_image_root_id: # NOTE: This is replicating our older response for this endpoint, but # returns empty for the metadata fields. This is to ensure back-compat # for callers still using the deprecated API. image = { "id": manifest.legacy_image_root_id, "created": format_date(datetime.utcnow()), "comment": "", "command": "", "size": 0, "uploading": False, "sort_index": 0, "ancestors": "", } return { "digest": manifest.digest, "is_manifest_list": manifest.is_manifest_list, "manifest_data": manifest.internal_manifest_bytes.as_unicode(), "config_media_type": manifest.config_media_type, "layers": ( [_layer_dict(lyr.layer_info, idx) for idx, lyr in enumerate(layers)] if layers else None ), "image": image, }
def test_vulnerability_report_incompatible_api_response(api, initialized_db): with fake_security_scanner(incompatible=True) as security_scanner: with pytest.raises(APIRequestFailure): manifest = manifest_for("devtable", "simple", "latest") layers = registry_model.list_manifest_layers(manifest, storage, True) api.vulnerability_report(manifest.digest)
def test_vulnerability_report(api, initialized_db): with fake_security_scanner() as security_scanner: manifest = manifest_for("devtable", "simple", "latest") layers = registry_model.list_manifest_layers(manifest, storage, True) assert manifest.digest not in security_scanner.index_reports.keys() assert api.vulnerability_report(manifest.digest) is None api.index(manifest, layers) report = api.vulnerability_report(manifest.digest) assert manifest.digest in security_scanner.vulnerability_reports.keys() assert report is not None
def verify_replication_for(namespace, repo_name, tag_name): repo_ref = registry_model.lookup_repository(namespace, repo_name) assert repo_ref tag = registry_model.get_repo_tag(repo_ref, tag_name) assert tag manifest = registry_model.get_manifest_for_tag(tag) assert manifest for layer in registry_model.list_manifest_layers(manifest, storage): if layer.blob.digest != EMPTY_LAYER_BLOB_DIGEST: QueueItem.select().where( QueueItem.queue_name**("%" + layer.blob.uuid + "%")).get() return "OK"
def _manifest_dict(manifest): layers = None if not manifest.is_manifest_list: layers = registry_model.list_manifest_layers(manifest, storage) if layers is None: logger.debug("Missing layers for manifest `%s`", manifest.digest) abort(404) return { "digest": manifest.digest, "is_manifest_list": manifest.is_manifest_list, "manifest_data": manifest.internal_manifest_bytes.as_unicode(), "config_media_type": manifest.config_media_type, "layers": ( [_layer_dict(lyr.layer_info, idx) for idx, lyr in enumerate(layers)] if layers else None ), }
def perform_indexing(self, start_token=None): whitelisted_namespaces = self.app.config.get( "SECURITY_SCANNER_V4_NAMESPACE_WHITELIST", []) try: indexer_state = self._secscan_api.state() except APIRequestFailure: return None def eligible_manifests(base_query): return (base_query.join(Repository).join(User).where( User.username << whitelisted_namespaces)) min_id = (start_token.min_id if start_token is not None else Manifest.select(fn.Min(Manifest.id)).scalar()) max_id = Manifest.select(fn.Max(Manifest.id)).scalar() if max_id is None or min_id is None or min_id > max_id: return None reindex_threshold = lambda: datetime.utcnow() - timedelta( seconds=self.app.config.get("SECURITY_SCANNER_V4_REINDEX_THRESHOLD" )) # TODO(alecmerdler): Filter out any `Manifests` that are still being uploaded def not_indexed_query(): return (eligible_manifests( Manifest.select()).switch(Manifest).join( ManifestSecurityStatus, JOIN.LEFT_OUTER).where(ManifestSecurityStatus.id >> None)) def index_error_query(): return (eligible_manifests(Manifest.select()).switch( Manifest).join(ManifestSecurityStatus).where( ManifestSecurityStatus.index_status == IndexStatus.FAILED, ManifestSecurityStatus.last_indexed < reindex_threshold(), )) def needs_reindexing_query(indexer_hash): return (eligible_manifests(Manifest.select()).switch( Manifest).join(ManifestSecurityStatus).where( ManifestSecurityStatus.indexer_hash != indexer_hash, ManifestSecurityStatus.last_indexed < reindex_threshold(), )) # 4^log10(total) gives us a scalable batch size into the billions. batch_size = int(4**log10(max(10, max_id - min_id))) iterator = itertools.chain( yield_random_entries( not_indexed_query, Manifest.id, batch_size, max_id, min_id, ), yield_random_entries( index_error_query, Manifest.id, batch_size, max_id, min_id, ), yield_random_entries( lambda: needs_reindexing_query(indexer_state.get("state", "")), Manifest.id, batch_size, max_id, min_id, ), ) for candidate, abt, num_remaining in iterator: manifest = ManifestDataType.for_manifest(candidate, None) layers = registry_model.list_manifest_layers( manifest, self.storage, True) logger.debug("Indexing %s/%s@%s" % (candidate.repository.namespace_user, candidate.repository.name, manifest.digest)) try: (report, state) = self._secscan_api.index(manifest, layers) except APIRequestFailure: logger.exception( "Failed to perform indexing, security scanner API error") return None with db_transaction(): ManifestSecurityStatus.delete().where( ManifestSecurityStatus.manifest == candidate).execute() ManifestSecurityStatus.create( manifest=candidate, repository=candidate.repository, error_json=report["err"], index_status=(IndexStatus.FAILED if report["state"] == IndexReportState.Index_Error else IndexStatus.COMPLETED), indexer_hash=state, indexer_version=IndexerVersion.V4, metadata_json={}, ) return ScanToken(max_id + 1)
def _index(self, iterator, reindex_threshold): def mark_manifest_unsupported(manifest): with db_transaction(): ManifestSecurityStatus.delete().where( ManifestSecurityStatus.manifest == manifest._db_id, ManifestSecurityStatus.repository == manifest.repository._db_id, ).execute() ManifestSecurityStatus.create( manifest=manifest._db_id, repository=manifest.repository._db_id, index_status=IndexStatus.MANIFEST_UNSUPPORTED, indexer_hash="none", indexer_version=IndexerVersion.V4, metadata_json={}, ) def should_skip_indexing(manifest_candidate): """Check whether this manifest was preempted by another worker. That would be the case if the manifest references a manifestsecuritystatus, or if the reindex threshold is no longer valid. """ if getattr(manifest_candidate, "manifestsecuritystatus", None): return manifest_candidate.manifestsecuritystatus.last_indexed >= reindex_threshold return len(manifest_candidate.manifestsecuritystatus_set) > 0 for candidate, abt, num_remaining in iterator: manifest = ManifestDataType.for_manifest(candidate, None) if manifest.is_manifest_list: mark_manifest_unsupported(manifest) continue layers = registry_model.list_manifest_layers( manifest, self.storage, True) if layers is None or len(layers) == 0: logger.warning( "Cannot index %s/%s@%s due to manifest being invalid (manifest has no layers)" % ( candidate.repository.namespace_user, candidate.repository.name, manifest.digest, )) mark_manifest_unsupported(manifest) continue if should_skip_indexing(candidate): logger.debug("Another worker preempted this worker") abt.set() continue logger.debug("Indexing manifest [%d] %s/%s@%s" % ( manifest._db_id, candidate.repository.namespace_user, candidate.repository.name, manifest.digest, )) try: (report, state) = self._secscan_api.index(manifest, layers) except InvalidContentSent as ex: mark_manifest_unsupported(manifest) logger.exception( "Failed to perform indexing, invalid content sent") continue except APIRequestFailure as ex: logger.exception( "Failed to perform indexing, security scanner API error") continue if report["state"] == IndexReportState.Index_Finished: index_status = IndexStatus.COMPLETED elif report["state"] == IndexReportState.Index_Error: index_status = IndexStatus.FAILED else: # Unknown state don't save anything continue with db_transaction(): ManifestSecurityStatus.delete().where( ManifestSecurityStatus.manifest == candidate).execute() ManifestSecurityStatus.create( manifest=candidate, repository=candidate.repository, error_json=report["err"], index_status=index_status, indexer_hash=state, indexer_version=IndexerVersion.V4, metadata_json={}, )
def perform_indexing(self, start_token=None): try: indexer_state = self._secscan_api.state() except APIRequestFailure: return None min_id = (start_token.min_id if start_token is not None else Manifest.select(fn.Min(Manifest.id)).scalar()) max_id = Manifest.select(fn.Max(Manifest.id)).scalar() if max_id is None or min_id is None or min_id > max_id: return None iterator = self._get_manifest_iterator(indexer_state, min_id, max_id) def mark_manifest_unsupported(manifest): with db_transaction(): ManifestSecurityStatus.delete().where( ManifestSecurityStatus.manifest == manifest._db_id, ManifestSecurityStatus.repository == manifest.repository._db_id, ).execute() ManifestSecurityStatus.create( manifest=manifest._db_id, repository=manifest.repository._db_id, index_status=IndexStatus.MANIFEST_UNSUPPORTED, indexer_hash="none", indexer_version=IndexerVersion.V4, metadata_json={}, ) for candidate, abt, num_remaining in iterator: manifest = ManifestDataType.for_manifest(candidate, None) if manifest.is_manifest_list: mark_manifest_unsupported(manifest) continue layers = registry_model.list_manifest_layers( manifest, self.storage, True) if layers is None or len(layers) == 0: logger.warning( "Cannot index %s/%s@%s due to manifest being invalid (manifest has no layers)" % ( candidate.repository.namespace_user, candidate.repository.name, manifest.digest, )) mark_manifest_unsupported(manifest) continue logger.debug("Indexing %s/%s@%s" % (candidate.repository.namespace_user, candidate.repository.name, manifest.digest)) try: (report, state) = self._secscan_api.index(manifest, layers) except InvalidContentSent as ex: mark_manifest_unsupported(manifest) logger.exception( "Failed to perform indexing, invalid content sent") return None except APIRequestFailure as ex: logger.exception( "Failed to perform indexing, security scanner API error") return None with db_transaction(): ManifestSecurityStatus.delete().where( ManifestSecurityStatus.manifest == candidate).execute() ManifestSecurityStatus.create( manifest=candidate, repository=candidate.repository, error_json=report["err"], index_status=(IndexStatus.FAILED if report["state"] == IndexReportState.Index_Error else IndexStatus.COMPLETED), indexer_hash=state, indexer_version=IndexerVersion.V4, metadata_json={}, ) return ScanToken(max_id + 1)