def _should_evaluate(self, cache_entry: CachedPolicyEvaluation): if cache_entry is None: metrics.counter_inc( name="anchore_policy_evaluation_cache_misses_notfound") return EvaluationCacheManager.CacheStatus.missing # The cached result is not for this exact bundle content, so result is invalid if cache_entry.bundle_id != self.bundle_id: log.warn( "Unexpectedly got a cached evaluation for a different bundle id" ) metrics.counter_inc( name="anchore_policy_evaluation_cache_misses_notfound") return EvaluationCacheManager.CacheStatus.missing if cache_entry.bundle_digest == self.bundle_digest: # A feed sync has occurred since the eval was done or the image has been updated/reloaded, so inputs can have changed. Must be stale if self._inputs_changed(cache_entry.last_modified): metrics.counter_inc( name="anchore_policy_evaluation_cache_misses_stale") return EvaluationCacheManager.CacheStatus.stale else: return EvaluationCacheManager.CacheStatus.valid else: metrics.counter_inc( name="anchore_policy_evaluation_cache_misses_invalid") return EvaluationCacheManager.CacheStatus.invalid
def check_user_image_inline(user_id, image_id, tag, bundle): """ Execute a policy evaluation using the info in the request body including the bundle content :param user_id: :param image_id: :param tag: :param bundle: :return: """ timer = time.time() db = get_session() cache_mgr = None try: # Input validation if tag is None: # set tag value to a value that only matches wildcards tag = "*/*:*" try: img_obj = db.query(Image).get((image_id, user_id)) except: return make_response_error("Image not found", in_httpcode=404), 404 if not img_obj: log.info( "Request for evaluation of image that cannot be found: user_id = {}, image_id = {}" .format(user_id, image_id)) return make_response_error("Image not found", in_httpcode=404), 404 if evaluation_cache_enabled: timer2 = time.time() try: try: conn_timeout = ( ApiRequestContextProxy.get_service().configuration.get( "catalog_client_conn_timeout", DEFAULT_CACHE_CONN_TIMEOUT)) read_timeout = ( ApiRequestContextProxy.get_service().configuration.get( "catalog_client_read_timeout", DEFAULT_CACHE_READ_TIMEOUT)) cache_mgr = EvaluationCacheManager(img_obj, tag, bundle, conn_timeout, read_timeout) except ValueError as err: log.warn( "Could not leverage cache due to error in bundle data: {}" .format(err)) cache_mgr = None if cache_mgr is None: log.info( "Could not initialize cache manager for policy evaluation, skipping cache usage" ) else: cached_result = cache_mgr.refresh() if cached_result: metrics.counter_inc( name="anchore_policy_evaluation_cache_hits") metrics.histogram_observe( "anchore_policy_evaluation_cache_access_latency", time.time() - timer2, status="hit", ) log.info( "Returning cached result of policy evaluation for {}/{}, with tag {} and bundle {} with digest {}. Last evaluation: {}" .format( user_id, image_id, tag, cache_mgr.bundle_id, cache_mgr.bundle_digest, cached_result.get("last_modified"), )) return cached_result else: metrics.counter_inc( name="anchore_policy_evaluation_cache_misses") metrics.histogram_observe( "anchore_policy_evaluation_cache_access_latency", time.time() - timer2, status="miss", ) log.info( "Policy evaluation not cached, or invalid, executing evaluation for {}/{} with tag {} and bundle {} with digest {}" .format( user_id, image_id, tag, cache_mgr.bundle_id, cache_mgr.bundle_digest, )) except Exception as ex: log.exception( "Unexpected error operating on policy evaluation cache. Skipping use of cache." ) else: log.info("Policy evaluation cache disabled. Executing evaluation") # Build bundle exec. problems = [] executable_bundle = None try: # Allow deprecated gates here to support upgrade cases from old policy bundles. executable_bundle = build_bundle(bundle, for_tag=tag, allow_deprecated=True) if executable_bundle.init_errors: problems = executable_bundle.init_errors except InitializationError as e: log.exception( "Bundle construction and initialization returned errors") problems = e.causes eval_result = None if not problems: # Execute bundle try: eval_result = executable_bundle.execute( img_obj, tag, ExecutionContext(db_session=db, configuration={})) except Exception as e: log.exception( "Error executing policy bundle {} against image {} w/tag {}: {}" .format(bundle["id"], image_id, tag, e)) return ( make_response_error( "Internal bundle evaluation error", details={ "message": "Cannot execute given policy against the image due to errors executing the policy bundle: {}" .format(e) }, in_httpcode=500, ), 500, ) else: # Construct a failure eval with details on the errors and mappings to send to client eval_result = build_empty_error_execution(img_obj, tag, executable_bundle, errors=problems, warnings=[]) if (executable_bundle and executable_bundle.mapping and len(executable_bundle.mapping.mapping_rules) == 1): eval_result.executed_mapping = executable_bundle.mapping.mapping_rules[ 0] resp = PolicyEvaluation() resp.user_id = user_id resp.image_id = image_id resp.tag = tag resp.bundle = bundle resp.matched_mapping_rule = (eval_result.executed_mapping.json() if eval_result.executed_mapping else False) resp.last_modified = int(time.time()) resp.final_action = eval_result.bundle_decision.final_decision.name resp.final_action_reason = eval_result.bundle_decision.reason resp.matched_whitelisted_images_rule = ( eval_result.bundle_decision.whitelisted_image.json() if eval_result.bundle_decision.whitelisted_image else False) resp.matched_blacklisted_images_rule = ( eval_result.bundle_decision.blacklisted_image.json() if eval_result.bundle_decision.blacklisted_image else False) resp.result = eval_result.as_table_json() resp.created_at = int(time.time()) resp.evaluation_problems = [ problem_from_exception(i) for i in eval_result.errors ] resp.evaluation_problems += [ problem_from_exception(i) for i in eval_result.warnings ] if resp.evaluation_problems: for i in resp.evaluation_problems: log.warn( "Returning evaluation response for image {}/{} w/tag {} and bundle {} that contains error: {}" .format(user_id, image_id, tag, bundle["id"], json.dumps(i.to_json()))) metrics.histogram_observe( "anchore_policy_evaluation_time_seconds", time.time() - timer, status="fail", ) else: metrics.histogram_observe( "anchore_policy_evaluation_time_seconds", time.time() - timer, status="success", ) result = resp.to_json() # Never let the cache block returning results try: if evaluation_cache_enabled and cache_mgr is not None: cache_mgr.save(result) except Exception as ex: log.exception( "Failed saving policy result in cache. Skipping and continuing." ) db.commit() return result except HTTPException as e: db.rollback() log.exception("Caught exception in execution: {}".format(e)) raise except Exception as e: db.rollback() log.exception("Failed processing bundle evaluation: {}".format(e)) return ( make_response_error( "Unexpected internal error", details={"message": str(e)}, in_httpcode=500, ), 500, ) finally: db.close()
def import_image(operation_id, account, import_manifest: InternalImportManifest): """ The main thread of exec for importing an image :param operation_id: :param account: :param import_manifest: :return: """ timer = int(time.time()) analysis_events = [] config = localconfig.get_config() all_content_types = config.get("image_content_types", []) + config.get( "image_metadata_types", []) image_digest = import_manifest.digest try: catalog_client = internal_client_for(CatalogClient, account) # check to make sure image is still in DB catalog_client = internal_client_for(CatalogClient, account) try: image_record = catalog_client.get_image(image_digest) if not image_record: raise Exception("empty image record from catalog") except Exception as err: logger.debug_exception("Could not get image record") logger.warn( "dequeued image cannot be fetched from catalog - skipping analysis (" + str(image_digest) + ") - exception: " + str(err)) return True if image_record["analysis_status"] != taskstate.base_state("analyze"): logger.info( "dequeued image to import is not in base 'not_analyzed' state - skipping import" ) return True try: last_analysis_status = image_record["analysis_status"] image_record = update_analysis_started(catalog_client, image_digest, image_record) logger.info("Loading content from import") sbom_map = get_content(import_manifest, catalog_client) manifest = sbom_map.get("manifest") try: logger.info("processing image import data") image_data, analysis_manifest = process_import( image_record, sbom_map, import_manifest) except AnchoreException as e: event = events.ImageAnalysisFailed(user_id=account, image_digest=image_digest, error=e.to_dict()) analysis_events.append(event) raise # Store the manifest in the object store logger.info("storing image manifest") catalog_client.put_document(bucket="manifest_data", name=image_digest, inobj=json.dumps(manifest)) # Save the results to the upstream components and data stores logger.info("storing import result") store_analysis_results( account, image_digest, image_record, image_data, manifest, analysis_events, all_content_types, ) logger.info("updating image catalog record analysis_status") last_analysis_status = image_record["analysis_status"] image_record = update_analysis_complete(catalog_client, image_digest, image_record) try: analysis_events.extend( notify_analysis_complete(image_record, last_analysis_status)) except Exception as err: logger.warn( "failed to enqueue notification on image analysis state update - exception: " + str(err)) logger.info("analysis complete: " + str(account) + " : " + str(image_digest)) try: catalog_client.update_image_import_status(operation_id, status="complete") except Exception as err: logger.debug_exception( "failed updating import status success, will continue and rely on expiration for GC later" ) try: metrics.counter_inc(name="anchore_import_success") run_time = float(time.time() - timer) metrics.histogram_observe( "anchore_import_time_seconds", run_time, buckets=IMPORT_TIME_SECONDS_BUCKETS, status="success", ) except Exception as err: logger.warn(str(err)) except Exception as err: run_time = float(time.time() - timer) logger.exception("problem importing image - exception: " + str(err)) analysis_failed_metrics(run_time) # Transition the image record to failure status image_record = update_analysis_failed(catalog_client, image_digest, image_record) try: catalog_client.update_image_import_status(operation_id, status="failed") except Exception as err: logger.debug_exception( "failed updating import status failure, will continue and rely on expiration for GC later" ) if account and image_digest: for image_detail in image_record["image_detail"]: fulltag = (image_detail["registry"] + "/" + image_detail["repo"] + ":" + image_detail["tag"]) event = events.UserAnalyzeImageFailed(user_id=account, full_tag=fulltag, error=str(err)) analysis_events.append(event) finally: if analysis_events: emit_events(catalog_client, analysis_events) except Exception as err: logger.debug_exception("Could not import image") logger.warn("job processing bailed - exception: " + str(err)) raise err return True