def _should_evaluate(self, cache_entry: CachedPolicyEvaluation):
if cache_entry is None:
metrics.counter_inc(
name="anchore_policy_evaluation_cache_misses_notfound")
return EvaluationCacheManager.CacheStatus.missing
# The cached result is not for this exact bundle content, so result is invalid
if cache_entry.bundle_id != self.bundle_id:
log.warn(
"Unexpectedly got a cached evaluation for a different bundle id"
)
metrics.counter_inc(
name="anchore_policy_evaluation_cache_misses_notfound")
return EvaluationCacheManager.CacheStatus.missing
if cache_entry.bundle_digest == self.bundle_digest:
# A feed sync has occurred since the eval was done or the image has been updated/reloaded, so inputs can have changed. Must be stale
if self._inputs_changed(cache_entry.last_modified):
metrics.counter_inc(
name="anchore_policy_evaluation_cache_misses_stale")
return EvaluationCacheManager.CacheStatus.stale
else:
return EvaluationCacheManager.CacheStatus.valid
else:
metrics.counter_inc(
name="anchore_policy_evaluation_cache_misses_invalid")
return EvaluationCacheManager.CacheStatus.invalid
def check_user_image_inline(user_id, image_id, tag, bundle):
"""
Execute a policy evaluation using the info in the request body including the bundle content
:param user_id:
:param image_id:
:param tag:
:param bundle:
:return:
"""
timer = time.time()
db = get_session()
cache_mgr = None
try:
# Input validation
if tag is None:
# set tag value to a value that only matches wildcards
tag = "*/*:*"
try:
img_obj = db.query(Image).get((image_id, user_id))
except:
return make_response_error("Image not found", in_httpcode=404), 404
if not img_obj:
log.info(
"Request for evaluation of image that cannot be found: user_id = {}, image_id = {}"
.format(user_id, image_id))
return make_response_error("Image not found", in_httpcode=404), 404
if evaluation_cache_enabled:
timer2 = time.time()
try:
try:
conn_timeout = (
ApiRequestContextProxy.get_service().configuration.get(
"catalog_client_conn_timeout",
DEFAULT_CACHE_CONN_TIMEOUT))
read_timeout = (
ApiRequestContextProxy.get_service().configuration.get(
"catalog_client_read_timeout",
DEFAULT_CACHE_READ_TIMEOUT))
cache_mgr = EvaluationCacheManager(img_obj, tag, bundle,
conn_timeout,
read_timeout)
except ValueError as err:
log.warn(
"Could not leverage cache due to error in bundle data: {}"
.format(err))
cache_mgr = None
if cache_mgr is None:
log.info(
"Could not initialize cache manager for policy evaluation, skipping cache usage"
)
else:
cached_result = cache_mgr.refresh()
if cached_result:
metrics.counter_inc(
name="anchore_policy_evaluation_cache_hits")
metrics.histogram_observe(
"anchore_policy_evaluation_cache_access_latency",
time.time() - timer2,
status="hit",
)
log.info(
"Returning cached result of policy evaluation for {}/{}, with tag {} and bundle {} with digest {}. Last evaluation: {}"
.format(
user_id,
image_id,
tag,
cache_mgr.bundle_id,
cache_mgr.bundle_digest,
cached_result.get("last_modified"),
))
return cached_result
else:
metrics.counter_inc(
name="anchore_policy_evaluation_cache_misses")
metrics.histogram_observe(
"anchore_policy_evaluation_cache_access_latency",
time.time() - timer2,
status="miss",
)
log.info(
"Policy evaluation not cached, or invalid, executing evaluation for {}/{} with tag {} and bundle {} with digest {}"
.format(
user_id,
image_id,
tag,
cache_mgr.bundle_id,
cache_mgr.bundle_digest,
))
except Exception as ex:
log.exception(
"Unexpected error operating on policy evaluation cache. Skipping use of cache."
)
else:
log.info("Policy evaluation cache disabled. Executing evaluation")
# Build bundle exec.
problems = []
executable_bundle = None
try:
# Allow deprecated gates here to support upgrade cases from old policy bundles.
executable_bundle = build_bundle(bundle,
for_tag=tag,
allow_deprecated=True)
if executable_bundle.init_errors:
problems = executable_bundle.init_errors
except InitializationError as e:
log.exception(
"Bundle construction and initialization returned errors")
problems = e.causes
eval_result = None
if not problems:
# Execute bundle
try:
eval_result = executable_bundle.execute(
img_obj, tag,
ExecutionContext(db_session=db, configuration={}))
except Exception as e:
log.exception(
"Error executing policy bundle {} against image {} w/tag {}: {}"
.format(bundle["id"], image_id, tag, e))
return (
make_response_error(
"Internal bundle evaluation error",
details={
"message":
"Cannot execute given policy against the image due to errors executing the policy bundle: {}"
.format(e)
},
in_httpcode=500,
),
500,
)
else:
# Construct a failure eval with details on the errors and mappings to send to client
eval_result = build_empty_error_execution(img_obj,
tag,
executable_bundle,
errors=problems,
warnings=[])
if (executable_bundle and executable_bundle.mapping
and len(executable_bundle.mapping.mapping_rules) == 1):
eval_result.executed_mapping = executable_bundle.mapping.mapping_rules[
0]
resp = PolicyEvaluation()
resp.user_id = user_id
resp.image_id = image_id
resp.tag = tag
resp.bundle = bundle
resp.matched_mapping_rule = (eval_result.executed_mapping.json() if
eval_result.executed_mapping else False)
resp.last_modified = int(time.time())
resp.final_action = eval_result.bundle_decision.final_decision.name
resp.final_action_reason = eval_result.bundle_decision.reason
resp.matched_whitelisted_images_rule = (
eval_result.bundle_decision.whitelisted_image.json()
if eval_result.bundle_decision.whitelisted_image else False)
resp.matched_blacklisted_images_rule = (
eval_result.bundle_decision.blacklisted_image.json()
if eval_result.bundle_decision.blacklisted_image else False)
resp.result = eval_result.as_table_json()
resp.created_at = int(time.time())
resp.evaluation_problems = [
problem_from_exception(i) for i in eval_result.errors
]
resp.evaluation_problems += [
problem_from_exception(i) for i in eval_result.warnings
]
if resp.evaluation_problems:
for i in resp.evaluation_problems:
log.warn(
"Returning evaluation response for image {}/{} w/tag {} and bundle {} that contains error: {}"
.format(user_id, image_id, tag, bundle["id"],
json.dumps(i.to_json())))
metrics.histogram_observe(
"anchore_policy_evaluation_time_seconds",
time.time() - timer,
status="fail",
)
else:
metrics.histogram_observe(
"anchore_policy_evaluation_time_seconds",
time.time() - timer,
status="success",
)
result = resp.to_json()
# Never let the cache block returning results
try:
if evaluation_cache_enabled and cache_mgr is not None:
cache_mgr.save(result)
except Exception as ex:
log.exception(
"Failed saving policy result in cache. Skipping and continuing."
)
db.commit()
return result
except HTTPException as e:
db.rollback()
log.exception("Caught exception in execution: {}".format(e))
raise
except Exception as e:
db.rollback()
log.exception("Failed processing bundle evaluation: {}".format(e))
return (
make_response_error(
"Unexpected internal error",
details={"message": str(e)},
in_httpcode=500,
),
500,
)
finally:
db.close()
def import_image(operation_id, account,
import_manifest: InternalImportManifest):
"""
The main thread of exec for importing an image
:param operation_id:
:param account:
:param import_manifest:
:return:
"""
timer = int(time.time())
analysis_events = []
config = localconfig.get_config()
all_content_types = config.get("image_content_types", []) + config.get(
"image_metadata_types", [])
image_digest = import_manifest.digest
try:
catalog_client = internal_client_for(CatalogClient, account)
# check to make sure image is still in DB
catalog_client = internal_client_for(CatalogClient, account)
try:
image_record = catalog_client.get_image(image_digest)
if not image_record:
raise Exception("empty image record from catalog")
except Exception as err:
logger.debug_exception("Could not get image record")
logger.warn(
"dequeued image cannot be fetched from catalog - skipping analysis ("
+ str(image_digest) + ") - exception: " + str(err))
return True
if image_record["analysis_status"] != taskstate.base_state("analyze"):
logger.info(
"dequeued image to import is not in base 'not_analyzed' state - skipping import"
)
return True
try:
last_analysis_status = image_record["analysis_status"]
image_record = update_analysis_started(catalog_client,
image_digest, image_record)
logger.info("Loading content from import")
sbom_map = get_content(import_manifest, catalog_client)
manifest = sbom_map.get("manifest")
try:
logger.info("processing image import data")
image_data, analysis_manifest = process_import(
image_record, sbom_map, import_manifest)
except AnchoreException as e:
event = events.ImageAnalysisFailed(user_id=account,
image_digest=image_digest,
error=e.to_dict())
analysis_events.append(event)
raise
# Store the manifest in the object store
logger.info("storing image manifest")
catalog_client.put_document(bucket="manifest_data",
name=image_digest,
inobj=json.dumps(manifest))
# Save the results to the upstream components and data stores
logger.info("storing import result")
store_analysis_results(
account,
image_digest,
image_record,
image_data,
manifest,
analysis_events,
all_content_types,
)
logger.info("updating image catalog record analysis_status")
last_analysis_status = image_record["analysis_status"]
image_record = update_analysis_complete(catalog_client,
image_digest, image_record)
try:
analysis_events.extend(
notify_analysis_complete(image_record,
last_analysis_status))
except Exception as err:
logger.warn(
"failed to enqueue notification on image analysis state update - exception: "
+ str(err))
logger.info("analysis complete: " + str(account) + " : " +
str(image_digest))
try:
catalog_client.update_image_import_status(operation_id,
status="complete")
except Exception as err:
logger.debug_exception(
"failed updating import status success, will continue and rely on expiration for GC later"
)
try:
metrics.counter_inc(name="anchore_import_success")
run_time = float(time.time() - timer)
metrics.histogram_observe(
"anchore_import_time_seconds",
run_time,
buckets=IMPORT_TIME_SECONDS_BUCKETS,
status="success",
)
except Exception as err:
logger.warn(str(err))
except Exception as err:
run_time = float(time.time() - timer)
logger.exception("problem importing image - exception: " +
str(err))
analysis_failed_metrics(run_time)
# Transition the image record to failure status
image_record = update_analysis_failed(catalog_client, image_digest,
image_record)
try:
catalog_client.update_image_import_status(operation_id,
status="failed")
except Exception as err:
logger.debug_exception(
"failed updating import status failure, will continue and rely on expiration for GC later"
)
if account and image_digest:
for image_detail in image_record["image_detail"]:
fulltag = (image_detail["registry"] + "/" +
image_detail["repo"] + ":" +
image_detail["tag"])
event = events.UserAnalyzeImageFailed(user_id=account,
full_tag=fulltag,
error=str(err))
analysis_events.append(event)
finally:
if analysis_events:
emit_events(catalog_client, analysis_events)
except Exception as err:
logger.debug_exception("Could not import image")
logger.warn("job processing bailed - exception: " + str(err))
raise err
return True