def delete_feed_group(feed, group):
httpcode = 500
try:
p_client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace())
return_object = p_client.delete_feed_group(feed, group)
if return_object is not None:
httpcode = 200
except Exception as err:
return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
return return_object, httpcode
def import_to_policy_engine(account: str, image_id: str, image_digest: str):
"""
Import the given image into the policy engine
:param account:
:param image_id:
:param image_digest:
:return:
"""
if image_id is None:
raise ValueError("image_id must not be None")
pe_client = internal_client_for(PolicyEngineClient, account)
try:
logger.debug(
"clearing any existing image record in policy engine: {} / {} / {}".format(
account, image_id, image_digest
)
)
rc = pe_client.delete_image(user_id=account, image_id=image_id)
except Exception as err:
logger.warn("exception on pre-delete - exception: " + str(err))
client_success = False
last_exception = None
# TODO: rework this wait logic using 'retrying.retry()' decorator on this whole function to allow new client on each call
for retry_wait in [1, 3, 5, 0]:
try:
logger.info(
"loading image into policy engine: account={} image_id={} image_digest={}".format(
account, image_id, image_digest
)
)
image_analysis_fetch_url = build_catalog_url(account, image_digest)
logger.debug(
"policy engine request catalog content url: " + image_analysis_fetch_url
)
resp = pe_client.ingress_image(account, image_id, image_analysis_fetch_url)
logger.debug("policy engine image add response: " + str(resp))
client_success = True
break
# TODO: add a vuln eval and policy eval (with active policy), here to prime any caches since this isn't a highly latency sensitive code section
except Exception as e:
logger.warn("attempt failed, will retry - exception: {}".format(e))
last_exception = e
time.sleep(retry_wait)
if not client_success:
raise last_exception
return True
def list_analysis_archive():
"""
GET /archives/images
:return: array of archivedimage json objects
"""
client = internal_client_for(CatalogClient,
ApiRequestContextProxy.namespace())
try:
return handle_proxy_response(client.list_archived_analyses())
except Exception as ex:
return handle_proxy_response(ex)
def list_archives():
"""
GET /archives
:return: JSON object for archive summary
"""
client = internal_client_for(CatalogClient,
ApiRequestContextProxy.namespace())
try:
return handle_proxy_response(client.list_archives())
except Exception as ex:
return handle_proxy_response(ex)
def archive_image_analysis(imageReferences):
"""
POST /archives/images
:param imageReferences: list of json object that reference images to archive
:return:
"""
client = internal_client_for(CatalogClient,
ApiRequestContextProxy.namespace())
try:
return handle_proxy_response(client.archive_analyses(imageReferences))
except Exception as ex:
return handle_proxy_response(ex)
def update_registry(registry, registrydata, validate=True):
"""
PUT /registries/<id>
:param registry:
:return:
"""
request_inputs = anchore_engine.apis.do_request_prep(
request, default_params={'validate': validate})
user_auth = request_inputs['auth']
method = request_inputs['method']
bodycontent = request_inputs['bodycontent']
params = request_inputs['params']
return_object = []
httpcode = 500
try:
registrydata = json.loads(bodycontent)
try:
input_registry = registrydata.get('registry', None)
if input_registry:
if input_registry != registry:
raise Exception(
"registry name in path does not equal registry name in body"
)
# do some input string checking
if re.match(".*\/.*", input_registry):
raise Exception(
"input registry name cannot contain '/' characters - valid registry names are of the form <host>:<port> where :<port> is optional"
)
except Exception as err:
httpcode = 409
raise err
client = internal_client_for(CatalogClient, request_inputs['userId'])
registry_records = client.update_registry(registry,
registrydata,
validate=validate)
for registry_record in registry_records:
return_object.append(
make_response_registry(user_auth, registry_record, params))
httpcode = 200
except Exception as err:
return_object = anchore_engine.common.helpers.make_response_error(
err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
return (return_object, httpcode)
def create_registry(registrydata, validate=True):
"""
POST /registries
:param registry:
:return:
"""
request_inputs = anchore_engine.apis.do_request_prep(
request, default_params={"validate": validate})
user_auth = request_inputs["auth"]
method = request_inputs["method"]
bodycontent = request_inputs["bodycontent"]
params = request_inputs["params"]
return_object = []
httpcode = 500
try:
registrydata = json.loads(bodycontent)
try:
input_registry = registrydata.get("registry", None)
if input_registry:
# do some input string checking
errmsg = None
if re.match(".*/+$", input_registry):
errmsg = (
"input registry name cannot end with trailing '/' characters"
)
elif re.match("^http[s]*://", input_registry):
errmsg = "input registry name must start with a hostname/ip, without URI schema (http://, https://)"
if errmsg:
raise Exception(errmsg)
except Exception as err:
httpcode = 409
raise err
client = internal_client_for(CatalogClient, request_inputs["userId"])
registry_records = client.add_registry(registrydata, validate=validate)
for registry_record in registry_records:
return_object.append(
make_response_registry(user_auth, registry_record, params))
httpcode = 200
except Exception as err:
return_object = anchore_engine.common.helpers.make_response_error(
err, in_httpcode=httpcode)
httpcode = return_object["httpcode"]
return return_object, httpcode
def _get_content(self, url):
"""
This can be *big*, as in hundreds of MB of data.
Supported url formats:
file://
http(s)://
catalog://<userId>/<bucket>/<name>
:param url:
:return:
"""
split_url = urllib.parse.splittype(url)
if split_url[0] == 'file':
path = split_url[1][2:] # Strip the leading '//'
return self._get_file(path)
elif split_url[0] == 'catalog':
userId, bucket, name = split_url[1][2:].split('/')
# Add auth if necessary
try:
catalog_client = internal_client_for(CatalogClient, userId)
with catalog_client.timeout_context(
self.content_conn_timeout,
self.content_read_timeout) as timeout_client:
doc = timeout_client.get_document(bucket, name)
return doc
except:
log.exception(
'Error retrieving analysis json from the catalog service')
raise
elif split_url[0].startswith('http'):
retry = 3
while retry > 0:
try:
data_response = requests.get(url=url)
content = data_response.json()
return content
except requests.HTTPError as ex:
log.exception('HTTP exception: {}. Retrying'.format(ex))
retry = retry - 1
time.sleep(retry * 3) # Backoff and retry
except:
log.exception('Non HTTP exception. Retrying')
retry = retry - 1
else:
raise Exception('Cannot get content from url: {}'.format(url))
def list_analysis_archive_rules(system_global=True):
"""
GET /archives/rules
:return:
"""
client = internal_client_for(CatalogClient,
ApiRequestContextProxy.namespace())
try:
return handle_proxy_response(
client.list_analysis_archive_rules(system_global=system_global))
except Exception as ex:
return handle_proxy_response(ex)
def delete_archived_analysis(imageDigest):
"""
DELETE /archives/images/{imageDigest}
:param imageDigest:
:return:
"""
client = internal_client_for(CatalogClient,
ApiRequestContextProxy.namespace())
try:
return handle_proxy_response(
client.delete_archived_analysis(imageDigest))
except Exception as e:
return handle_proxy_response(e)
def describe_policy():
request_inputs = anchore_engine.apis.do_request_prep(request, default_params={})
return_object = []
httpcode = 500
try:
p_client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace())
return_object = p_client.describe_policy()
if return_object:
httpcode = 200
except Exception as err:
return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
return return_object, httpcode
def get_service_detail():
"""
GET /system
:return: list of service details
"""
request_inputs = anchore_engine.apis.do_request_prep(request, default_params={})
user_auth = request_inputs['auth']
method = request_inputs['method']
params = request_inputs['params']
httpcode = 500
service_detail = {}
try:
try:
try:
service_detail['service_states'] = []
try:
up_services = {}
client = internal_client_for(CatalogClient, request_inputs['userId'])
service_records = client.get_service()
for service in service_records:
el = make_response_service(user_auth, service, params)
service_detail['service_states'].append(el)
if el['servicename'] not in up_services:
up_services[el['servicename']] = 0
if el['status']:
up_services[el['servicename']] += 1
except Exception as err:
pass
httpcode = 200
except Exception as err:
return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
except:
service_detail = {}
return_object = service_detail
except Exception as err:
return_object = str(err)
return return_object, httpcode
def query_vulnerabilities_get(id=None, affected_package=None, affected_package_version=None):
try:
request_inputs = anchore_engine.apis.do_request_prep(connexion.request, default_params={'id': id, 'affected_package': affected_package, 'affected_package_version': affected_package_version})
client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace())
resp = client.query_vulnerabilities(vuln_id=request_inputs.get('params',{}).get('id'),
affected_package=request_inputs.get('params',{}).get('affected_package'),
affected_package_version=request_inputs.get('params',{}).get('affected_package_version'))
code = 200
except Exception as err:
logger.exception('Error dispatching/receiving request from policy engine for vulnerability query')
resp = str(err)
code = 500
return resp, code
def images_imageDigest(request_inputs, imageDigest):
user_auth = request_inputs['auth']
method = request_inputs['method']
bodycontent = request_inputs['bodycontent']
params = request_inputs['params']
return_object = {}
httpcode = 500
username, pw = user_auth
userId = request_inputs['userId']
try:
client = internal_client_for(CatalogClient, request_inputs['userId'])
if method == 'GET':
logger.debug("handling GET on imageDigest: " + str(imageDigest))
image_records = client.get_image(imageDigest=imageDigest)
if image_records:
return_object = []
for image_record in image_records:
return_object.append(make_response_image(user_auth, image_record, params))
httpcode = 200
else:
httpcode = 404
raise Exception("cannot locate specified image")
elif method == 'DELETE':
logger.debug("handling DELETE on imageDigest: " + str(imageDigest))
rc = False
try:
rc = client.delete_image(imageDigest, force=params['force'])
except Exception as err:
raise err
if rc:
return_object = rc
httpcode = 200
else:
httpcode = 500
raise Exception("failed to delete")
except Exception as err:
logger.debug("operation exception: " + str(err))
return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
return (return_object, httpcode)
def push_sync_task(system_user):
all_ready = anchore_engine.clients.services.common.check_services_ready(['simplequeue'])
if not all_ready:
logger.info("simplequeue service not yet ready, will retry")
raise Exception("Simplequeue service not yet ready")
else:
#q_client = SimpleQueueClient(user=system_user[0], password=system_user[1])
q_client = internal_client_for(SimpleQueueClient, userId=None)
if not q_client.is_inqueue(name=feed_sync_queuename, inobj=feed_sync_msg):
try:
q_client.enqueue(name=feed_sync_queuename, inobj=feed_sync_msg)
except:
logger.error('Could not enqueue message for a feed sync')
raise
def get_system_feeds():
request_inputs = anchore_engine.apis.do_request_prep(request, default_params={})
return_object = []
httpcode = 500
try:
p_client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace())
# do the p.e. feed get call
return_object = p_client.list_feeds(include_counts=True)
if return_object is not None:
httpcode = 200
except Exception as err:
return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
return return_object, httpcode
def query_images_by_vulnerability_get(vulnerability_id=None, severity=None, namespace=None, affected_package=None, vendor_only=True):
try:
request_inputs = anchore_engine.apis.do_request_prep(connexion.request, default_params={'vulnerability_id': vulnerability_id, 'severity': severity, 'namespace': namespace, 'affected_package': affected_package, 'vendor_only': vendor_only})
client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace())
return_object = client.query_images_by_vulnerability(user_id=ApiRequestContextProxy.namespace(),
vulnerability_id=request_inputs.get('params',{}).get('vulnerability_id'),
severity=request_inputs.get('params',{}).get('severity'),
namespace=request_inputs.get('params',{}).get('namespace'),
affected_package=request_inputs.get('params',{}).get('affected_package'),
vendor_only=request_inputs.get('params',{}).get('vendor_only'))
httpcode = 200
except Exception as err:
httpcode = 500
return_object = str(err)
return (return_object, httpcode)
def post_system_feeds(flush=False):
request_inputs = anchore_engine.apis.do_request_prep(request, default_params={'flush': flush})
return_object = []
httpcode = 500
try:
p_client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace())
# do the p.e. feed post call
return_object = p_client.sync_feeds(force_flush=flush)
if return_object is not None:
httpcode = 200
except Exception as err:
return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
return return_object, httpcode
def post_system_prune_candidates(resourcetype, bodycontent):
request_inputs = anchore_engine.apis.do_request_prep(request,
default_params={})
return_object = []
httpcode = 500
try:
client = internal_client_for(CatalogClient, request_inputs['userId'])
return_object = client.perform_prune(resourcetype, bodycontent)
if return_object:
httpcode = 200
except Exception as err:
return_object = anchore_engine.common.helpers.make_response_error(
err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
return (return_object, httpcode)
def delete_policy(policyId):
request_inputs = anchore_engine.apis.do_request_prep(request,
default_params={})
user_auth = request_inputs['auth']
return_object = {}
httpcode = 500
userId, pw = user_auth
try:
logger.debug("Delete policy {}".format(policyId))
client = internal_client_for(CatalogClient, request_inputs['userId'])
try:
try:
policy_record = client.get_policy(policyId=policyId)
except Exception as err:
logger.warn("unable to get policy_records for user (" +
str(userId) + ") - exception: " + str(err))
raise err
if not policy_record:
rc = True
else:
if policy_record['active']:
httpcode = 500
raise Exception(
"cannot delete an active policy - activate a different policy then delete this one"
)
rc = client.delete_policy(policyId=policyId)
except Exception as err:
raise err
if rc:
httpcode = 200
return_object = "deleted"
else:
httpcode = 500
raise Exception('not deleted')
except Exception as err:
return_object = anchore_engine.common.helpers.make_response_error(
err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
return (return_object, httpcode)
def query_images_by_package_get(name=None, version=None, package_type=None):
try:
request_inputs = anchore_engine.apis.do_request_prep(connexion.request, default_params={'name': name, 'version': version, 'package_type': package_type})
client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace())
logger.info('Params for image by_package: {}'.format(request_inputs))
return_object = client.query_images_by_package(user_id=ApiRequestContextProxy.namespace(),
name=request_inputs.get('params',{}).get('name'),
version=request_inputs.get('params',{}).get('version'),
package_type=request_inputs.get('params',{}).get('package_type'))
httpcode = 200
except Exception as err:
logger.exception('Error dispatching/receiving request from policy engine for image query by package')
httpcode = 500
return_object = str(err)
return (return_object, httpcode)
def delete_events(before=None, since=None, level=None):
request_inputs = anchore_engine.apis.do_request_prep(request, default_params={})
user_auth = request_inputs['auth']
method = request_inputs['method']
bodycontent = request_inputs['bodycontent']
params = request_inputs['params']
return_object = {}
httpcode = 500
try:
client = internal_client_for(CatalogClient, request_inputs['userId'])
return_object = client.delete_events(since=since, before=before, level=level)
httpcode = 200
except Exception as err:
return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
return return_object, httpcode
def _init_policy(accountname, config):
"""
Initialize a new bundle for the given accountname
:return: bool indicating if bundle was created or not (False means one already existed)
"""
client = internal_client_for(CatalogClient, accountname)
policies = client.list_policies()
if len(policies) == 0:
logger.debug(
"Account {} has no policy bundle - installing default".format(
accountname))
if config.get('default_bundle_file', None) and os.path.exists(
config['default_bundle_file']):
logger.info("loading def bundle: " +
str(config['default_bundle_file']))
try:
default_bundle = {}
with open(config['default_bundle_file'], 'r') as FH:
default_bundle = json.loads(FH.read())
if default_bundle:
resp = client.add_policy(default_bundle, active=True)
if not resp:
raise Exception("policy bundle DB add failed")
return True
else:
raise Exception('No default bundle found')
except Exception as err:
logger.error(
"could not load up default bundle for user - exception: " +
str(err))
raise
else:
logger.debug(
'Existing bundle found for account: {}. Not expected on invocations of this function in most uses'
.format(accountname))
return False
def get_event(eventId):
request_inputs = anchore_engine.apis.do_request_prep(request, default_params={})
user_auth = request_inputs['auth']
method = request_inputs['method']
bodycontent = request_inputs['bodycontent']
params = request_inputs['params']
return_object = {}
httpcode = 500
userId, pw = user_auth
try:
client = internal_client_for(CatalogClient, request_inputs['userId'])
return_object = client.get_event(eventId)
httpcode = 200
except Exception as err:
return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
return return_object, httpcode
def create_operation():
"""
POST /imports/images
:return:
"""
try:
client = internal_client_for(
CatalogClient, userId=ApiRequestContextProxy.namespace()
)
resp = client.create_image_import()
return resp, 200
except api_exceptions.AnchoreApiError as ex:
return (
make_response_error(ex, in_httpcode=ex.__response_code__),
ex.__response_code__,
)
except Exception as ex:
logger.exception("Unexpected error in api processing")
return make_response_error(ex, in_httpcode=500), 500
def list_events(source_servicename=None, source_hostid=None, resource_type=None, resource_id=None, level=None, since=None, before=None, page=None, limit=None):
request_inputs = anchore_engine.apis.do_request_prep(request, default_params={})
user_auth = request_inputs['auth']
method = request_inputs['method']
bodycontent = request_inputs['bodycontent']
params = request_inputs['params']
return_object = {}
httpcode = 500
try:
client = internal_client_for(CatalogClient, request_inputs['userId'])
return_object = client.get_events(source_servicename=source_servicename, source_hostid=source_hostid,
resource_type=resource_type, resource_id=resource_id, level=level, since=since,
before=before, page=page, limit=limit)
httpcode = 200
except Exception as err:
return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
return return_object, httpcode
def authz_heartbeat(*args, **kwargs):
cycle_timer = kwargs["mythread"]["cycle_timer"]
logger.info("Checking authz availability")
try:
host_id = localconfig.get_host_id()
authz_handlr = get_authorizer()
handler = authz_handlr.__class__.__name__
ex = None
try:
result = authz_handlr.healthcheck()
except Exception as e:
ex = e
result = False
if not result:
fail_event = ServiceAuthzPluginHealthCheckFailed(
user_id=localconfig.ADMIN_ACCOUNT_NAME,
name=service_name,
host=host_id,
plugin=handler,
details=str(ex),
)
logger.info("Sending healthcheck failure event: {}".format(
fail_event.__event_type__))
try:
client = internal_client_for(
CatalogClient, localconfig.ADMIN_ACCOUNT_NAME)
client.add_event(fail_event)
except Exception as ex:
logger.exception(
"Failure to send authz healthcheck failure event: {}"
.format(fail_event.to_json()))
except Exception as e:
logger.exception(
"Caught unexpected exception from the authz heartbeat handler"
)
time.sleep(cycle_timer)
return True
def list_import_image_configs(operation_id):
"""
GET /imports/images/{operation_id}/image_config
:param operation_id:
:return:
"""
try:
client = internal_client_for(
CatalogClient, userId=ApiRequestContextProxy.namespace()
)
resp = client.list_import_content(operation_id, "image_config")
return resp, 200
except api_exceptions.AnchoreApiError as ex:
return (
make_response_error(ex, in_httpcode=ex.__response_code__),
ex.__response_code__,
)
except Exception as ex:
logger.exception("Unexpected error in api processing")
return make_response_error(ex, in_httpcode=500), 500
def repositories(request_inputs):
method = request_inputs['method']
bodycontent = request_inputs['bodycontent']
params = request_inputs['params']
return_object = {}
httpcode = 500
input_repo = None
if params and 'repository' in params:
input_repo = params['repository']
autosubscribe = False
if params and 'autosubscribe' in params:
autosubscribe = params['autosubscribe']
lookuptag = None
if params and 'lookuptag' in params:
lookuptag = params['lookuptag']
try:
if method == 'POST':
logger.debug("handling POST: ")
try:
client = internal_client_for(CatalogClient, request_inputs['userId'])
return_object = []
repo_records = client.add_repo(regrepo=input_repo, autosubscribe=autosubscribe, lookuptag=lookuptag)
for repo_record in repo_records:
return_object.append(repo_record)
httpcode = 200
except Exception as err:
raise err
except Exception as err:
logger.debug("operation exception: " + str(err))
return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode)
httpcode = return_object['httpcode']
return(return_object, httpcode)
def queue_notification(userId, subscription_key, subscription_type, payload):
"""
Put a Notification in the Queue!
"""
q_client = internal_client_for(SimpleQueueClient, None)
rc = False
try:
nobj = {
"userId": userId,
"subscription_key": subscription_key,
"notificationId": str(uuid.uuid4()),
}
if payload:
nobj.update(payload)
if not q_client.is_inqueue(subscription_type, nobj):
rc = q_client.enqueue(subscription_type, nobj)
except Exception as err:
logger.warn("failed to create/enqueue notification")
raise err
return rc
