def delete_feed_group(feed, group): httpcode = 500 try: p_client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace()) return_object = p_client.delete_feed_group(feed, group) if return_object is not None: httpcode = 200 except Exception as err: return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return return_object, httpcode
def import_to_policy_engine(account: str, image_id: str, image_digest: str): """ Import the given image into the policy engine :param account: :param image_id: :param image_digest: :return: """ if image_id is None: raise ValueError("image_id must not be None") pe_client = internal_client_for(PolicyEngineClient, account) try: logger.debug( "clearing any existing image record in policy engine: {} / {} / {}".format( account, image_id, image_digest ) ) rc = pe_client.delete_image(user_id=account, image_id=image_id) except Exception as err: logger.warn("exception on pre-delete - exception: " + str(err)) client_success = False last_exception = None # TODO: rework this wait logic using 'retrying.retry()' decorator on this whole function to allow new client on each call for retry_wait in [1, 3, 5, 0]: try: logger.info( "loading image into policy engine: account={} image_id={} image_digest={}".format( account, image_id, image_digest ) ) image_analysis_fetch_url = build_catalog_url(account, image_digest) logger.debug( "policy engine request catalog content url: " + image_analysis_fetch_url ) resp = pe_client.ingress_image(account, image_id, image_analysis_fetch_url) logger.debug("policy engine image add response: " + str(resp)) client_success = True break # TODO: add a vuln eval and policy eval (with active policy), here to prime any caches since this isn't a highly latency sensitive code section except Exception as e: logger.warn("attempt failed, will retry - exception: {}".format(e)) last_exception = e time.sleep(retry_wait) if not client_success: raise last_exception return True
def list_analysis_archive(): """ GET /archives/images :return: array of archivedimage json objects """ client = internal_client_for(CatalogClient, ApiRequestContextProxy.namespace()) try: return handle_proxy_response(client.list_archived_analyses()) except Exception as ex: return handle_proxy_response(ex)
def list_archives(): """ GET /archives :return: JSON object for archive summary """ client = internal_client_for(CatalogClient, ApiRequestContextProxy.namespace()) try: return handle_proxy_response(client.list_archives()) except Exception as ex: return handle_proxy_response(ex)
def archive_image_analysis(imageReferences): """ POST /archives/images :param imageReferences: list of json object that reference images to archive :return: """ client = internal_client_for(CatalogClient, ApiRequestContextProxy.namespace()) try: return handle_proxy_response(client.archive_analyses(imageReferences)) except Exception as ex: return handle_proxy_response(ex)
def update_registry(registry, registrydata, validate=True): """ PUT /registries/<id> :param registry: :return: """ request_inputs = anchore_engine.apis.do_request_prep( request, default_params={'validate': validate}) user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = [] httpcode = 500 try: registrydata = json.loads(bodycontent) try: input_registry = registrydata.get('registry', None) if input_registry: if input_registry != registry: raise Exception( "registry name in path does not equal registry name in body" ) # do some input string checking if re.match(".*\/.*", input_registry): raise Exception( "input registry name cannot contain '/' characters - valid registry names are of the form <host>:<port> where :<port> is optional" ) except Exception as err: httpcode = 409 raise err client = internal_client_for(CatalogClient, request_inputs['userId']) registry_records = client.update_registry(registry, registrydata, validate=validate) for registry_record in registry_records: return_object.append( make_response_registry(user_auth, registry_record, params)) httpcode = 200 except Exception as err: return_object = anchore_engine.common.helpers.make_response_error( err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)
def create_registry(registrydata, validate=True): """ POST /registries :param registry: :return: """ request_inputs = anchore_engine.apis.do_request_prep( request, default_params={"validate": validate}) user_auth = request_inputs["auth"] method = request_inputs["method"] bodycontent = request_inputs["bodycontent"] params = request_inputs["params"] return_object = [] httpcode = 500 try: registrydata = json.loads(bodycontent) try: input_registry = registrydata.get("registry", None) if input_registry: # do some input string checking errmsg = None if re.match(".*/+$", input_registry): errmsg = ( "input registry name cannot end with trailing '/' characters" ) elif re.match("^http[s]*://", input_registry): errmsg = "input registry name must start with a hostname/ip, without URI schema (http://, https://)" if errmsg: raise Exception(errmsg) except Exception as err: httpcode = 409 raise err client = internal_client_for(CatalogClient, request_inputs["userId"]) registry_records = client.add_registry(registrydata, validate=validate) for registry_record in registry_records: return_object.append( make_response_registry(user_auth, registry_record, params)) httpcode = 200 except Exception as err: return_object = anchore_engine.common.helpers.make_response_error( err, in_httpcode=httpcode) httpcode = return_object["httpcode"] return return_object, httpcode
def _get_content(self, url): """ This can be *big*, as in hundreds of MB of data. Supported url formats: file:// http(s):// catalog://<userId>/<bucket>/<name> :param url: :return: """ split_url = urllib.parse.splittype(url) if split_url[0] == 'file': path = split_url[1][2:] # Strip the leading '//' return self._get_file(path) elif split_url[0] == 'catalog': userId, bucket, name = split_url[1][2:].split('/') # Add auth if necessary try: catalog_client = internal_client_for(CatalogClient, userId) with catalog_client.timeout_context( self.content_conn_timeout, self.content_read_timeout) as timeout_client: doc = timeout_client.get_document(bucket, name) return doc except: log.exception( 'Error retrieving analysis json from the catalog service') raise elif split_url[0].startswith('http'): retry = 3 while retry > 0: try: data_response = requests.get(url=url) content = data_response.json() return content except requests.HTTPError as ex: log.exception('HTTP exception: {}. Retrying'.format(ex)) retry = retry - 1 time.sleep(retry * 3) # Backoff and retry except: log.exception('Non HTTP exception. Retrying') retry = retry - 1 else: raise Exception('Cannot get content from url: {}'.format(url))
def list_analysis_archive_rules(system_global=True): """ GET /archives/rules :return: """ client = internal_client_for(CatalogClient, ApiRequestContextProxy.namespace()) try: return handle_proxy_response( client.list_analysis_archive_rules(system_global=system_global)) except Exception as ex: return handle_proxy_response(ex)
def delete_archived_analysis(imageDigest): """ DELETE /archives/images/{imageDigest} :param imageDigest: :return: """ client = internal_client_for(CatalogClient, ApiRequestContextProxy.namespace()) try: return handle_proxy_response( client.delete_archived_analysis(imageDigest)) except Exception as e: return handle_proxy_response(e)
def describe_policy(): request_inputs = anchore_engine.apis.do_request_prep(request, default_params={}) return_object = [] httpcode = 500 try: p_client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace()) return_object = p_client.describe_policy() if return_object: httpcode = 200 except Exception as err: return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return return_object, httpcode
def get_service_detail(): """ GET /system :return: list of service details """ request_inputs = anchore_engine.apis.do_request_prep(request, default_params={}) user_auth = request_inputs['auth'] method = request_inputs['method'] params = request_inputs['params'] httpcode = 500 service_detail = {} try: try: try: service_detail['service_states'] = [] try: up_services = {} client = internal_client_for(CatalogClient, request_inputs['userId']) service_records = client.get_service() for service in service_records: el = make_response_service(user_auth, service, params) service_detail['service_states'].append(el) if el['servicename'] not in up_services: up_services[el['servicename']] = 0 if el['status']: up_services[el['servicename']] += 1 except Exception as err: pass httpcode = 200 except Exception as err: return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] except: service_detail = {} return_object = service_detail except Exception as err: return_object = str(err) return return_object, httpcode
def query_vulnerabilities_get(id=None, affected_package=None, affected_package_version=None): try: request_inputs = anchore_engine.apis.do_request_prep(connexion.request, default_params={'id': id, 'affected_package': affected_package, 'affected_package_version': affected_package_version}) client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace()) resp = client.query_vulnerabilities(vuln_id=request_inputs.get('params',{}).get('id'), affected_package=request_inputs.get('params',{}).get('affected_package'), affected_package_version=request_inputs.get('params',{}).get('affected_package_version')) code = 200 except Exception as err: logger.exception('Error dispatching/receiving request from policy engine for vulnerability query') resp = str(err) code = 500 return resp, code
def images_imageDigest(request_inputs, imageDigest): user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 username, pw = user_auth userId = request_inputs['userId'] try: client = internal_client_for(CatalogClient, request_inputs['userId']) if method == 'GET': logger.debug("handling GET on imageDigest: " + str(imageDigest)) image_records = client.get_image(imageDigest=imageDigest) if image_records: return_object = [] for image_record in image_records: return_object.append(make_response_image(user_auth, image_record, params)) httpcode = 200 else: httpcode = 404 raise Exception("cannot locate specified image") elif method == 'DELETE': logger.debug("handling DELETE on imageDigest: " + str(imageDigest)) rc = False try: rc = client.delete_image(imageDigest, force=params['force']) except Exception as err: raise err if rc: return_object = rc httpcode = 200 else: httpcode = 500 raise Exception("failed to delete") except Exception as err: logger.debug("operation exception: " + str(err)) return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)
def push_sync_task(system_user): all_ready = anchore_engine.clients.services.common.check_services_ready(['simplequeue']) if not all_ready: logger.info("simplequeue service not yet ready, will retry") raise Exception("Simplequeue service not yet ready") else: #q_client = SimpleQueueClient(user=system_user[0], password=system_user[1]) q_client = internal_client_for(SimpleQueueClient, userId=None) if not q_client.is_inqueue(name=feed_sync_queuename, inobj=feed_sync_msg): try: q_client.enqueue(name=feed_sync_queuename, inobj=feed_sync_msg) except: logger.error('Could not enqueue message for a feed sync') raise
def get_system_feeds(): request_inputs = anchore_engine.apis.do_request_prep(request, default_params={}) return_object = [] httpcode = 500 try: p_client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace()) # do the p.e. feed get call return_object = p_client.list_feeds(include_counts=True) if return_object is not None: httpcode = 200 except Exception as err: return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return return_object, httpcode
def query_images_by_vulnerability_get(vulnerability_id=None, severity=None, namespace=None, affected_package=None, vendor_only=True): try: request_inputs = anchore_engine.apis.do_request_prep(connexion.request, default_params={'vulnerability_id': vulnerability_id, 'severity': severity, 'namespace': namespace, 'affected_package': affected_package, 'vendor_only': vendor_only}) client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace()) return_object = client.query_images_by_vulnerability(user_id=ApiRequestContextProxy.namespace(), vulnerability_id=request_inputs.get('params',{}).get('vulnerability_id'), severity=request_inputs.get('params',{}).get('severity'), namespace=request_inputs.get('params',{}).get('namespace'), affected_package=request_inputs.get('params',{}).get('affected_package'), vendor_only=request_inputs.get('params',{}).get('vendor_only')) httpcode = 200 except Exception as err: httpcode = 500 return_object = str(err) return (return_object, httpcode)
def post_system_feeds(flush=False): request_inputs = anchore_engine.apis.do_request_prep(request, default_params={'flush': flush}) return_object = [] httpcode = 500 try: p_client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace()) # do the p.e. feed post call return_object = p_client.sync_feeds(force_flush=flush) if return_object is not None: httpcode = 200 except Exception as err: return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return return_object, httpcode
def post_system_prune_candidates(resourcetype, bodycontent): request_inputs = anchore_engine.apis.do_request_prep(request, default_params={}) return_object = [] httpcode = 500 try: client = internal_client_for(CatalogClient, request_inputs['userId']) return_object = client.perform_prune(resourcetype, bodycontent) if return_object: httpcode = 200 except Exception as err: return_object = anchore_engine.common.helpers.make_response_error( err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)
def delete_policy(policyId): request_inputs = anchore_engine.apis.do_request_prep(request, default_params={}) user_auth = request_inputs['auth'] return_object = {} httpcode = 500 userId, pw = user_auth try: logger.debug("Delete policy {}".format(policyId)) client = internal_client_for(CatalogClient, request_inputs['userId']) try: try: policy_record = client.get_policy(policyId=policyId) except Exception as err: logger.warn("unable to get policy_records for user (" + str(userId) + ") - exception: " + str(err)) raise err if not policy_record: rc = True else: if policy_record['active']: httpcode = 500 raise Exception( "cannot delete an active policy - activate a different policy then delete this one" ) rc = client.delete_policy(policyId=policyId) except Exception as err: raise err if rc: httpcode = 200 return_object = "deleted" else: httpcode = 500 raise Exception('not deleted') except Exception as err: return_object = anchore_engine.common.helpers.make_response_error( err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)
def query_images_by_package_get(name=None, version=None, package_type=None): try: request_inputs = anchore_engine.apis.do_request_prep(connexion.request, default_params={'name': name, 'version': version, 'package_type': package_type}) client = internal_client_for(PolicyEngineClient, userId=ApiRequestContextProxy.namespace()) logger.info('Params for image by_package: {}'.format(request_inputs)) return_object = client.query_images_by_package(user_id=ApiRequestContextProxy.namespace(), name=request_inputs.get('params',{}).get('name'), version=request_inputs.get('params',{}).get('version'), package_type=request_inputs.get('params',{}).get('package_type')) httpcode = 200 except Exception as err: logger.exception('Error dispatching/receiving request from policy engine for image query by package') httpcode = 500 return_object = str(err) return (return_object, httpcode)
def delete_events(before=None, since=None, level=None): request_inputs = anchore_engine.apis.do_request_prep(request, default_params={}) user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 try: client = internal_client_for(CatalogClient, request_inputs['userId']) return_object = client.delete_events(since=since, before=before, level=level) httpcode = 200 except Exception as err: return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return return_object, httpcode
def _init_policy(accountname, config): """ Initialize a new bundle for the given accountname :return: bool indicating if bundle was created or not (False means one already existed) """ client = internal_client_for(CatalogClient, accountname) policies = client.list_policies() if len(policies) == 0: logger.debug( "Account {} has no policy bundle - installing default".format( accountname)) if config.get('default_bundle_file', None) and os.path.exists( config['default_bundle_file']): logger.info("loading def bundle: " + str(config['default_bundle_file'])) try: default_bundle = {} with open(config['default_bundle_file'], 'r') as FH: default_bundle = json.loads(FH.read()) if default_bundle: resp = client.add_policy(default_bundle, active=True) if not resp: raise Exception("policy bundle DB add failed") return True else: raise Exception('No default bundle found') except Exception as err: logger.error( "could not load up default bundle for user - exception: " + str(err)) raise else: logger.debug( 'Existing bundle found for account: {}. Not expected on invocations of this function in most uses' .format(accountname)) return False
def get_event(eventId): request_inputs = anchore_engine.apis.do_request_prep(request, default_params={}) user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 userId, pw = user_auth try: client = internal_client_for(CatalogClient, request_inputs['userId']) return_object = client.get_event(eventId) httpcode = 200 except Exception as err: return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return return_object, httpcode
def create_operation(): """ POST /imports/images :return: """ try: client = internal_client_for( CatalogClient, userId=ApiRequestContextProxy.namespace() ) resp = client.create_image_import() return resp, 200 except api_exceptions.AnchoreApiError as ex: return ( make_response_error(ex, in_httpcode=ex.__response_code__), ex.__response_code__, ) except Exception as ex: logger.exception("Unexpected error in api processing") return make_response_error(ex, in_httpcode=500), 500
def list_events(source_servicename=None, source_hostid=None, resource_type=None, resource_id=None, level=None, since=None, before=None, page=None, limit=None): request_inputs = anchore_engine.apis.do_request_prep(request, default_params={}) user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 try: client = internal_client_for(CatalogClient, request_inputs['userId']) return_object = client.get_events(source_servicename=source_servicename, source_hostid=source_hostid, resource_type=resource_type, resource_id=resource_id, level=level, since=since, before=before, page=page, limit=limit) httpcode = 200 except Exception as err: return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return return_object, httpcode
def authz_heartbeat(*args, **kwargs): cycle_timer = kwargs["mythread"]["cycle_timer"] logger.info("Checking authz availability") try: host_id = localconfig.get_host_id() authz_handlr = get_authorizer() handler = authz_handlr.__class__.__name__ ex = None try: result = authz_handlr.healthcheck() except Exception as e: ex = e result = False if not result: fail_event = ServiceAuthzPluginHealthCheckFailed( user_id=localconfig.ADMIN_ACCOUNT_NAME, name=service_name, host=host_id, plugin=handler, details=str(ex), ) logger.info("Sending healthcheck failure event: {}".format( fail_event.__event_type__)) try: client = internal_client_for( CatalogClient, localconfig.ADMIN_ACCOUNT_NAME) client.add_event(fail_event) except Exception as ex: logger.exception( "Failure to send authz healthcheck failure event: {}" .format(fail_event.to_json())) except Exception as e: logger.exception( "Caught unexpected exception from the authz heartbeat handler" ) time.sleep(cycle_timer) return True
def list_import_image_configs(operation_id): """ GET /imports/images/{operation_id}/image_config :param operation_id: :return: """ try: client = internal_client_for( CatalogClient, userId=ApiRequestContextProxy.namespace() ) resp = client.list_import_content(operation_id, "image_config") return resp, 200 except api_exceptions.AnchoreApiError as ex: return ( make_response_error(ex, in_httpcode=ex.__response_code__), ex.__response_code__, ) except Exception as ex: logger.exception("Unexpected error in api processing") return make_response_error(ex, in_httpcode=500), 500
def repositories(request_inputs): method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 input_repo = None if params and 'repository' in params: input_repo = params['repository'] autosubscribe = False if params and 'autosubscribe' in params: autosubscribe = params['autosubscribe'] lookuptag = None if params and 'lookuptag' in params: lookuptag = params['lookuptag'] try: if method == 'POST': logger.debug("handling POST: ") try: client = internal_client_for(CatalogClient, request_inputs['userId']) return_object = [] repo_records = client.add_repo(regrepo=input_repo, autosubscribe=autosubscribe, lookuptag=lookuptag) for repo_record in repo_records: return_object.append(repo_record) httpcode = 200 except Exception as err: raise err except Exception as err: logger.debug("operation exception: " + str(err)) return_object = anchore_engine.common.helpers.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return(return_object, httpcode)
def queue_notification(userId, subscription_key, subscription_type, payload): """ Put a Notification in the Queue! """ q_client = internal_client_for(SimpleQueueClient, None) rc = False try: nobj = { "userId": userId, "subscription_key": subscription_key, "notificationId": str(uuid.uuid4()), } if payload: nobj.update(payload) if not q_client.is_inqueue(subscription_type, nobj): rc = q_client.enqueue(subscription_type, nobj) except Exception as err: logger.warn("failed to create/enqueue notification") raise err return rc