def images_imageDigest_check(request_inputs, imageDigest): user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 userId, pw = user_auth try: image_records = catalog.get_image(user_auth, imageDigest=imageDigest) for image_record in image_records: if image_record['analysis_status'] != taskstate.complete_state( 'analyze'): httpcode = 404 raise Exception("image is not analyzed - analysis_status: " + str(image_record['analysis_status'])) return_object, httpcode = images_check_impl(request_inputs, image_records) except Exception as err: logger.debug("operation exception: " + str(err)) return_object = anchore_engine.services.common.make_response_error( err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)
def get_content(request_inputs, content_type, doformat=False): user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 userId, pw = user_auth try: tag = params.pop('tag', None) imageDigest = params.pop('imageDigest', None) digest = params.pop('digest', None) image_reports = catalog.get_image(user_auth, tag=tag, digest=digest, imageDigest=imageDigest) for image_report in image_reports: if image_report['analysis_status'] != taskstate.complete_state('analyze'): httpcode = 404 raise Exception("image is not analyzed - analysis_status: " + image_report['analysis_status']) imageDigest = image_report['imageDigest'] image_content_data = catalog.get_document(user_auth, 'image_content_data', imageDigest) return_object[imageDigest] = make_response_content(content_type, image_content_data[content_type]) httpcode = 200 except Exception as err: return_object = anchore_engine.services.common.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)
def lookup_image_id_by_ref(self, context, fromline_ref): """ Find the image id for the fromline_reference value. The FROM line of a Dockerfile. Either a tag reference or a digest reference. Scans the service db, not the source registry. :param context: :param fromline_ref: :return: image_id mapped to tag (as far as we know) or None if there isn't one or the reference is a digest rather than tag """ if re.match('@sha256:[a-f0-9]+$', fromline_ref): # it's a digest, no match since those are immutable return else: name_type = 'tag' creds = self._get_catalog_creds(context.db) if name_type == 'tag': if not creds: raise TriggerEvaluationError( self, 'Could not locate credentials for querying the catalog') try: record = get_image(userId=(creds['userId'], creds['password']), tag=fromline_ref) return record[0]['image_detail'][0]['imageId'] except Exception as e: log.exception( 'Received exception looking up image in catalog by ref: {}. {}' .format(fromline_ref, e)) # Assume this means the image is not available return None elif name_type == 'digest': try: record = get_image(userId=(creds['userId'], creds['password']), digest=fromline_ref) return record[0]['image_detail'][0]['imageDigest'] except Exception as e: # Assume this means the image is not available log.exception( 'Received exception looking up image in catalog by ref: {}' .format(fromline_ref, e)) return None else: return None
def get_content(request_inputs, content_type, doformat=False): user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 userId, pw = user_auth try: if content_type not in anchore_engine.services.common.image_content_types: httpcode = 404 raise Exception("content type (" + str(content_type) + ") not available") tag = params.pop('tag', None) imageDigest = params.pop('imageDigest', None) digest = params.pop('digest', None) image_reports = catalog.get_image(user_auth, tag=tag, digest=digest, imageDigest=imageDigest) for image_report in image_reports: if image_report['analysis_status'] != taskstate.complete_state( 'analyze'): httpcode = 404 raise Exception("image is not analyzed - analysis_status: " + image_report['analysis_status']) imageDigest = image_report['imageDigest'] try: image_content_data = catalog.get_document( user_auth, 'image_content_data', imageDigest) except Exception as err: raise anchore_engine.services.common.make_anchore_exception( err, input_message="cannot fetch content data from archive", input_httpcode=500) if content_type not in image_content_data: httpcode = 404 raise Exception( "image content of type (" + str(content_type) + ") was not an available type at analysis time for this image" ) return_object[imageDigest] = make_response_content( content_type, image_content_data[content_type]) httpcode = 200 except Exception as err: return_object = anchore_engine.services.common.make_response_error( err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)
def images_imageDigest(request_inputs, imageDigest): user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 userId, pw = user_auth try: if method == 'GET': logger.debug("handling GET on imageDigest: " + str(imageDigest)) image_records = catalog.get_image(user_auth, imageDigest=imageDigest) if image_records: return_object = [] for image_record in image_records: #try: # query_data = catalog.get_document(user_auth, 'query_data', imageDigest) # if 'anchore_image_summary' in query_data and query_data['anchore_image_summary']: # logger.debug("getting image summary data") #except Exception as err: # logger.warn("cannot get image summary data for image: " + str(imageDigest)) #try: # image_content_metadata = get_image_summary(user_auth, image_record) #except: # image_content_metadata = {} return_object.append(make_response_image(user_auth, image_record, params)) httpcode = 200 else: httpcode = 404 raise Exception("cannot locate specified image") elif method == 'DELETE': logger.debug("handling DELETE on imageDigest: " + str(imageDigest)) rc = False try: rc = catalog.delete_image(user_auth, imageDigest) except Exception as err: raise err if rc: return_object = rc httpcode = 200 else: httpcode = 500 raise Exception("failed to delete") except Exception as err: logger.debug("operation exception: " + str(err)) return_object = anchore_engine.services.common.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)
def vulnerability_query(request_inputs, vulnerability_type, doformat=False): user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 userId, pw = user_auth localconfig = anchore_engine.configuration.localconfig.get_config() system_user_auth = localconfig['system_user_auth'] verify = localconfig['internal_ssl_verify'] try: if vulnerability_type not in anchore_engine.services.common.image_vulnerability_types: httpcode = 404 raise Exception("content type ("+str(vulnerability_type)+") not available") tag = params.pop('tag', None) imageDigest = params.pop('imageDigest', None) digest = params.pop('digest', None) image_reports = catalog.get_image(user_auth, tag=tag, digest=digest, imageDigest=imageDigest) for image_report in image_reports: if image_report['analysis_status'] != taskstate.complete_state('analyze'): httpcode = 404 raise Exception("image is not analyzed - analysis_status: " + image_report['analysis_status']) imageDigest = image_report['imageDigest'] try: if vulnerability_type == 'os': image_detail = image_report['image_detail'][0] imageId = image_detail['imageId'] client = anchore_engine.clients.policy_engine.get_client(user=system_user_auth[0], password=system_user_auth[1], verify_ssl=verify) resp = client.get_image_vulnerabilities(user_id=userId, image_id=imageId, force_refresh=False) if doformat: return_object[imageDigest] = make_response_vulnerability(vulnerability_type, resp.to_dict()) else: return_object[imageDigest] = resp.to_dict() else: return_object[imageDigest] = [] httpcode = 200 except Exception as err: httpcode = 500 raise Exception("could not fetch vulnerabilities - exception: " + str(err)) httpcode = 200 except Exception as err: return_object = anchore_engine.services.common.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)
def images_imageDigest(request_inputs, imageDigest): user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 userId, pw = user_auth try: if method == 'GET': logger.debug("handling GET on imageDigest: " + str(imageDigest)) image_records = catalog.get_image(user_auth, imageDigest=imageDigest) if image_records: return_object = [] for image_record in image_records: return_object.append( make_response_image(user_auth, image_record, params)) httpcode = 200 else: httpcode = 404 raise Exception("cannot locate specified image") elif method == 'DELETE': logger.debug("handling DELETE on imageDigest: " + str(imageDigest)) rc = False try: rc = catalog.delete_image(user_auth, imageDigest, force=params['force']) except Exception as err: raise err if rc: return_object = rc httpcode = 200 else: httpcode = 500 raise Exception("failed to delete") except Exception as err: logger.debug("operation exception: " + str(err)) return_object = anchore_engine.services.common.make_response_error( err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)
def lookup_imageDigest_from_imageId(request_inputs, imageId): user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] userId, pw = user_auth ret = None try: image_records = catalog.get_image(user_auth, imageId=imageId) if image_records: image_record = image_records[0] imageDigest = image_record['imageDigest'] ret = imageDigest except Exception as err: logger.debug("operation exception: " + str(err)) raise err return (ret)
def process_analyzer_job(system_user_auth, qobj, layer_cache_enable): global current_avg, current_avg_count timer = int(time.time()) try: record = qobj['data'] userId = record['userId'] image_record = record['image_record'] manifest = record['manifest'] imageDigest = image_record['imageDigest'] user_record = catalog.get_user(system_user_auth, userId) user_auth = (user_record['userId'], user_record['password']) # check to make sure image is still in DB try: image_records = catalog.get_image(user_auth, imageDigest=imageDigest) if image_records: image_record = image_records[0] else: raise Exception("empty image record from catalog") except Exception as err: logger.warn( "dequeued image cannot be fetched from catalog - skipping analysis (" + str(imageDigest) + ") - exception: " + str(err)) return (True) logger.info("image dequeued for analysis: " + str(userId) + " : " + str(imageDigest)) try: logger.spew("TIMING MARK0: " + str(int(time.time()) - timer)) last_analysis_status = image_record['analysis_status'] image_record[ 'analysis_status'] = anchore_engine.subsys.taskstate.working_state( 'analyze') rc = catalog.update_image(user_auth, imageDigest, image_record) # disable the webhook call for image state transistion to 'analyzing' #try: # for image_detail in image_record['image_detail']: # fulltag = image_detail['registry'] + "/" + image_detail['repo'] + ":" + image_detail['tag'] # npayload = { # 'last_eval': {'imageDigest': imageDigest, 'analysis_status': last_analysis_status}, # 'curr_eval': {'imageDigest': imageDigest, 'analysis_status': image_record['analysis_status']}, # } # rc = anchore_engine.subsys.notifications.queue_notification(userId, fulltag, 'analysis_update', npayload) #except Exception as err: # logger.warn("failed to enqueue notification on image analysis state update - exception: " + str(err)) # actually do analysis registry_creds = catalog.get_registry(user_auth) image_data = perform_analyze(userId, manifest, image_record, registry_creds, layer_cache_enable=layer_cache_enable) imageId = None try: imageId = image_data[0]['image']['imageId'] except Exception as err: logger.warn( "could not get imageId after analysis or from image record - exception: " + str(err)) logger.debug("archiving analysis data") rc = catalog.put_document(user_auth, 'analysis_data', imageDigest, image_data) if rc: try: logger.debug("extracting image content data") image_content_data = {} for content_type in anchore_engine.services.common.image_content_types: try: image_content_data[ content_type] = anchore_engine.services.common.extract_analyzer_content( image_data, content_type) except: image_content_data[content_type] = {} if image_content_data: logger.debug("adding image content data to archive") rc = catalog.put_document(user_auth, 'image_content_data', imageDigest, image_content_data) try: logger.debug( "adding image analysis data to image_record") anchore_engine.services.common.update_image_record_with_analysis_data( image_record, image_data) except Exception as err: raise err except Exception as err: logger.warn( "could not store image content metadata to archive - exception: " + str(err)) logger.debug("adding image record to policy-engine service (" + str(userId) + " : " + str(imageId) + ")") try: if not imageId: raise Exception( "cannot add image to policy engine without an imageId" ) localconfig = anchore_engine.configuration.localconfig.get_config( ) verify = localconfig['internal_ssl_verify'] client = anchore_engine.clients.policy_engine.get_client( user=system_user_auth[0], password=system_user_auth[1], verify_ssl=verify) try: logger.debug( "clearing any existing record in policy engine for image: " + str(imageId)) rc = client.delete_image(user_id=userId, image_id=imageId) except Exception as err: logger.warn("exception on pre-delete - exception: " + str(err)) request = ImageIngressRequest() request.user_id = userId request.image_id = imageId request.fetch_url = 'catalog://' + str( userId) + '/analysis_data/' + str(imageDigest) logger.debug("policy engine request: " + str(request)) resp = client.ingress_image(request) logger.debug("policy engine image add response: " + str(resp)) try: # force a fresh CVE scan resp = client.get_image_vulnerabilities( user_id=userId, image_id=imageId, force_refresh=True) except Exception as err: logger.warn( "post analysis CVE scan failed for image: " + str(imageId)) except Exception as err: raise Exception( "adding image to policy-engine failed - exception: " + str(err)) logger.debug("updating image catalog record analysis_status") last_analysis_status = image_record['analysis_status'] image_record[ 'analysis_status'] = anchore_engine.subsys.taskstate.complete_state( 'analyze') rc = catalog.update_image(user_auth, imageDigest, image_record) try: annotations = {} try: annotations = json.loads( image_record.get('annotations', {})) except Exception as err: logger.warn( "could not marshal annotations from json - exception: " + str(err)) for image_detail in image_record['image_detail']: fulltag = image_detail['registry'] + "/" + image_detail[ 'repo'] + ":" + image_detail['tag'] last_payload = { 'imageDigest': imageDigest, 'analysis_status': last_analysis_status, 'annotations': annotations } curr_payload = { 'imageDigest': imageDigest, 'analysis_status': image_record['analysis_status'], 'annotations': annotations } npayload = { 'last_eval': last_payload, 'curr_eval': curr_payload, } if annotations: npayload['annotations'] = annotations rc = anchore_engine.subsys.notifications.queue_notification( userId, fulltag, 'analysis_update', npayload) except Exception as err: logger.warn( "failed to enqueue notification on image analysis state update - exception: " + str(err)) else: raise Exception("analysis archive failed to store") logger.info("analysis complete: " + str(userId) + " : " + str(imageDigest)) logger.spew("TIMING MARK1: " + str(int(time.time()) - timer)) try: run_time = float(time.time() - timer) current_avg_count = current_avg_count + 1.0 new_avg = current_avg + ( (run_time - current_avg) / current_avg_count) current_avg = new_avg except: pass except Exception as err: logger.exception("problem analyzing image - exception: " + str(err)) image_record[ 'analysis_status'] = anchore_engine.subsys.taskstate.fault_state( 'analyze') image_record[ 'image_status'] = anchore_engine.subsys.taskstate.fault_state( 'image_status') rc = catalog.update_image(user_auth, imageDigest, image_record) except Exception as err: logger.warn("job processing bailed - exception: " + str(err)) raise err return (True)
def process_analyzer_job(system_user_auth, qobj): global current_avg, current_avg_count timer = int(time.time()) try: record = qobj['data'] userId = record['userId'] image_record = record['image_record'] imageDigest = image_record['imageDigest'] user_record = catalog.get_user(system_user_auth, userId) user_auth = (user_record['userId'], user_record['password']) # check to make sure image is still in DB try: image_records = catalog.get_image(user_auth, imageDigest=imageDigest) if image_records: image_record = image_records[0] else: raise Exception("empty image record from catalog") except Exception as err: logger.warn("dequeued image cannot be fetched from catalog - skipping analysis (" + str( imageDigest) + ") - exception: " + str(err)) return (True) logger.info("image dequeued for analysis: " + str(userId) + " : " + str(imageDigest)) try: logger.spew("TIMING MARK0: " + str(int(time.time()) - timer)) image_record['analysis_status'] = anchore_engine.subsys.taskstate.working_state('analyze') rc = catalog.update_image(user_auth, imageDigest, image_record) # actually do analysis # for pullstring in pullstrings.keys(): for image_detail in image_record['image_detail']: pullstring = image_detail['registry'] + "/" + image_detail['repo'] + "@" + image_detail['digest'] fulltag = image_detail['registry'] + "/" + image_detail['repo'] + ":" + image_detail['tag'] imageId = None if 'imageId' in image_detail and image_detail['imageId']: imageId = image_detail['imageId'] logger.info("analysis starting: " + str(userId) + " : " + str(imageDigest) + " : " + str(fulltag) + " : " + str(imageId)) logger.spew("TIMING MARKX: " + str(int(time.time()) - timer)) registry_creds = catalog.get_registry(user_auth) image_data, query_data = perform_analyze(userId, pullstring, fulltag, image_detail, registry_creds) logger.spew("TIMING MARKY: " + str(int(time.time()) - timer)) logger.debug("archiving query data") rc = catalog.put_document(user_auth, 'query_data', imageDigest, query_data) if rc: logger.debug("storing image query data to catalog") else: raise Exception("query archive failed to store") if not imageId: try: imageId = image_data[0]['image']['imageId'] except Exception as err: logger.warn("could not get imageId after analysis or from image record - exception: " + str(err)) logger.debug("archiving analysis data") rc = catalog.put_document(user_auth, 'analysis_data', imageDigest, image_data) if rc: try: logger.debug("extracting image content data") image_content_data = {} for content_type in anchore_engine.services.common.image_content_types: try: image_content_data[content_type] = anchore_engine.services.common.extract_analyzer_content(image_data, content_type) except: image_content_data[content_type] = {} if image_content_data: logger.debug("adding image content data to archive") rc = catalog.put_document(user_auth, 'image_content_data', imageDigest, image_content_data) image_summary_data = {} try: image_summary_data = anchore_engine.services.common.extract_analyzer_content(image_data, 'metadata') except: image_summary_data = {} if image_summary_data: logger.debug("adding image summary data to archive") rc = catalog.put_document(user_auth, 'image_summary_data', imageDigest, image_summary_data) except Exception as err: logger.warn("could not store image content metadata to archive - exception: " + str(err)) logger.debug("adding image record to policy-engine service (" + str(userId) + " : " + str(imageId) + ")") try: if not imageId: raise Exception("cannot add image to policy engine without an imageId") localconfig = anchore_engine.configuration.localconfig.get_config() verify = localconfig['internal_ssl_verify'] client = anchore_engine.clients.policy_engine.get_client(user=system_user_auth[0], password=system_user_auth[1], verify_ssl=verify) try: logger.debug("clearing any existing record in policy engine for image: " + str(imageId)) rc = client.delete_image(user_id=userId, image_id=imageId) except Exception as err: logger.warn("exception on pre-delete - exception: " + str(err)) request = ImageIngressRequest() request.user_id = userId request.image_id = imageId request.fetch_url='catalog://'+str(userId)+'/analysis_data/'+str(imageDigest) logger.debug("policy engine request: " + str(request)) resp = client.ingress_image(request) logger.debug("policy engine image add response: " + str(resp)) try: # force a fresh CVE scan resp = client.get_image_vulnerabilities(user_id=userId, image_id=imageId, force_refresh=True) except Exception as err: logger.warn("post analysis CVE scan failed for image: " + str(imageId)) except Exception as err: raise Exception("adding image to policy-engine failed - exception: " + str(err)) logger.debug("updating image catalog record analysis_status") image_record['analysis_status'] = anchore_engine.subsys.taskstate.complete_state('analyze') rc = catalog.update_image(user_auth, imageDigest, image_record) else: raise Exception("analysis archive failed to store") logger.info("analysis complete: " + str(userId) + " : " + str(imageDigest) + " : " + str(fulltag)) logger.spew("TIMING MARK1: " + str(int(time.time()) - timer)) try: run_time = float(time.time() - timer) current_avg_count = current_avg_count + 1.0 new_avg = current_avg + ((run_time - current_avg) / current_avg_count) current_avg = new_avg except: pass except Exception as err: logger.exception("problem analyzing image - exception: " + str(err)) image_record['analysis_status'] = anchore_engine.subsys.taskstate.fault_state('analyze') image_record['image_status'] = anchore_engine.subsys.taskstate.fault_state('image_status') rc = catalog.update_image(user_auth, imageDigest, image_record) except Exception as err: logger.warn("job processing bailed - exception: " + str(err)) raise err return (True)
def images(request_inputs): user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 userId, pw = user_auth digest = tag = imageId = imageDigest = dockerfile = None history = False if params and 'history' in params: history = params['history'] force = False if params and 'force' in params: force = params['force'] if bodycontent: jsondata = json.loads(bodycontent) if 'digest' in jsondata: digest = jsondata['digest'] elif 'tag' in jsondata: tag = jsondata['tag'] elif 'imageDigest' in jsondata: imageDigest = jsondata['imageDigest'] elif 'imageId' in jsondata: imageId = jsondata['imageId'] if 'dockerfile' in jsondata: dockerfile = jsondata['dockerfile'] try: if method == 'GET': logger.debug("handling GET: ") try: return_object = [] image_records = catalog.get_image(user_auth, digest=digest, tag=tag, imageId=imageId, imageDigest=imageDigest, history=history) for image_record in image_records: return_object.append(make_response_image(user_auth, image_record, params)) httpcode = 200 except Exception as err: raise err elif method == 'POST': logger.debug("handling POST: ") # if not, add it and set it up to be analyzed if not tag: # dont support digest add, yet httpcode = 500 raise Exception("digest add unsupported") # add the image to the catalog image_record = catalog.add_image(user_auth, tag=tag, dockerfile=dockerfile) imageDigest = image_record['imageDigest'] # finally, do any state updates and return if image_record: #logger.debug("fetched image_info: " + json.dumps(image_record, indent=4)) logger.debug("added image: " + str(imageDigest)) # auto-subscribe for NOW for image_detail in image_record['image_detail']: fulltag = image_detail['registry'] + "/" + image_detail['repo'] + ":" + image_detail['tag'] foundtypes = [] try: subscription_records = catalog.get_subscription(user_auth, subscription_key=fulltag) for subscription_record in subscription_records: if subscription_record['subscription_key'] == fulltag: foundtypes.append(subscription_record['subscription_type']) except Exception as err: logger.warn("cannot load subscription records - exception: " + str(err)) sub_types = anchore_engine.services.common.subscription_types for sub_type in sub_types: if sub_type in ['repo_update']: continue if sub_type not in foundtypes: try: default_active = False if sub_type in ['tag_update']: default_active = True catalog.add_subscription(user_auth, {'active': default_active, 'subscription_type': sub_type, 'subscription_key': fulltag}) except: try: catalog.update_subscription(user_auth, {'subscription_type': sub_type, 'subscription_key': fulltag}) except: pass # set the state of the image appropriately currstate = image_record['analysis_status'] if not currstate: newstate = taskstate.init_state('analyze', None) elif force or currstate == taskstate.fault_state('analyze'): newstate = taskstate.reset_state('analyze') elif image_record['image_status'] == 'deleted': newstate = taskstate.reset_state('analyze') else: newstate = currstate if (currstate != newstate) or (force): logger.debug("state change detected: " + str(currstate) + " : " + str(newstate)) image_record.update({'image_status': 'active', 'analysis_status': newstate}) updated_image_record = catalog.update_image(user_auth, imageDigest, image_record) if updated_image_record: image_record = updated_image_record[0] else: logger.debug("no state change detected: " + str(currstate) + " : " + str(newstate)) httpcode = 200 return_object = [make_response_image(user_auth, image_record, params)] else: httpcode = 500 raise Exception("failed to add image") except Exception as err: logger.debug("operation exception: " + str(err)) return_object = anchore_engine.services.common.make_response_error(err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)
def process_analyzer_job(system_user_auth, qobj, layer_cache_enable): global servicename #current_avg, current_avg_count timer = int(time.time()) event = None try: logger.debug('dequeued object: {}'.format(qobj)) record = qobj['data'] userId = record['userId'] imageDigest = record['imageDigest'] manifest = record['manifest'] user_record = catalog.get_user(system_user_auth, userId) user_auth = (user_record['userId'], user_record['password']) # check to make sure image is still in DB try: image_records = catalog.get_image(user_auth, imageDigest=imageDigest) if image_records: image_record = image_records[0] else: raise Exception("empty image record from catalog") except Exception as err: logger.warn( "dequeued image cannot be fetched from catalog - skipping analysis (" + str(imageDigest) + ") - exception: " + str(err)) return (True) logger.info("image dequeued for analysis: " + str(userId) + " : " + str(imageDigest)) if image_record[ 'analysis_status'] != anchore_engine.subsys.taskstate.base_state( 'analyze'): logger.debug( "dequeued image is not in base state - skipping analysis") return (True) try: logger.spew("TIMING MARK0: " + str(int(time.time()) - timer)) last_analysis_status = image_record['analysis_status'] image_record[ 'analysis_status'] = anchore_engine.subsys.taskstate.working_state( 'analyze') rc = catalog.update_image(user_auth, imageDigest, image_record) # disable the webhook call for image state transistion to 'analyzing' #try: # for image_detail in image_record['image_detail']: # fulltag = image_detail['registry'] + "/" + image_detail['repo'] + ":" + image_detail['tag'] # npayload = { # 'last_eval': {'imageDigest': imageDigest, 'analysis_status': last_analysis_status}, # 'curr_eval': {'imageDigest': imageDigest, 'analysis_status': image_record['analysis_status']}, # } # rc = anchore_engine.subsys.notifications.queue_notification(userId, fulltag, 'analysis_update', npayload) #except Exception as err: # logger.warn("failed to enqueue notification on image analysis state update - exception: " + str(err)) # actually do analysis registry_creds = catalog.get_registry(user_auth) try: image_data = perform_analyze( userId, manifest, image_record, registry_creds, layer_cache_enable=layer_cache_enable) except AnchoreException as e: event = events.AnalyzeImageFail(user_id=userId, image_digest=imageDigest, error=e.to_dict()) raise imageId = None try: imageId = image_data[0]['image']['imageId'] except Exception as err: logger.warn( "could not get imageId after analysis or from image record - exception: " + str(err)) try: logger.debug("archiving analysis data") rc = catalog.put_document(user_auth, 'analysis_data', imageDigest, image_data) except Exception as e: err = CatalogClientError( msg='Failed to upload analysis data to catalog', cause=e) event = events.ArchiveAnalysisFail(user_id=userId, image_digest=imageDigest, error=err.to_dict()) raise err if rc: try: logger.debug("extracting image content data") image_content_data = {} for content_type in anchore_engine.services.common.image_content_types + anchore_engine.services.common.image_metadata_types: try: image_content_data[ content_type] = anchore_engine.services.common.extract_analyzer_content( image_data, content_type, manifest=manifest) except: image_content_data[content_type] = {} if image_content_data: logger.debug("adding image content data to archive") rc = catalog.put_document(user_auth, 'image_content_data', imageDigest, image_content_data) try: logger.debug( "adding image analysis data to image_record") anchore_engine.services.common.update_image_record_with_analysis_data( image_record, image_data) except Exception as err: raise err except Exception as err: logger.warn( "could not store image content metadata to archive - exception: " + str(err)) logger.debug("adding image record to policy-engine service (" + str(userId) + " : " + str(imageId) + ")") try: if not imageId: raise Exception( "cannot add image to policy engine without an imageId" ) localconfig = anchore_engine.configuration.localconfig.get_config( ) verify = localconfig['internal_ssl_verify'] client = anchore_engine.clients.policy_engine.get_client( user=system_user_auth[0], password=system_user_auth[1], verify_ssl=verify) try: logger.debug( "clearing any existing record in policy engine for image: " + str(imageId)) rc = client.delete_image(user_id=userId, image_id=imageId) except Exception as err: logger.warn("exception on pre-delete - exception: " + str(err)) logger.info('Loading image: {} {}'.format(userId, imageId)) request = ImageIngressRequest( user_id=userId, image_id=imageId, fetch_url='catalog://' + str(userId) + '/analysis_data/' + str(imageDigest)) logger.debug("policy engine request: " + str(request)) resp = client.ingress_image(request) logger.debug("policy engine image add response: " + str(resp)) except Exception as err: import traceback traceback.print_exc() newerr = PolicyEngineClientError( msg='Adding image to policy-engine failed', cause=str(err)) event = events.LoadAnalysisFail(user_id=userId, image_digest=imageDigest, error=newerr.to_dict()) raise newerr logger.debug("updating image catalog record analysis_status") last_analysis_status = image_record['analysis_status'] image_record[ 'analysis_status'] = anchore_engine.subsys.taskstate.complete_state( 'analyze') image_record['analyzed_at'] = int(time.time()) rc = catalog.update_image(user_auth, imageDigest, image_record) try: annotations = {} try: if image_record.get('annotations', '{}'): annotations = json.loads( image_record.get('annotations', '{}')) except Exception as err: logger.warn( "could not marshal annotations from json - exception: " + str(err)) for image_detail in image_record['image_detail']: fulltag = image_detail['registry'] + "/" + image_detail[ 'repo'] + ":" + image_detail['tag'] last_payload = { 'imageDigest': imageDigest, 'analysis_status': last_analysis_status, 'annotations': annotations } curr_payload = { 'imageDigest': imageDigest, 'analysis_status': image_record['analysis_status'], 'annotations': annotations } npayload = { 'last_eval': last_payload, 'curr_eval': curr_payload, } if annotations: npayload['annotations'] = annotations rc = anchore_engine.subsys.notifications.queue_notification( userId, fulltag, 'analysis_update', npayload) except Exception as err: logger.warn( "failed to enqueue notification on image analysis state update - exception: " + str(err)) else: err = CatalogClientError( msg='Failed to upload analysis data to catalog', cause='Invalid response from catalog API - {}'.format( str(rc))) event = events.ArchiveAnalysisFail(user_id=userId, image_digest=imageDigest, error=err.to_dict()) raise err logger.info("analysis complete: " + str(userId) + " : " + str(imageDigest)) logger.spew("TIMING MARK1: " + str(int(time.time()) - timer)) try: run_time = float(time.time() - timer) #current_avg_count = current_avg_count + 1.0 #new_avg = current_avg + ((run_time - current_avg) / current_avg_count) #current_avg = new_avg anchore_engine.subsys.metrics.histogram_observe( 'anchore_analysis_time_seconds', run_time, buckets=[ 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1800.0, 3600.0 ], status="success") #anchore_engine.subsys.metrics.counter_inc('anchore_images_analyzed_total') #localconfig = anchore_engine.configuration.localconfig.get_config() #service_record = {'hostid': localconfig['host_id'], 'servicename': servicename} #anchore_engine.subsys.servicestatus.set_status(service_record, up=True, available=True, detail={'avg_analysis_time_sec': current_avg, 'total_analysis_count': current_avg_count}, update_db=True) except Exception as err: logger.warn(str(err)) pass except Exception as err: run_time = float(time.time() - timer) logger.exception("problem analyzing image - exception: " + str(err)) anchore_engine.subsys.metrics.histogram_observe( 'anchore_analysis_time_seconds', run_time, buckets=[ 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1800.0, 3600.0 ], status="fail") image_record[ 'analysis_status'] = anchore_engine.subsys.taskstate.fault_state( 'analyze') image_record[ 'image_status'] = anchore_engine.subsys.taskstate.fault_state( 'image_status') rc = catalog.update_image(user_auth, imageDigest, image_record) finally: if event: try: catalog.add_event(user_auth, event) except: logger.error( 'Ignoring error creating analysis failure event') except Exception as err: logger.warn("job processing bailed - exception: " + str(err)) raise err return (True)
def imagepolicywebhook(bodycontent): # TODO - while the image policy webhook feature is in k8s beta, we've decided to make any errors that occur during check still respond with 'allowed: True'. This should be reverted to default to 'False' on any error, once the k8s feature is further along return_object = { "apiVersion": "imagepolicy.k8s.io/v1alpha1", "kind": "ImageReview", "status": { "allowed": True, "reason": "all images passed anchore policy evaluation" } } httpcode = 200 try: request_inputs = anchore_engine.services.common.do_request_prep( connexion.request, default_params={}) user_auth = request_inputs['auth'] method = request_inputs['method'] params = request_inputs['params'] userId = request_inputs['userId'] try: final_allowed = True reason = "unset" try: try: #incoming = json.loads(bodycontent) incoming = bodycontent logger.debug("incoming post data: " + json.dumps(incoming, indent=4)) except Exception as err: raise Exception("could not load post data as json: " + str(err)) try: requestUserId = None requestPolicyId = None # see if the request from k8s contains an anchore policy and/or whitelist name if 'annotations' in incoming['spec']: logger.debug( "incoming request contains annotations: " + json.dumps(incoming['spec']['annotations'], indent=4)) requestUserId = incoming['spec']['annotations'].pop( "anchore.image-policy.k8s.io/userId", None) requestPolicyId = incoming['spec']['annotations'].pop( "anchore.image-policy.k8s.io/policyBundleId", None) except Exception as err: raise Exception("could not parse out annotations: " + str(err)) if not requestUserId: raise Exception( "need to specify an anchore.image-policy.k8s.io/userId annotation with a valid anchore service username as a value" ) # TODO - get anchore system uber cred to access # this data on behalf of user? tough...maybe see # if kuber can make request as anchore-system so # that we can switch roles? localconfig = anchore_engine.configuration.localconfig.get_config( ) system_user_auth = localconfig['system_user_auth'] user_record = catalog.get_user(system_user_auth, requestUserId) request_user_auth = (user_record['userId'], user_record['password']) reason = "all images passed anchore policy checks" final_action = False for el in incoming['spec']['containers']: image = el['image'] logger.debug("found image in request: " + str(image)) image_records = catalog.get_image(request_user_auth, tag=image) if not image_records: raise Exception("could not find requested image (" + str(image) + ") in anchore service DB") for image_record in image_records: imageDigest = image_record['imageDigest'] for image_detail in image_record['image_detail']: fulltag = image_detail[ 'registry'] + "/" + image_detail[ 'repo'] + ':' + image_detail['tag'] result = catalog.get_eval_latest( request_user_auth, tag=fulltag, imageDigest=imageDigest, policyId=requestPolicyId) if result: httpcode = 200 if result['final_action'].upper() not in [ 'GO', 'WARN' ]: final_action = False raise Exception( "image failed anchore policy check: " + json.dumps(result, indent=4)) else: final_action = True else: httpcode = 404 final_action = False raise Exception( "no anchore evaluation available for image: " + str(image)) final_allowed = final_action except Exception as err: reason = str(err) final_allowed = False httpcode = 200 return_object['status']['allowed'] = final_allowed return_object['status']['reason'] = reason anchore_engine.subsys.metrics.counter_inc( "anchore_image_policy_webhooks_evaluation_total", allowed=final_allowed) #logger.debug("final return: " + json.dumps(return_object, indent=4)) httpcode = 200 except Exception as err: return_object['reason'] = str(err) httpcode = 500 except Exception as err: return_object['reason'] = str(err) httpcode = 500 return (return_object, httpcode)
def get_content(request_inputs, content_type, doformat=False): user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 userId, pw = user_auth try: if content_type not in anchore_engine.services.common.image_content_types + anchore_engine.services.common.image_metadata_types: httpcode = 404 raise Exception("content type (" + str(content_type) + ") not available") tag = params.pop('tag', None) imageDigest = params.pop('imageDigest', None) digest = params.pop('digest', None) image_reports = catalog.get_image(user_auth, tag=tag, digest=digest, imageDigest=imageDigest) for image_report in image_reports: if image_report['analysis_status'] != taskstate.complete_state( 'analyze'): httpcode = 404 raise Exception("image is not analyzed - analysis_status: " + image_report['analysis_status']) imageDigest = image_report['imageDigest'] if content_type == 'manifest': try: image_manifest_data = catalog.get_document( user_auth, 'manifest_data', imageDigest) except Exception as err: raise anchore_engine.services.common.make_anchore_exception( err, input_message="cannot fetch content data {} from archive" .format(content_type), input_httpcode=500) image_content_data = {'manifest': image_manifest_data} else: try: image_content_data = catalog.get_document( user_auth, 'image_content_data', imageDigest) except Exception as err: raise anchore_engine.services.common.make_anchore_exception( err, input_message="cannot fetch content data from archive", input_httpcode=500) # special handler for dockerfile contents from old method to new if content_type == 'dockerfile' and not image_content_data.get( 'dockerfile', None): try: if image_report.get('dockerfile_mode', None) == 'Actual': for image_detail in image_report.get( 'image_detail', []): if image_detail.get('dockerfile', None): logger.debug( "migrating old dockerfile content form into new" ) image_content_data[ 'dockerfile'] = image_detail.get( 'dockerfile', "").decode('base64') catalog.put_document( user_auth, 'image_content_data', imageDigest, image_content_data) break except Exception as err: logger.warn( "cannot fetch/decode dockerfile contents from image_detail - {}" .format(err)) if content_type not in image_content_data: httpcode = 404 raise Exception( "image content of type (" + str(content_type) + ") was not an available type at analysis time for this image" ) return_object[imageDigest] = make_response_content( content_type, image_content_data[content_type]) httpcode = 200 except Exception as err: return_object = anchore_engine.services.common.make_response_error( err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)
def images(request_inputs): user_auth = request_inputs['auth'] method = request_inputs['method'] bodycontent = request_inputs['bodycontent'] params = request_inputs['params'] return_object = {} httpcode = 500 userId, pw = user_auth fulltag = digest = tag = imageId = imageDigest = dockerfile = annotations = None history = False force = False autosubscribe = True query_fulltag = None if params: if 'history' in params: history = params['history'] if 'force' in params: force = params['force'] if 'autosubscribe' in params: autosubscribe = params['autosubscribe'] if 'fulltag' in params: query_fulltag = params['fulltag'] if bodycontent: jsondata = json.loads(bodycontent) if 'digest' in jsondata: digest = jsondata['digest'] if 'tag' in jsondata: tag = jsondata['tag'] #elif 'imageDigest' in jsondata: # imageDigest = jsondata['imageDigest'] #elif 'imageId' in jsondata: # imageId = jsondata['imageId'] if 'dockerfile' in jsondata: dockerfile = jsondata['dockerfile'] if 'annotations' in jsondata: annotations = jsondata['annotations'] autosubscribes = ['analysis_update'] if autosubscribe: autosubscribes.append("tag_update") try: if method == 'GET': logger.debug("handling GET: ") try: return_object = [] # Query param fulltag has precedence for search if query_fulltag: tag = query_fulltag imageId = imageDigest = digest = None image_records = catalog.get_image(user_auth, digest=digest, tag=tag, imageId=imageId, imageDigest=imageDigest, history=history) for image_record in image_records: return_object.append( make_response_image(user_auth, image_record, params)) httpcode = 200 except Exception as err: raise err elif method == 'POST': logger.debug( "handling POST: input_tag={} input_digest={} input_force={}". format(tag, digest, force)) # if not, add it and set it up to be analyzed if not tag: # dont support digest add, yet httpcode = 400 raise Exception("tag is required for image add") if digest and tag: if not force: httpcode = 400 raise Exception("force is required to add digest+tag") else: try: image_check = catalog.get_image(user_auth, digest=digest, tag=tag, imageId=None, imageDigest=digest, history=False) except Exception as err: httpcode = 400 raise Exception( "image digest must already exist to force re-analyze using tag+digest" ) # add the image to the catalog image_record = catalog.add_image(user_auth, tag=tag, digest=digest, dockerfile=dockerfile, annotations=annotations) imageDigest = image_record['imageDigest'] # finally, do any state updates and return if image_record: logger.debug("added image: " + str(imageDigest)) # auto-subscribe for NOW for image_detail in image_record['image_detail']: fulltag = image_detail['registry'] + "/" + image_detail[ 'repo'] + ":" + image_detail['tag'] foundtypes = [] try: subscription_records = catalog.get_subscription( user_auth, subscription_key=fulltag) except Exception as err: subscription_records = [] for subscription_record in subscription_records: if subscription_record['subscription_key'] == fulltag: foundtypes.append( subscription_record['subscription_type']) sub_types = anchore_engine.services.common.subscription_types for sub_type in sub_types: if sub_type in ['repo_update']: continue if sub_type not in foundtypes: try: default_active = False if sub_type in autosubscribes: logger.debug("auto-subscribing image: " + str(sub_type)) default_active = True catalog.add_subscription( user_auth, { 'active': default_active, 'subscription_type': sub_type, 'subscription_key': fulltag }) except: try: catalog.update_subscription( user_auth, { 'subscription_type': sub_type, 'subscription_key': fulltag }) except: pass # set the state of the image appropriately currstate = image_record['analysis_status'] if not currstate: newstate = taskstate.init_state('analyze', None) elif force or currstate == taskstate.fault_state('analyze'): newstate = taskstate.reset_state('analyze') elif image_record['image_status'] == 'deleted': newstate = taskstate.reset_state('analyze') else: newstate = currstate if (currstate != newstate) or (force): logger.debug("state change detected: " + str(currstate) + " : " + str(newstate)) image_record.update({ 'image_status': 'active', 'analysis_status': newstate }) updated_image_record = catalog.update_image( user_auth, imageDigest, image_record) if updated_image_record: image_record = updated_image_record[0] else: logger.debug("no state change detected: " + str(currstate) + " : " + str(newstate)) httpcode = 200 return_object = [ make_response_image(user_auth, image_record, params) ] else: httpcode = 500 raise Exception("failed to add image") except Exception as err: logger.debug("operation exception: " + str(err)) return_object = anchore_engine.services.common.make_response_error( err, in_httpcode=httpcode) httpcode = return_object['httpcode'] return (return_object, httpcode)