def analysis_complete_notification_factory( account, image_digest: str, last_analysis_status: str, analysis_status: str, image_detail: dict, annotations: dict, ) -> events.UserAnalyzeImageCompleted: """ Return a constructed UserAnalysImageCompleted event from the input data :param account: :param image_digest: :param last_analysis_status: :param analysis_status: :param image_detail: :param annotations: :return: """ payload = { "last_eval": { "imageDigest": image_digest, "analysis_status": last_analysis_status, "annotations": annotations, }, "curr_eval": { "imageDigest": image_digest, "analysis_status": analysis_status, "annotations": annotations, }, "subscription_type": "analysis_update", "annotations": annotations or {}, } fulltag = fulltag_from_detail(image_detail) return events.UserAnalyzeImageCompleted(user_id=account, full_tag=fulltag, data=payload)
def process_analyzer_job(system_user_auth, qobj, layer_cache_enable): global servicename #current_avg, current_avg_count timer = int(time.time()) analysis_events = [] userId = None imageDigest = None localconfig = anchore_engine.configuration.localconfig.get_config() myconfig = localconfig['services']['analyzer'] try: logger.debug('dequeued object: {}'.format(qobj)) record = qobj['data'] userId = record['userId'] imageDigest = record['imageDigest'] manifest = record['manifest'] parent_manifest = record.get('parent_manifest', None) # check to make sure image is still in DB catalog_client = internal_client_for(CatalogClient, userId) try: image_record = catalog_client.get_image(imageDigest) if not image_record: raise Exception("empty image record from catalog") except Exception as err: logger.warn( "dequeued image cannot be fetched from catalog - skipping analysis (" + str(imageDigest) + ") - exception: " + str(err)) return (True) logger.info("image dequeued for analysis: " + str(userId) + " : " + str(imageDigest)) if image_record[ 'analysis_status'] != anchore_engine.subsys.taskstate.base_state( 'analyze'): logger.debug( "dequeued image is not in base state - skipping analysis") return (True) try: logger.spew("TIMING MARK0: " + str(int(time.time()) - timer)) last_analysis_status = image_record['analysis_status'] image_record[ 'analysis_status'] = anchore_engine.subsys.taskstate.working_state( 'analyze') rc = catalog_client.update_image(imageDigest, image_record) # actually do analysis registry_creds = catalog_client.get_registry() try: image_data = perform_analyze( userId, manifest, image_record, registry_creds, layer_cache_enable=layer_cache_enable, parent_manifest=parent_manifest) except AnchoreException as e: event = events.ImageAnalysisFailed(user_id=userId, image_digest=imageDigest, error=e.to_dict()) analysis_events.append(event) raise imageId = None try: imageId = image_data[0]['image']['imageId'] except Exception as err: logger.warn( "could not get imageId after analysis or from image record - exception: " + str(err)) logger.info( "adding image analysis data to catalog: userid={} imageId={} imageDigest={}" .format(userId, imageId, imageDigest)) try: logger.debug("archiving analysis data") rc = catalog_client.put_document('analysis_data', imageDigest, image_data) except Exception as e: err = CatalogClientError( msg='Failed to upload analysis data to catalog', cause=e) event = events.SaveAnalysisFailed(user_id=userId, image_digest=imageDigest, error=err.to_dict()) analysis_events.append(event) raise err if rc: try: logger.debug("extracting image content data locally") image_content_data = {} all_content_types = localconfig.get( 'image_content_types', []) + localconfig.get( 'image_metadata_types', []) for content_type in all_content_types: try: image_content_data[ content_type] = anchore_engine.common.helpers.extract_analyzer_content( image_data, content_type, manifest=manifest) except Exception as err: logger.warn("ERR: {}".format(err)) image_content_data[content_type] = {} if image_content_data: logger.debug("adding image content data to archive") rc = catalog_client.put_document( 'image_content_data', imageDigest, image_content_data) try: logger.debug( "adding image analysis data to image_record") anchore_engine.common.helpers.update_image_record_with_analysis_data( image_record, image_data) except Exception as err: raise err except Exception as err: import traceback traceback.print_exc() logger.warn( "could not store image content metadata to archive - exception: " + str(err)) logger.info( "adding image to policy engine: userid={} imageId={} imageDigest={}" .format(userId, imageId, imageDigest)) try: if not imageId: raise Exception( "cannot add image to policy engine without an imageId" ) #localconfig = anchore_engine.configuration.localconfig.get_config() verify = localconfig['internal_ssl_verify'] pe_client = internal_client_for(PolicyEngineClient, userId) try: logger.debug( "clearing any existing image record in policy engine: {} / {} / {}" .format(userId, imageId, imageDigest)) rc = pe_client.delete_image(user_id=userId, image_id=imageId) except Exception as err: logger.warn("exception on pre-delete - exception: " + str(err)) client_success = False last_exception = None for retry_wait in [1, 3, 5, 0]: try: logger.debug( 'loading image into policy engine: {} / {} / {}' .format(userId, imageId, imageDigest)) image_analysis_fetch_url = 'catalog://' + str( userId) + '/analysis_data/' + str(imageDigest) logger.debug("policy engine request: " + image_analysis_fetch_url) resp = pe_client.ingress_image( userId, imageId, image_analysis_fetch_url) logger.debug("policy engine image add response: " + str(resp)) client_success = True break except Exception as e: logger.warn( "attempt failed, will retry - exception: {}". format(e)) last_exception = e time.sleep(retry_wait) if not client_success: raise last_exception except Exception as err: newerr = PolicyEngineClientError( msg='Adding image to policy-engine failed', cause=str(err)) event = events.PolicyEngineLoadAnalysisFailed( user_id=userId, image_digest=imageDigest, error=newerr.to_dict()) analysis_events.append(event) raise newerr logger.debug("updating image catalog record analysis_status") last_analysis_status = image_record['analysis_status'] image_record[ 'analysis_status'] = anchore_engine.subsys.taskstate.complete_state( 'analyze') image_record['analyzed_at'] = int(time.time()) rc = catalog_client.update_image(imageDigest, image_record) try: annotations = {} try: if image_record.get('annotations', '{}'): annotations = json.loads( image_record.get('annotations', '{}')) except Exception as err: logger.warn( "could not marshal annotations from json - exception: " + str(err)) for image_detail in image_record['image_detail']: fulltag = image_detail['registry'] + "/" + image_detail[ 'repo'] + ":" + image_detail['tag'] last_payload = { 'imageDigest': imageDigest, 'analysis_status': last_analysis_status, 'annotations': annotations } curr_payload = { 'imageDigest': imageDigest, 'analysis_status': image_record['analysis_status'], 'annotations': annotations } npayload = { 'last_eval': last_payload, 'curr_eval': curr_payload, } if annotations: npayload['annotations'] = annotations #original method #rc = anchore_engine.subsys.notifications.queue_notification(userId, fulltag, 'analysis_update', npayload) # new method npayload['subscription_type'] = 'analysis_update' event = events.UserAnalyzeImageCompleted( user_id=userId, full_tag=fulltag, data=npayload) analysis_events.append(event) except Exception as err: logger.warn( "failed to enqueue notification on image analysis state update - exception: " + str(err)) else: err = CatalogClientError( msg='Failed to upload analysis data to catalog', cause='Invalid response from catalog API - {}'.format( str(rc))) event = events.SaveAnalysisFailed(user_id=userId, image_digest=imageDigest, error=err.to_dict()) analysis_events.append(event) raise err logger.info("analysis complete: " + str(userId) + " : " + str(imageDigest)) logger.spew("TIMING MARK1: " + str(int(time.time()) - timer)) try: run_time = float(time.time() - timer) anchore_engine.subsys.metrics.histogram_observe( 'anchore_analysis_time_seconds', run_time, buckets=[ 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1800.0, 3600.0 ], status="success") except Exception as err: logger.warn(str(err)) pass except Exception as err: run_time = float(time.time() - timer) logger.exception("problem analyzing image - exception: " + str(err)) anchore_engine.subsys.metrics.histogram_observe( 'anchore_analysis_time_seconds', run_time, buckets=[ 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1800.0, 3600.0 ], status="fail") image_record[ 'analysis_status'] = anchore_engine.subsys.taskstate.fault_state( 'analyze') image_record[ 'image_status'] = anchore_engine.subsys.taskstate.fault_state( 'image_status') rc = catalog_client.update_image(imageDigest, image_record) if userId and imageDigest: for image_detail in image_record['image_detail']: fulltag = image_detail['registry'] + "/" + image_detail[ 'repo'] + ":" + image_detail['tag'] event = events.UserAnalyzeImageFailed(user_id=userId, full_tag=fulltag, error=str(err)) analysis_events.append(event) finally: if analysis_events: for event in analysis_events: try: catalog_client.add_event(event) except: logger.error('Ignoring error sending event') except Exception as err: logger.warn("job processing bailed - exception: " + str(err)) raise err return (True)