def __init__(self, kafka_topics_in): # connect to the Messaging Service self.consumer = mqueue.MQReader(kafka_topics_in) # [kafka_evaluator_topic] LOGGER.info("Using BOOTSTRAP_SERVERS: %s", mqueue.BOOTSTRAP_SERVERS) LOGGER.info("Using GROUP_ID: %s", mqueue.GROUP_ID) LOGGER.info("Using TOPICS: %s", ", ".join(kafka_topics_in)) self.producer = mqueue.MQWriter(kafka_evaluator_topic) # get DB connection init_db() self.conn = DatabaseHandler.get_connection() self.session = requests.Session()
MISSING_ID = Counter('ve_listener_upl_missing_inventory_id', '# of upload-msgs missing inventory_id') ARCHIVE_PARSE_FAILURE = Counter('ve_listener_upl_archive_exceptions', '# of exceptions during archive-processing') ARCHIVE_RETRIEVE_ATTEMPT = Counter('ve_listener_upl_archive_tgz_attempt', '# of times retried archive-retrieval') ARCHIVE_RETRIEVE_FAILURE = Counter('ve_listener_upl_archive_tgz_failure', '# of times gave up on archive retrieval') ARCHIVE_RETRIEVE_INVALID_HTTP = Counter( 've_listener_upl_archive_tgz_invalid_http', '# archive downloaded with invalid http code') ARCHIVE_NO_RPMDB = Counter('ve_listener_upl_no_rpmdb', '# of systems ignored due to missing rpmdb') # kafka clients UPLOAD_QUEUE = mqueue.MQReader(mqueue.UPLOAD_TOPIC) EVALUATOR_QUEUE = mqueue.MQWriter(mqueue.EVALUATOR_TOPIC) async def terminate(_, loop): """Trigger shutdown.""" LOGGER.info("Signal received, stopping kafka consumers.") await UPLOAD_QUEUE.stop() await EVALUATOR_QUEUE.stop() loop.stop() def on_thread_done(future): """Callback to call after ThreadPoolExecutor worker finishes.""" try: future.result()
'# of message parse errors') NEW_REPO = Counter('ve_listener_upl_new_repo', '# of new repos inserted') NEW_RH_ACCOUNT = Counter('ve_listener_upl_new_rh_account', '# of new rh accounts inserted') NEW_SYSTEM_REPO = Counter('ve_listener_upl_new_system_repo', '# of new system_repo pairs inserted') DELETED_SYSTEM_REPO = Counter('ve_listener_upl_system_repo_deleted', '# deleted system_repo pairs') INVALID_IDENTITY = Counter('ve_listener_upl_invalid_identity', '# of skipped uploads because of invalid identity') MISSING_SMART_MANAGEMENT = Counter( 've_listener_upl_non_smart_management', '# of skipped uploads because of entitlement check') # kafka clients LISTENER_QUEUE = mqueue.MQReader([mqueue.UPLOAD_TOPIC, mqueue.EVENTS_TOPIC]) EVALUATOR_QUEUE = mqueue.MQWriter(mqueue.EVALUATOR_TOPIC) PAYLOAD_TRACKER_PRODUCER = mqueue.MQWriter(mqueue.PAYLOAD_TRACKER_TOPIC) # caching repo names to id REPO_ID_CACHE = {} REQUIRED_UPLOAD_MESSAGE_FIELDS = { "host": ["id", "account"], "platform_metadata": ["b64_identity", "url"], "timestamp": [], "type": [] } REQUIRED_EVENT_MESSAGE_FIELDS = {"id": [], "type": []}
'# of systems uploaded after being deleted') NEW_RH_ACCOUNT = Counter('ve_advisor_listener_upl_new_rh_account', '# of new rh accounts inserted') INVALID_INSIGHTS_ACC = Counter('ve_advisor_listener_invalid_insights_acc', '# of non-insights messages') UPLOAD_NO_RESULTS = Counter( 've_advisor_listener_upl_no_result', '# of systems ignored due to missing reports and passes') UNCHANGED_SYSTEM = Counter( 've_advisor_listener_upl_unchanged_system', '# of system-updates with same advisor results info') DELETED_SYSTEM_FROM_INVENTORY = Counter( 've_advisor_listener_deleted_from_inventory', '# of systems which are already deleted from inventory') ADVISOR_QUEUE = mqueue.MQReader([CFG.advisor_results_topic]) REMEDIATIONS_PRODUCER = mqueue.MQWriter(CFG.remediation_updates_topic) PAYLOAD_TRACKER_PRODUCER = mqueue.MQWriter(CFG.payload_tracker_topic) RULE_BLACKLIST = [ 'CVE_2017_5715_cpu_virt|VIRT_CVE_2017_5715_CPU_3_ONLYKERNEL', 'CVE_2017_5715_cpu_virt' ] RULES_CACHE = {} CVES_CACHE = {} REQUIRED_MESSAGE_FIELDS = {"input": [{"host": [("insights_id", False)]}]} PAYLOAD_TRACKER_SERVICE = "vulnerability-rules" DB_POOL: Optional[asyncpg.pool.Pool] = None
VMAAS_COUNT = Counter('ve_evaluator_vmaas_calls', 'Number of VMaaS-evaluations attempted') INV_ID_NOT_FOUND = Counter( 've_evaluator_inventory_not_found', 'Number of times inventory-id not in SystemPlatform') UNKNOWN_MSG = Counter('ve_evaluator_unknown_msg', 'Number of unrecognized messages delivered from queue') UNKNOWN_TOPIC = Counter( 've_evaluator_unknown_topic', 'Number of times message delivered from unsupported topic') MESSAGE_PARSE_ERROR = Counter('ve_evaluator_message_parse_error', '# of message parse errors') VMAAS_ERRORS_SKIP = Counter('ve_evaluator_vmaas_errors_skip', '# of evaluations skipped due to VMaaS errors') CONSUMER_QUEUE = mqueue.MQReader(CFG.evaluator_topics) PAYLOAD_TRACKER_PRODUCER = mqueue.MQWriter(CFG.payload_tracker_topic) REMEDIATIONS_PRODUCER = mqueue.MQWriter(CFG.remediation_updates_topic) MAIN_LOOP = asyncio.get_event_loop() MAX_MESSAGES_SEMAPHORE = asyncio.BoundedSemaphore( CFG.max_loaded_evaluator_msgs) async def _load_cves_for_inventory_id(rh_account_id, system_id, conn): system_cves_map = {} async for record in conn.cursor( """select cm.id, cm.cve, sv.when_mitigated, sv.mitigation_reason, ir.active, ir.playbook_count from system_vulnerabilities sv join cve_metadata cm on sv.cve_id = cm.id left outer join insights_rule ir on sv.rule_id = ir.id
'# of message parse errors') NEW_REPO = Counter('ve_listener_upl_new_repo', '# of new repos inserted') NEW_RH_ACCOUNT = Counter('ve_listener_upl_new_rh_account', '# of new rh accounts inserted') NEW_SYSTEM_REPO = Counter('ve_listener_upl_new_system_repo', '# of new system_repo pairs inserted') DELETED_SYSTEM_REPO = Counter('ve_listener_upl_system_repo_deleted', '# deleted system_repo pairs') INVALID_IDENTITY = Counter('ve_listener_upl_invalid_identity', '# of skipped uploads because of invalid identity') MISSING_INSIGHTS_ENTITLEMENT = Counter( 've_listener_upl_non_insights_entitlement', '# of skipped uploads because of entitlement check') # kafka clients LISTENER_QUEUE = mqueue.MQReader([CFG.events_topic]) EVALUATOR_QUEUE = mqueue.MQWriter(CFG.evaluator_upload_topic) PAYLOAD_TRACKER_PRODUCER = mqueue.MQWriter(CFG.payload_tracker_topic) WORKER_THREADS = CFG.worker_threads or 30 # caching repo names to id REPO_ID_CACHE = {} REQUIRED_CREATED_UPDATED_MESSAGE_FIELDS = { "host": [ "id", "account", "system_profile", "display_name", ("insights_id", False) ], "timestamp": [], "type": []
'# of system-updates with same rules hit') MESSAGE_PARSE_ERROR = Counter('ve_advisor_listener_message_parse_error', '# of message parse errors') INVALID_IDENTITY = Counter('ve_advisor_listener_invalid_identity', '# of skipped uploads because of invalid identity') MISSING_INSIGHTS_ENTITLEMENT = Counter( 've_advisor_listener_non_insights_entitlement', '# of skipped uploads because of entitlement check') DATABASE_ERROR = Counter('ve_advisor_listener_database_error', '# of database errors') DELETED_UPLOADED = Counter('ve_advisor_listener_deleted_uploaded', '# of systems uploaded after being deleted') NEW_RH_ACCOUNT = Counter('ve_advisor_listener_upl_new_rh_account', '# of new rh accounts inserted') ADVISOR_QUEUE = mqueue.MQReader([mqueue.ADVISOR_RESULTS_TOPIC]) RULE_BLACKLIST = ['CVE_2017_5715_cpu_virt|VIRT_CVE_2017_5715_CPU_3_ONLYKERNEL'] RULES_CACHE = {} CVES_CACHE = {} async def terminate(_, loop): """Trigger shutdown.""" LOGGER.info('Signal received, stopping kafka consumers.') await ADVISOR_QUEUE.stop() loop.stop() def db_import_cve(cve: str):
'Time spent checking a system for vmaas hits') # counts VMAAS_COUNT = Counter('ve_evaluator_vmaas_calls', 'Number of VMaaS-evaluations attempted') INV_ID_NOT_FOUND = Counter( 've_evaluator_inventory_not_found', 'Number of times inventory-id not in SystemPlatform') UNKNOWN_MSG = Counter('ve_evaluator_unknown_msg', 'Number of unrecognized messages delivered from queue') UNKNOWN_TOPIC = Counter( 've_evaluator_unknown_topic', 'Number of times message delivered from unsupported topic') MESSAGE_PARSE_ERROR = Counter('ve_evaluator_message_parse_error', '# of message parse errors') CONSUMER_QUEUE = mqueue.MQReader(kafka_evaluator_topic) PAYLOAD_TRACKER_PRODUCER = mqueue.MQWriter(mqueue.PAYLOAD_TRACKER_TOPIC) async def terminate(_, loop): """Trigger shutdown.""" LOGGER.info("Signal received, stopping kafka consumers.") await CONSUMER_QUEUE.stop() await PAYLOAD_TRACKER_PRODUCER.stop() loop.stop() class QueueEvaluator: """ This class contains logic for the processing vulnerabilities using VMaaS. """ def __init__(self):