def process_upload(upload_data, loop=None): """Parse system upload msg and store it, ASSUMING vmaas-json has changed.""" vmaas_request, repo_list = parse_inventory_data(upload_data) sent = False if vmaas_request: import_status = db_import_system(upload_data, vmaas_request, repo_list) # only give evaluator work if the system's vmaas-call has changed since the last time we did this if ImportStatus.CHANGED in import_status or DISABLE_OPTIMISATION: new_upload_msg = { "type": "upload_new_file", "host": { "id": upload_data["host"]["id"], "account": upload_data["host"]["account"] }, "platform_metadata": { "request_id": upload_data['platform_metadata'].get("request_id") }, "timestamp": upload_data["timestamp"] } EVALUATOR_QUEUE.send(new_upload_msg, loop=loop) LOGGER.info('Sent message to topic %s: %s', mqueue.EVALUATOR_TOPIC, json.dumps(new_upload_msg).encode("utf8")) sent = True elif ImportStatus.FAILED not in import_status: UNCHANGED_SYSTEM.inc() send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, upload_data, 'success', status_msg='unchanged system and not evaluated', loop=loop) return sent
def process_message(message): """Message procession logic""" try: msg_dict = json.loads(message.value.decode('utf-8')) FailedCache.process_failed_cache(FailedCache.upload_cache, executor, self.process_upload_or_re_evaluate, loop) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception("Unable to parse message: ") return if message.topic in kafka_evaluator_topic: if 'type' not in msg_dict: LOGGER.error("Received message is missing type field: %s", msg_dict) return if msg_dict['type'] in ['upload_new_file', 're-evaluate_system']: process_func = self.process_upload_or_re_evaluate if msg_dict['type'] == 'upload_new_file': send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'processing', status_msg='Scheduled for evaluation', loop=loop) else: UNKNOWN_MSG.inc() LOGGER.error("Received unknown message type: %s", msg_dict['type']) return future = executor.submit(process_func, msg_dict, loop=loop) future.add_done_callback(on_thread_done) else: UNKNOWN_TOPIC.inc() LOGGER.error("Received message on unsupported topic: %s", message.topic)
def process_upload_or_re_evaluate(self, msg_dict: dict, loop=None): """ Process function to upload new file or re-evaluate system """ with DatabasePoolConnection() as conn: with conn.cursor() as cur: try: LOGGER.info("Received message type: %s", msg_dict['type']) # Lock the system for processing cur.execute("""SELECT id, inventory_id, vmaas_json, rh_account_id, opt_out FROM system_platform WHERE inventory_id = %s FOR UPDATE""", (msg_dict['host']['id'],)) system_platform = cur.fetchone() if system_platform is not None: self.evaluate_vmaas(system_platform, cur, loop=loop) conn.commit() if msg_dict['type'] == 'upload_new_file': send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'success', loop=loop) else: INV_ID_NOT_FOUND.inc() LOGGER.error("System with inventory_id not found in DB: %s", msg_dict['host']['id']) send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'error', status_msg='System with inventory_id not found in DB: %s' % msg_dict['host']['id'], loop=loop) except DatabaseError: LOGGER.exception("Unable to store data: ") FailedCache.push(FailedCache.upload_cache, msg_dict) LOGGER.info("Remembered failed upload: %s", str(msg_dict)) conn.rollback()
async def db_import_system(msg_dict: dict, rule_hits: dict, advisor_json: str): """Import results from advisor into DB""" status = ImportStatus.FAILED # TODO: insert system into database if it's 1st upload, shall we update last seen? system_data = { 'rh_account': msg_dict['input']['host']['account'], 'display_name': msg_dict['input']['host']['display_name'], 'inventory_id': msg_dict['input']['host']['id'], 'stale_timestamp': msg_dict['input']['host']['stale_timestamp'], 'stale_warning_timestamp': msg_dict['input']['host']['stale_warning_timestamp'], 'culled_timestamp': msg_dict['input']['host']['culled_timestamp'] } async with DB_POOL.acquire() as conn: try: async with conn.transaction(): rh_account_id, system_id, import_status = await db_import_system_platform( conn, system_data, advisor_json) if import_status is None: raise DeletedSystemException() status |= import_status if ImportStatus.CHANGED in status: await db_import_rule_hits(conn, rh_account_id, system_data['inventory_id'], system_id, rule_hits) status -= ImportStatus.FAILED except DeletedSystemException: LOGGER.debug("Skip recently deleted system: %s", system_data['inventory_id']) except Exception: # pylint: disable=broad-except status = ImportStatus.FAILED DATABASE_ERROR.inc() LOGGER.exception("Error importing system: ") send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Error importing system to vulnerability', service=PAYLOAD_TRACKER_SERVICE, loop=asyncio.get_running_loop()) return status
async def process_upload_or_re_evaluate(msg_dict: dict): """ Process function to upload new file or re-evaluate system """ async with DB_POOL.acquire() as conn: try: async with conn.transaction(): LOGGER.info("Received message type: %s", msg_dict['type']) # Lock the system for processing system_platform = await conn.fetchrow( """SELECT id, inventory_id, vmaas_json, rh_account_id, opt_out, stale FROM system_platform WHERE inventory_id = $1 AND when_deleted IS NULL FOR UPDATE""", msg_dict['host']['id']) if system_platform is not None: await evaluate_vmaas(system_platform, conn) if msg_dict['type'] == 'upload_new_file': send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'success', loop=MAIN_LOOP) else: INV_ID_NOT_FOUND.inc() LOGGER.error( "System with inventory_id not found in DB: %s", msg_dict['host']['id']) if msg_dict['type'] == 'upload_new_file': send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict, 'error', status_msg= 'System with inventory_id not found in DB: %s' % msg_dict['host']['id'], loop=MAIN_LOOP) # pylint: disable=broad-except except Exception: LOGGER.exception("Unable to store data: ") FailedCache.push(FailedCache.upload_cache, msg_dict) LOGGER.info("Remembered failed upload: %s", str(msg_dict))
def process_upload(upload_data, loop=None): """Parse system upload msg and store it, ASSUMING vmaas-json has changed.""" sent = False if not validate_system_inventory(upload_data["host"]["id"], upload_data["timestamp"]): LOGGER.info( "Skipping upload, due to system not in inventory anymore, inventory_id: %s", upload_data["host"]["id"]) DELETED_UPLOADED.inc() return sent vmaas_request, repo_list = parse_inventory_data(upload_data) if vmaas_request: import_status = db_import_system(upload_data, vmaas_request, repo_list) # only give evaluator work if the system's vmaas-call has changed since the last time we did this if ImportStatus.CHANGED in import_status or CFG.disable_optimisation: new_upload_msg = { "type": "upload_new_file", "host": { "id": upload_data["host"]["id"], "account": upload_data["host"]["account"] }, "platform_metadata": { "request_id": upload_data['platform_metadata'].get("request_id") }, "timestamp": upload_data["timestamp"] } EVALUATOR_QUEUE.send(new_upload_msg, loop=loop) LOGGER.info('Sent message to topic %s: %s', CFG.evaluator_upload_topic, json.dumps(new_upload_msg).encode("utf8")) sent = True send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, upload_data, "processing_success", status_msg= "system successfully uploaded, sending for evaluation", loop=loop) elif ImportStatus.FAILED not in import_status: UNCHANGED_SYSTEM.inc() send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, upload_data, 'success', status_msg='unchanged system and not evaluated', loop=loop) return sent
async def process_message(message): """Message procession logic""" try: msg_dict = json.loads(message.value.decode('utf-8')) # Can't use FailedCache.process_failed_cache here because it's tied # to ThreadExecutor. So do it the asyncio way if FailedCache.upload_cache: cache = FailedCache.upload_cache LOGGER.info("Start processing %d failed uploads", len(cache)) for msg in cache: LOGGER.info("Processing failed upload: %s", str(msg)) await process_upload_or_re_evaluate(msg) LOGGER.info("Cleared failed cache") FailedCache.clear_cache(cache) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception("Unable to parse message: ") return if message.topic in CFG.evaluator_topics: if 'type' not in msg_dict: LOGGER.error("Received message is missing type field: %s", msg_dict) return if msg_dict['type'] in ['upload_new_file', 're-evaluate_system']: await process_upload_or_re_evaluate(msg_dict) if msg_dict['type'] == 'upload_new_file': send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict, 'processing', status_msg='Scheduled for evaluation', loop=MAIN_LOOP) else: UNKNOWN_MSG.inc() LOGGER.error("Received unknown message type: %s", msg_dict['type']) else: UNKNOWN_TOPIC.inc() LOGGER.error("Received message on unsupported topic: %s", message.topic)
def process_message(msg): # pylint: disable=too-many-return-statements,too-many-branches """Message processing logic""" PROCESS_MESSAGES.inc() LOGGER.info('Received message from topic %s: %s', msg.topic, msg.value) try: msg_dict = json.loads(msg.value.decode("utf8")) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception("Unable to parse message: ") return FailedCache.process_failed_cache(FailedCache.upload_cache, executor, process_upload, loop) FailedCache.process_failed_cache(FailedCache.delete_cache, executor, process_delete, loop) if msg.topic == mqueue.UPLOAD_TOPIC: if not validate_msg(msg_dict, "upload", REQUIRED_UPLOAD_MESSAGE_FIELDS): return # send message to payload tracker send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'received', loop=loop) # proces only archives from smart_management accounts identity = get_identity( msg_dict["platform_metadata"]["b64_identity"]) if identity is None: INVALID_IDENTITY.inc() error_msg = "Skipped upload due to invalid identity header." LOGGER.warning(error_msg) send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'error', status_msg=error_msg, loop=loop) return if not is_entitled_smart_management(identity, allow_missing_section=True): MISSING_SMART_MANAGEMENT.inc() error_msg = "Skipped upload due to missing smart_management entitlement." LOGGER.debug(error_msg) send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'error', status_msg=error_msg, loop=loop) return process_func = process_upload elif msg.topic == mqueue.EVENTS_TOPIC: if not validate_msg(msg_dict, "event", REQUIRED_EVENT_MESSAGE_FIELDS): return if msg_dict['type'] == 'delete': process_func = process_delete else: UNKNOWN_EVENT_TYPE.inc() LOGGER.error("Received unknown event type: %s", msg_dict['type']) return else: UNKNOWN_TOPIC.inc() LOGGER.error("Received message on unsupported topic: %s", msg.topic) return future = executor.submit(process_func, msg_dict, loop=loop) future.add_done_callback(on_thread_done)
async def process_message(msg): """Message processing logic""" PROCESS_MESSAGES.inc() LOGGER.debug('Message from topic %s, body: %s', msg.topic, msg.value) try: msg_dict = json.loads(msg.value.decode('utf8')) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception('Unable to parse message: ') return send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'processing', 'Starting advisor evaluation', service=PAYLOAD_TRACKER_SERVICE, loop=MAIN_LOOP) if not validate_kafka_msg(msg_dict, REQUIRED_MESSAGE_FIELDS): INVALID_INSIGHTS_ACC.inc() send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Skipped advisor result due to message coming from non-insights account.', service=PAYLOAD_TRACKER_SERVICE, loop=MAIN_LOOP) LOGGER.debug( 'Skipped advisor result due to coming from non-insights account.') return identity = get_identity( msg_dict['input']['platform_metadata']['b64_identity']) if identity is None: INVALID_IDENTITY.inc() send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Skipped advisor result due to invalid identity header.', service=PAYLOAD_TRACKER_SERVICE, loop=MAIN_LOOP) LOGGER.debug('Skipped advisor result due to invalid identity header.') return if not is_entitled_insights(identity, allow_missing_section=True): MISSING_INSIGHTS_ENTITLEMENT.inc() send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Skipped advisor result due to missing insights entitlement.', service=PAYLOAD_TRACKER_SERVICE, loop=MAIN_LOOP) LOGGER.debug( 'Skipped advisor result due to missing insights entitlement.') return if not validate_system_inventory(msg_dict["input"]["host"]["id"], msg_dict["input"]["timestamp"]): DELETED_SYSTEM_FROM_INVENTORY.inc() send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Skipped advisor result due to system not valid in inventory anymore.', service=PAYLOAD_TRACKER_SERVICE, loop=MAIN_LOOP) LOGGER.info( 'Skipped advisor result due to system not valid in inventory anymore.' ) return advisor_json, rule_hits = await parse_inventory_data(msg_dict) if advisor_json: LOGGER.info("Evaluating rule hits for inventory_id: %s", msg_dict['input']['host']['id']) status = await db_import_system(msg_dict, rule_hits, advisor_json) if ImportStatus.CHANGED in status: LOGGER.debug("Finished evaluating rule hits for inventory_id: %s", msg_dict['input']['host']['id']) send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'success', 'System successfully uploaded and evaluated', service=PAYLOAD_TRACKER_SERVICE, loop=MAIN_LOOP) elif ImportStatus.FAILED not in status: LOGGER.info( "Skipping evaluating rule hits for inventory_id %s due to unchanged system", msg_dict['input']['host']['id']) UNCHANGED_SYSTEM.inc() send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'success', 'Unchanged system and not evaluated', service=PAYLOAD_TRACKER_SERVICE, loop=MAIN_LOOP)
def process_message(msg): # pylint: disable=too-many-return-statements,too-many-branches """Message processing logic""" PROCESS_MESSAGES.inc() LOGGER.debug('Received message from topic %s: %s', msg.topic, msg.value) try: msg_dict = json.loads(msg.value.decode("utf8")) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception("Unable to parse message: ") return FailedCache.process_failed_cache(FailedCache.upload_cache, ListenerCtx.executor, process_upload, ListenerCtx.loop) FailedCache.process_failed_cache(FailedCache.delete_cache, ListenerCtx.executor, process_delete, ListenerCtx.loop) if msg.topic == CFG.events_topic: if msg_dict.get("type", "") in ["created", "updated"]: if not validate_kafka_msg(msg_dict, REQUIRED_CREATED_UPDATED_MESSAGE_FIELDS): SKIPPED_MESSAGES.inc() return if msg_dict.get("platform_metadata"): if not validate_kafka_msg(msg_dict, REQUIRED_UPLOAD_MESSAGE_FIELDS): SKIPPED_MESSAGES.inc() return LOGGER.info( "Received created/updated msg, inventory_id: %s, type: %s", msg_dict["host"]["id"], msg_dict["type"]) # send message to payload tracker send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'received', loop=ListenerCtx.loop) # process only system uploads from insights entitled accounts identity = get_identity( msg_dict["platform_metadata"]["b64_identity"]) if identity is None: INVALID_IDENTITY.inc() error_msg = "Skipped upload due to invalid identity header." LOGGER.warning(error_msg) send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'error', status_msg=error_msg, loop=ListenerCtx.loop) return if not is_entitled_insights(identity, allow_missing_section=True): MISSING_INSIGHTS_ENTITLEMENT.inc() error_msg = "Skipped upload due to missing insights entitlement." LOGGER.debug(error_msg) send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'error', status_msg=error_msg, loop=ListenerCtx.loop) return process_func = process_upload else: # display name change message doesn't have platform_metadata section, cannot validate identity and track payload, # support only display name change LOGGER.info("Received update event msg, inventory_id: %s", msg_dict["host"]["id"]) process_func = process_update elif msg_dict.get("type", "") == "delete": if not validate_kafka_msg(msg_dict, REQUIRED_DELETE_MESSAGE_FIELDS): SKIPPED_MESSAGES.inc() return LOGGER.info("Received delete msg, inventory_id: %s", msg_dict["id"]) process_func = process_delete else: UNKNOWN_EVENT_TYPE.inc() LOGGER.error("Received unknown event type: %s", msg_dict.get('type', 'missing event type')) return else: UNKNOWN_TOPIC.inc() LOGGER.error("Received message on unsupported topic: %s", msg.topic) return future = ListenerCtx.executor.submit(process_func, msg_dict, loop=ListenerCtx.loop) future.add_done_callback(on_thread_done)
def process_message(msg): """Message processing logic""" PROCESS_MESSAGES.inc() LOGGER.debug('Message from topic %s, body: %s', msg.topic, msg.value) try: msg_dict = json.loads(msg.value.decode('utf8')) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception('Unable to parse message: ') return send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'processing', 'Starting advisor evaluation') if not validate_kafka_msg(msg_dict, REQUIRED_MESSAGE_FIELDS): INVALID_INSIGHTS_ACC.inc() send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Skipped advisor result due to message coming from non-insights account.' ) LOGGER.debug( 'Skipped advisor result due to coming from non-insights account.' ) identity = get_identity( msg_dict['input']['platform_metadata']['b64_identity']) if identity is None: INVALID_IDENTITY.inc() send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Skipped advisor result due to invalid identity header.') LOGGER.debug( 'Skipped advisor result due to invalid identity header.') return if not is_entitled_insights(identity, allow_missing_section=True): MISSING_INSIGHTS_ENTITLEMENT.inc() send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Skipped advisor result due to missing insights entitlement.') LOGGER.debug( 'Skipped advisor result due to missing insights entitlement.') return # TODO: insert system into database if it's 1st upload, shall we update last seen? system_data = { 'rh_account': msg_dict['input']['host']['account'], 'display_name': msg_dict['input']['host']['display_name'], 'inventory_id': msg_dict['input']['host']['id'], 'stale_timestamp': msg_dict['input']['host']['stale_timestamp'], 'stale_warning_timestamp': msg_dict['input']['host']['stale_warning_timestamp'], 'culled_timestamp': msg_dict['input']['host']['culled_timestamp'] } LOGGER.info("Evaluating rule hits for inventory_id: %s", system_data["inventory_id"]) rule_hits = {} reports = msg_dict['results']['reports'] for report in reports: if 'cves' in report['details']: rule = report['rule_id'] if rule in RULE_BLACKLIST: # TODO: remove this once CVE_2017_5753_4_cpu_kernel and CVE_2017_5715_cpu_virt are merged continue if rule not in RULES_CACHE: db_import_rule(rule, list(report['details']['cves'].keys())) for cve in report['details']['cves']: if cve not in CVES_CACHE: db_import_cve(cve) if not report['details']['cves'][ cve]: # False in the CVE dict indicates failed rule rule_hits[CVES_CACHE[cve]] = { 'id': RULES_CACHE[rule], 'details': json.dumps(report['details']), 'cve_name': cve } elif report['details']['cves'][cve]: rule_hits[CVES_CACHE[cve]] = { 'id': RULES_CACHE[rule], 'mitigation_reason': report['details']['cves'][cve] } try: success = db_import_system(system_data, rule_hits, loop) except DatabaseError as exc: success = False # The exception should not get lost raise exc finally: LOGGER.debug("Finished evaluating rule hits for inventory_id: %s", system_data["inventory_id"]) if success: send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'success') else: send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Error importing system to vulnerability')