def main(): """Main kafka listener entrypoint.""" start_http_server(int(PROMETHEUS_PORT)) init_logging() init_db() LOGGER.info("Starting upload listener.") # get DB connection conn = DatabaseHandler.get_connection() loop = asyncio.get_event_loop() signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: loop.add_signal_handler( sig, lambda sig=sig: loop.create_task(terminate(sig, loop))) executor = BoundedExecutor(MAX_QUEUE_SIZE, max_workers=WORKER_THREADS) def process_message(msg): """Message processing logic""" PROCESS_MESSAGES.inc() LOGGER.info('Received message from topic %s: %s', msg.topic, msg.value) try: msg_dict = json.loads(msg.value.decode("utf8")) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception("Unable to parse message: ") return if msg.topic == mqueue.UPLOAD_TOPIC: process_func = process_upload elif msg.topic == mqueue.EVENTS_TOPIC: if msg_dict['type'] == 'delete': process_func = process_delete else: UNKNOWN_EVENT_TYPE.inc() LOGGER.error("Received unknown event type: %s", msg_dict['type']) return else: UNKNOWN_TOPIC.inc() LOGGER.error("Received message on unsupported topic: %s", msg.topic) return if 'id' not in msg_dict or msg_dict["id"] is None: MISSING_ID.inc() LOGGER.warning( "Unable to process message, inventory ID is missing.") return future = executor.submit(process_func, msg_dict, conn, loop=loop) future.add_done_callback(on_thread_done) LISTENER_QUEUE.listen(process_message) # wait until loop is stopped from terminate callback loop.run_forever() LOGGER.info("Shutting down.") executor.shutdown()
def test_executor_single(self): """Test executor as single task""" executor = BoundedExecutor(1) number = 1 future = executor.submit(TestBoundedExecutor._executor_func_mock, number) res = future.result() assert res == number executor.shutdown()
def run(): """Application entrypoint""" LOGGER.info("Started cacheman job.") conn = get_conn() cur = conn.cursor() current_cache = {} cur.execute("""SELECT rh_account_id, cve_id, systems_affected, systems_status_divergent FROM cve_account_cache""") for rh_account_id, cve_id, systems_affected, systems_status_divergent in cur.fetchall(): current_cache.setdefault(rh_account_id, {})[cve_id] = (systems_affected, systems_status_divergent) cur.execute("""SELECT sp.rh_account_id, a.name, a.cve_cache_from, GREATEST(MAX(sp.last_evaluation), MAX(sp.advisor_evaluated), MAX(sp.when_deleted), MAX(a.last_status_change)) AS last_system_change, a.cve_cache_keepalive, COUNT(*) AS total_systems FROM system_platform sp INNER JOIN rh_account a on sp.rh_account_id = a.id GROUP BY sp.rh_account_id, a.name, a.cve_cache_from, a.cve_cache_keepalive HAVING COUNT(*) >= %s""", (CFG.cache_minimal_account_systems,)) accounts = [(account_id, account_name, cve_cache_from, last_system_change, cve_cache_keepalive) for account_id, account_name, cve_cache_from, last_system_change, cve_cache_keepalive, _ in cur.fetchall() if validate_cve_cache_keepalive(cve_cache_keepalive, 2)] LOGGER.info("Accounts with enabled cache: %s", len(accounts)) accounts_to_refresh = [account for account in accounts if account[3] and (not account[2] or account[3] > account[2])] LOGGER.info("Accounts requiring cache refresh: %s", len(accounts_to_refresh)) # Process accounts in parallel with DatabasePool(CACHE_WORKERS): executor = BoundedExecutor(CACHE_WORKERS, max_workers=CACHE_WORKERS) futures = [] for account_id, account_name, _, _, _ in accounts_to_refresh: futures.append(executor.submit(_materialize_account_cache, account_id, account_name, current_cache)) for future in futures: future.result() executor.shutdown() # Pop out cached accounts after all workers are done for account_id, _, _, _, _ in accounts: current_cache.pop(account_id, None) LOGGER.info("Accounts to disable cache: %s", len(current_cache)) for account_id in current_cache: cur.execute("""DELETE FROM cve_account_cache WHERE rh_account_id = %s""", (account_id,)) cur.execute("""UPDATE rh_account SET cve_cache_from = NULL WHERE id = %s""", (account_id,)) conn.commit() cur.close() conn.close() LOGGER.info("Finished cacheman job.")
def run(self): """ This method evaluates incoming system package profiles using VMaaS :return: """ loop = asyncio.get_event_loop() signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: loop.add_signal_handler( sig, lambda sig=sig: loop.create_task(terminate(sig, loop))) executor = BoundedExecutor(MAX_QUEUE_SIZE, max_workers=WORKER_THREADS) # pylint: disable=too-many-branches def process_message(message): """Message procession logic""" try: msg_dict = json.loads(message.value.decode('utf-8')) FailedCache.process_failed_cache(FailedCache.upload_cache, executor, self.process_upload_or_re_evaluate, loop) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception("Unable to parse message: ") return if message.topic in kafka_evaluator_topic: if 'type' not in msg_dict: LOGGER.error("Received message is missing type field: %s", msg_dict) return if msg_dict['type'] in ['upload_new_file', 're-evaluate_system']: process_func = self.process_upload_or_re_evaluate if msg_dict['type'] == 'upload_new_file': send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'processing', status_msg='Scheduled for evaluation', loop=loop) else: UNKNOWN_MSG.inc() LOGGER.error("Received unknown message type: %s", msg_dict['type']) return future = executor.submit(process_func, msg_dict, loop=loop) future.add_done_callback(on_thread_done) else: UNKNOWN_TOPIC.inc() LOGGER.error("Received message on unsupported topic: %s", message.topic) with DatabasePool(WORKER_THREADS): CONSUMER_QUEUE.listen(process_message) # wait until loop is stopped from terminate callback loop.run_forever() LOGGER.info("Shutting down.") executor.shutdown()
def run(self): """ This method evaluates incoming system package profiles using VMaaS :return: """ loop = asyncio.get_event_loop() signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: loop.add_signal_handler( sig, lambda sig=sig: loop.create_task(terminate(sig, loop))) executor = BoundedExecutor(MAX_QUEUE_SIZE, max_workers=WORKER_THREADS) # pylint: disable=too-many-branches def process_message(message): """Message procession logic""" if message.topic in kafka_evaluator_topic: msg_dict = json.loads(message.value.decode('utf-8')) if 'type' not in msg_dict: LOGGER.error("Received message is missing type field: %s", msg_dict) return if msg_dict['type'] in [ 'upload_new_file', 're-evaluate_system' ]: process_func = self.process_upload_or_re_evaluate else: UNKNOWN_MSG.inc() LOGGER.error("Received unknown message type: %s", msg_dict['type']) return future = executor.submit(process_func, msg_dict, loop=loop) future.add_done_callback(on_thread_done) else: UNKNOWN_TOPIC.inc() LOGGER.error("Received message on unsupported topic: %s", message.topic) with DatabasePool(WORKER_THREADS): CONSUMER_QUEUE.listen(process_message) # wait until loop is stopped from terminate callback loop.run_forever() LOGGER.info("Shutting down.") executor.shutdown()
def test_executor_blocking(self, caplog): """Test executor if is blocking""" executor = BoundedExecutor(1) number = 1 with caplog.at_level(log.INFO): future1 = executor.submit(TestBoundedExecutor._executor_func_mock, number) number = 2 future2 = executor.submit(TestBoundedExecutor._executor_func_mock, number) res1 = future1.result() res2 = future2.result() assert res1 == 1 assert res2 == 2 assert "1" in caplog.messages[0] assert "2" in caplog.messages[1]
def main(): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) executor = BoundedExecutor(MAX_QUEUE_SIZE, max_workers=WORKER_THREADS) bench = Benchmark() def process(msg): executor.submit(process_msg, msg, bench) reader = MQReader(LISTENER_TOPIC, ) reader.listen(process) loop.run_forever() LOGGER.info("Shutting down.")
def main(): # pylint: disable=too-many-statements """Main kafka listener entrypoint.""" start_http_server(int(PROMETHEUS_PORT)) init_logging() LOGGER.info("Starting upload listener.") loop = asyncio.get_event_loop() signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: loop.add_signal_handler( sig, lambda sig=sig: loop.create_task(terminate(sig, loop))) executor = BoundedExecutor(MAX_QUEUE_SIZE, max_workers=WORKER_THREADS) def process_message(msg): # pylint: disable=too-many-return-statements,too-many-branches """Message processing logic""" PROCESS_MESSAGES.inc() LOGGER.info('Received message from topic %s: %s', msg.topic, msg.value) try: msg_dict = json.loads(msg.value.decode("utf8")) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception("Unable to parse message: ") return FailedCache.process_failed_cache(FailedCache.upload_cache, executor, process_upload, loop) FailedCache.process_failed_cache(FailedCache.delete_cache, executor, process_delete, loop) if msg.topic == mqueue.UPLOAD_TOPIC: if not validate_msg(msg_dict, "upload", REQUIRED_UPLOAD_MESSAGE_FIELDS): return # send message to payload tracker send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'received', loop=loop) # proces only archives from smart_management accounts identity = get_identity( msg_dict["platform_metadata"]["b64_identity"]) if identity is None: INVALID_IDENTITY.inc() error_msg = "Skipped upload due to invalid identity header." LOGGER.warning(error_msg) send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'error', status_msg=error_msg, loop=loop) return if not is_entitled_smart_management(identity, allow_missing_section=True): MISSING_SMART_MANAGEMENT.inc() error_msg = "Skipped upload due to missing smart_management entitlement." LOGGER.debug(error_msg) send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'error', status_msg=error_msg, loop=loop) return process_func = process_upload elif msg.topic == mqueue.EVENTS_TOPIC: if not validate_msg(msg_dict, "event", REQUIRED_EVENT_MESSAGE_FIELDS): return if msg_dict['type'] == 'delete': process_func = process_delete else: UNKNOWN_EVENT_TYPE.inc() LOGGER.error("Received unknown event type: %s", msg_dict['type']) return else: UNKNOWN_TOPIC.inc() LOGGER.error("Received message on unsupported topic: %s", msg.topic) return future = executor.submit(process_func, msg_dict, loop=loop) future.add_done_callback(on_thread_done) with DatabasePool(WORKER_THREADS): # prepare repo name to id cache db_init_repo_cache() LISTENER_QUEUE.listen(process_message) # wait until loop is stopped from terminate callback loop.run_forever() LOGGER.info("Shutting down.") executor.shutdown()
def main(): """Main kafka listener entrypoint.""" start_http_server(int(PROMETHEUS_PORT)) init_logging() LOGGER.info("Starting upload listener.") loop = asyncio.get_event_loop() signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: loop.add_signal_handler( sig, lambda sig=sig: loop.create_task(terminate(sig, loop))) executor = BoundedExecutor(MAX_QUEUE_SIZE, max_workers=WORKER_THREADS) def process_message(msg): """Message processing logic""" PROCESS_MESSAGES.inc() LOGGER.info('Received message from topic %s: %s', msg.topic, msg.value) try: msg_dict = json.loads(msg.value.decode("utf8")) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception("Unable to parse message: ") return if msg.topic == mqueue.UPLOAD_TOPIC: # proces only archives from smart_management accounts identity = get_identity(msg_dict.get("b64_identity", "")) if identity is None: INVALID_IDENTITY.inc() LOGGER.warning( "Skipped upload due to invalid identity header.") return if not is_entitled_smart_management(identity, allow_missing_section=True): MISSING_SMART_MANAGEMENT.inc() LOGGER.debug( "Skipped upload due to missing smart_management entitlement." ) return process_func = process_upload elif msg.topic == mqueue.EVENTS_TOPIC: if msg_dict['type'] == 'delete': process_func = process_delete else: UNKNOWN_EVENT_TYPE.inc() LOGGER.error("Received unknown event type: %s", msg_dict['type']) return else: UNKNOWN_TOPIC.inc() LOGGER.error("Received message on unsupported topic: %s", msg.topic) return if 'id' not in msg_dict or msg_dict["id"] is None: MISSING_ID.inc() LOGGER.warning( "Unable to process message, inventory ID is missing.") return future = executor.submit(process_func, msg_dict, loop=loop) future.add_done_callback(on_thread_done) with DatabasePool(WORKER_THREADS): # prepare repo name to id cache db_init_repo_cache() LISTENER_QUEUE.listen(process_message) # wait until loop is stopped from terminate callback loop.run_forever() LOGGER.info("Shutting down.") executor.shutdown()
def set_listener_ctx(): """Setup upload listener context""" ListenerCtx.loop = asyncio.get_event_loop() ListenerCtx.executor = BoundedExecutor(CFG.max_queue_size, max_workers=WORKER_THREADS)
def main(): """Application entrypoint""" start_http_server(int(PROMETHEUS_PORT)) init_logging() LOGGER.info('Starting advisor listener.') loop = asyncio.get_event_loop() signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: loop.add_signal_handler( sig, lambda sig=sig: loop.create_task(terminate(sig, loop))) executor = BoundedExecutor(MAX_QUEUE_SIZE, max_workers=WORKER_THREADS) def process_message(msg): """Message processing logic""" PROCESS_MESSAGES.inc() LOGGER.info('Received message on topic %s', msg.topic) LOGGER.debug('Message body: %s', msg.value) try: msg_dict = json.loads(msg.value.decode('utf8')) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception('Unable to parse message: ') return identity = get_identity( msg_dict['input']['platform_metadata']['b64_identity']) if identity is None: INVALID_IDENTITY.inc() LOGGER.warning( 'Skipped advisor result due to invalid identity header.') return if not is_entitled_insights(identity, allow_missing_section=True): MISSING_INSIGHTS_ENTITLEMENT.inc() LOGGER.debug( 'Skipped advisor result due to missing insights entitlement.') return # TODO: insert system into database if it's 1st upload, shall we update last seen? system_data = { 'rh_account': msg_dict['input']['host']['account'], 'display_name': msg_dict['input']['host']['display_name'], 'inventory_id': msg_dict['input']['host']['id'] } rule_hits = {} reports = msg_dict['results']['reports'] for report in reports: if 'cves' in report['details']: rule = report['rule_id'] if rule in RULE_BLACKLIST: # TODO: remove this once CVE_2017_5753_4_cpu_kernel and CVE_2017_5715_cpu_virt are merged continue if rule not in RULES_CACHE: db_import_rule(rule, list(report['details']['cves'].keys())) for cve in report['details']['cves']: if cve not in CVES_CACHE: db_import_cve(cve) if not report['details']['cves'][ cve]: # False in the CVE dict indicates failed rule rule_hits[CVES_CACHE[cve]] = { 'id': RULES_CACHE[rule], 'details': json.dumps(report['details']) } db_import_system(system_data, rule_hits) with DatabasePool(WORKER_THREADS): db_init_caches() ADVISOR_QUEUE.listen(process_message) loop.run_forever() LOGGER.info('Shutting down.') executor.shutdown()
def main(): # pylint: disable=too-many-branches, too-many-statements """Application entrypoint""" start_http_server(int(PROMETHEUS_PORT)) init_logging() ensure_minimal_schema_version() LOGGER.info('Starting advisor listener.') loop = asyncio.get_event_loop() signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: loop.add_signal_handler( sig, lambda sig=sig: loop.create_task(terminate(sig, loop))) executor = BoundedExecutor(MAX_QUEUE_SIZE, max_workers=WORKER_THREADS) def process_message(msg): """Message processing logic""" PROCESS_MESSAGES.inc() LOGGER.debug('Message from topic %s, body: %s', msg.topic, msg.value) try: msg_dict = json.loads(msg.value.decode('utf8')) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception('Unable to parse message: ') return send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'processing', 'Starting advisor evaluation') if not validate_kafka_msg(msg_dict, REQUIRED_MESSAGE_FIELDS): INVALID_INSIGHTS_ACC.inc() send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Skipped advisor result due to message coming from non-insights account.' ) LOGGER.debug( 'Skipped advisor result due to coming from non-insights account.' ) identity = get_identity( msg_dict['input']['platform_metadata']['b64_identity']) if identity is None: INVALID_IDENTITY.inc() send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Skipped advisor result due to invalid identity header.') LOGGER.debug( 'Skipped advisor result due to invalid identity header.') return if not is_entitled_insights(identity, allow_missing_section=True): MISSING_INSIGHTS_ENTITLEMENT.inc() send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Skipped advisor result due to missing insights entitlement.') LOGGER.debug( 'Skipped advisor result due to missing insights entitlement.') return # TODO: insert system into database if it's 1st upload, shall we update last seen? system_data = { 'rh_account': msg_dict['input']['host']['account'], 'display_name': msg_dict['input']['host']['display_name'], 'inventory_id': msg_dict['input']['host']['id'], 'stale_timestamp': msg_dict['input']['host']['stale_timestamp'], 'stale_warning_timestamp': msg_dict['input']['host']['stale_warning_timestamp'], 'culled_timestamp': msg_dict['input']['host']['culled_timestamp'] } LOGGER.info("Evaluating rule hits for inventory_id: %s", system_data["inventory_id"]) rule_hits = {} reports = msg_dict['results']['reports'] for report in reports: if 'cves' in report['details']: rule = report['rule_id'] if rule in RULE_BLACKLIST: # TODO: remove this once CVE_2017_5753_4_cpu_kernel and CVE_2017_5715_cpu_virt are merged continue if rule not in RULES_CACHE: db_import_rule(rule, list(report['details']['cves'].keys())) for cve in report['details']['cves']: if cve not in CVES_CACHE: db_import_cve(cve) if not report['details']['cves'][ cve]: # False in the CVE dict indicates failed rule rule_hits[CVES_CACHE[cve]] = { 'id': RULES_CACHE[rule], 'details': json.dumps(report['details']), 'cve_name': cve } elif report['details']['cves'][cve]: rule_hits[CVES_CACHE[cve]] = { 'id': RULES_CACHE[rule], 'mitigation_reason': report['details']['cves'][cve] } try: success = db_import_system(system_data, rule_hits, loop) except DatabaseError as exc: success = False # The exception should not get lost raise exc finally: LOGGER.debug("Finished evaluating rule hits for inventory_id: %s", system_data["inventory_id"]) if success: send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'success') else: send_msg_to_payload_tracker( PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error', 'Error importing system to vulnerability') with DatabasePool(WORKER_THREADS): db_init_caches() ADVISOR_QUEUE.listen(process_message) loop.run_forever() LOGGER.info('Shutting down.') executor.shutdown()