def start(self):
"""Start websocket server."""
# Sync CVEs always when app starts
self.evaluator_queue = mqueue.MQWriter(mqueue.EVALUATOR_TOPIC)
self.webhook_queue = mqueue.MQWriter(mqueue.WEBHOOKS_TOPIC)
sync_cve_md(self.webhook_queue)
self._websocket_reconnect()
self.reconnect_callback = PeriodicCallback(self._websocket_reconnect,
WEBSOCKET_RECONNECT_INTERVAL * 1000)
self.reconnect_callback.start()
self.instance.start()
def __init__(self):
handlers = [
(r"/api/v1/upload/?", UploadHandler),
(r"/api/v1/download/(.+)", DownloadHandler),
(r"/api/v1/delete/(.+)", DeleteHandler),
]
Application.__init__(self, handlers)
self.instance = IOLoop.instance()
self.upload_queue = mqueue.MQWriter(mqueue.UPLOAD_TOPIC, bootstrap_servers="localhost:9092")
self.events_queue = mqueue.MQWriter(mqueue.EVENTS_TOPIC, bootstrap_servers="localhost:9092")
self.archive_to_facts_cache = {}
def __init__(self):
handlers = [
(r"/api/v1/upload/?", UploadHandler),
(r"/api/v1/download/(.+)", DownloadHandler),
(r"/api/v1/delete/(.+)", DeleteHandler),
(r"/api/rbac/v1/access.+", RbacHandler),
(r"/api/inventory/v1/hosts/(.+)/system_profile", InventoryHandler),
(r"/api/insights/v1/rule/(.+)", InsightsRulesHandler),
]
Application.__init__(self, handlers)
self.instance = IOLoop.instance()
self.upload_queue = mqueue.MQWriter(mqueue.UPLOAD_TOPIC, bootstrap_servers="localhost:9092")
self.events_queue = mqueue.MQWriter(mqueue.EVENTS_TOPIC, bootstrap_servers="localhost:9092")
self.archive_to_profile_cache = {}
self.inventory_id_to_profile_cache = {}
async def start(self, _):
"""Start websocket server."""
# Sync CVEs always when app starts
VmaasSyncContext.evaluator_queue = mqueue.MQWriter(
CFG.evaluator_recalc_topic)
VmaasSyncContext.session = ClientSession()
if CFG.sync_on_start:
await SyncHandler.a_sync_cve_md()
VmaasSyncContext.websocket_task = asyncio.ensure_future(
self._websocket_loop())
def __init__(self, kafka_topics_in):
# connect to the Messaging Service
self.consumer = mqueue.MQReader(kafka_topics_in) # [kafka_evaluator_topic]
LOGGER.info("Using BOOTSTRAP_SERVERS: %s", mqueue.BOOTSTRAP_SERVERS)
LOGGER.info("Using GROUP_ID: %s", mqueue.GROUP_ID)
LOGGER.info("Using TOPICS: %s", ", ".join(kafka_topics_in))
self.producer = mqueue.MQWriter(kafka_evaluator_topic)
# get DB connection
init_db()
self.conn = DatabaseHandler.get_connection()
self.session = requests.Session()
def __init__(self):
handlers = [
(r"/api/v1/upload/?", UploadHandler),
(r"/api/v1/download/(.+)", DownloadHandler),
(r"/api/v1/delete/(.+)", DeleteHandler),
(r"/api/rbac/v1/access.+", RbacHandler),
(r"/api/inventory/v1/hosts/(.+)/system_profile", InventoryHandler),
(r"/api/insights/v1/rule/(.+)", InsightsRulesHandler),
(r"/api/patch/v1/systems/(.+)/advisories", PatchAdvisoriesHandler),
(r"/api/patch/v1/advisories/(.+)/systems", PatchSystemsHandler),
(r"/api/patch/v1/views/systems/advisories", PatchViewsSystemsAdvHandler),
(r"/v1/exploits", ExploitsHandler),
]
Application.__init__(self, handlers)
self.instance = IOLoop.instance()
self.queue = mqueue.MQWriter(mqueue.EVENTS_TOPIC, bootstrap_servers="localhost:9092")
self.archive_to_profile_cache = {}
self.inventory_id_to_profile_cache = {}
'# of upload-msgs missing inventory_id')
ARCHIVE_PARSE_FAILURE = Counter('ve_listener_upl_archive_exceptions',
'# of exceptions during archive-processing')
ARCHIVE_RETRIEVE_ATTEMPT = Counter('ve_listener_upl_archive_tgz_attempt',
'# of times retried archive-retrieval')
ARCHIVE_RETRIEVE_FAILURE = Counter('ve_listener_upl_archive_tgz_failure',
'# of times gave up on archive retrieval')
ARCHIVE_RETRIEVE_INVALID_HTTP = Counter(
've_listener_upl_archive_tgz_invalid_http',
'# archive downloaded with invalid http code')
ARCHIVE_NO_RPMDB = Counter('ve_listener_upl_no_rpmdb',
'# of systems ignored due to missing rpmdb')
# kafka clients
UPLOAD_QUEUE = mqueue.MQReader(mqueue.UPLOAD_TOPIC)
EVALUATOR_QUEUE = mqueue.MQWriter(mqueue.EVALUATOR_TOPIC)
async def terminate(_, loop):
"""Trigger shutdown."""
LOGGER.info("Signal received, stopping kafka consumers.")
await UPLOAD_QUEUE.stop()
await EVALUATOR_QUEUE.stop()
loop.stop()
def on_thread_done(future):
"""Callback to call after ThreadPoolExecutor worker finishes."""
try:
future.result()
except Exception: # pylint: disable=broad-except
NEW_REPO = Counter('ve_listener_upl_new_repo', '# of new repos inserted')
NEW_RH_ACCOUNT = Counter('ve_listener_upl_new_rh_account',
'# of new rh accounts inserted')
NEW_SYSTEM_REPO = Counter('ve_listener_upl_new_system_repo',
'# of new system_repo pairs inserted')
DELETED_SYSTEM_REPO = Counter('ve_listener_upl_system_repo_deleted',
'# deleted system_repo pairs')
INVALID_IDENTITY = Counter('ve_listener_upl_invalid_identity',
'# of skipped uploads because of invalid identity')
MISSING_SMART_MANAGEMENT = Counter(
've_listener_upl_non_smart_management',
'# of skipped uploads because of entitlement check')
# kafka clients
LISTENER_QUEUE = mqueue.MQReader([mqueue.UPLOAD_TOPIC, mqueue.EVENTS_TOPIC])
EVALUATOR_QUEUE = mqueue.MQWriter(mqueue.EVALUATOR_TOPIC)
PAYLOAD_TRACKER_PRODUCER = mqueue.MQWriter(mqueue.PAYLOAD_TRACKER_TOPIC)
# caching repo names to id
REPO_ID_CACHE = {}
REQUIRED_UPLOAD_MESSAGE_FIELDS = {
"host": ["id", "account"],
"platform_metadata": ["b64_identity", "url"],
"timestamp": [],
"type": []
}
REQUIRED_EVENT_MESSAGE_FIELDS = {"id": [], "type": []}
class ImportStatus(flags.Flags):
NEW_RH_ACCOUNT = Counter('ve_advisor_listener_upl_new_rh_account',
'# of new rh accounts inserted')
INVALID_INSIGHTS_ACC = Counter('ve_advisor_listener_invalid_insights_acc',
'# of non-insights messages')
UPLOAD_NO_RESULTS = Counter(
've_advisor_listener_upl_no_result',
'# of systems ignored due to missing reports and passes')
UNCHANGED_SYSTEM = Counter(
've_advisor_listener_upl_unchanged_system',
'# of system-updates with same advisor results info')
DELETED_SYSTEM_FROM_INVENTORY = Counter(
've_advisor_listener_deleted_from_inventory',
'# of systems which are already deleted from inventory')
ADVISOR_QUEUE = mqueue.MQReader([CFG.advisor_results_topic])
REMEDIATIONS_PRODUCER = mqueue.MQWriter(CFG.remediation_updates_topic)
PAYLOAD_TRACKER_PRODUCER = mqueue.MQWriter(CFG.payload_tracker_topic)
RULE_BLACKLIST = [
'CVE_2017_5715_cpu_virt|VIRT_CVE_2017_5715_CPU_3_ONLYKERNEL',
'CVE_2017_5715_cpu_virt'
]
RULES_CACHE = {}
CVES_CACHE = {}
REQUIRED_MESSAGE_FIELDS = {"input": [{"host": [("insights_id", False)]}]}
PAYLOAD_TRACKER_SERVICE = "vulnerability-rules"
DB_POOL: Optional[asyncpg.pool.Pool] = None
MAIN_LOOP = asyncio.get_event_loop()
NEW_REPO = Counter('ve_listener_upl_new_repo', '# of new repos inserted')
NEW_RH_ACCOUNT = Counter('ve_listener_upl_new_rh_account',
'# of new rh accounts inserted')
NEW_SYSTEM_REPO = Counter('ve_listener_upl_new_system_repo',
'# of new system_repo pairs inserted')
DELETED_SYSTEM_REPO = Counter('ve_listener_upl_system_repo_deleted',
'# deleted system_repo pairs')
INVALID_IDENTITY = Counter('ve_listener_upl_invalid_identity',
'# of skipped uploads because of invalid identity')
MISSING_INSIGHTS_ENTITLEMENT = Counter(
've_listener_upl_non_insights_entitlement',
'# of skipped uploads because of entitlement check')
# kafka clients
LISTENER_QUEUE = mqueue.MQReader([CFG.events_topic])
EVALUATOR_QUEUE = mqueue.MQWriter(CFG.evaluator_upload_topic)
PAYLOAD_TRACKER_PRODUCER = mqueue.MQWriter(CFG.payload_tracker_topic)
WORKER_THREADS = CFG.worker_threads or 30
# caching repo names to id
REPO_ID_CACHE = {}
REQUIRED_CREATED_UPDATED_MESSAGE_FIELDS = {
"host": [
"id", "account", "system_profile", "display_name",
("insights_id", False)
],
"timestamp": [],
"type": []
}
MAX_QUEUE_SIZE = int(getenv('MAX_QUEUE_SIZE', '30'))
SYSTEM_DELETION_THRESHOLD = int(getenv('SYSTEM_DELETION_THRESHOLD', '24')) # 24 hours
PROCESS_MESSAGES = Counter('ve_advisor_listener_messages_processed', '# of messages processed')
NEW_SYSTEM = Counter('ve_advisor_listener_upl_new_system', '# of new systems inserted by advisor')
UPDATE_SYSTEM = Counter('ve_advisor_listener_upl_update_system', '# of systems updated')
MESSAGE_PARSE_ERROR = Counter('ve_advisor_listener_message_parse_error', '# of message parse errors')
INVALID_IDENTITY = Counter('ve_advisor_listener_invalid_identity', '# of skipped uploads because of invalid identity')
MISSING_INSIGHTS_ENTITLEMENT = Counter('ve_advisor_listener_non_insights_entitlement', '# of skipped uploads because of entitlement check')
DATABASE_ERROR = Counter('ve_advisor_listener_database_error', '# of database errors')
DELETED_UPLOADED = Counter('ve_advisor_listener_deleted_uploaded', '# of systems uploaded after being deleted')
NEW_RH_ACCOUNT = Counter('ve_advisor_listener_upl_new_rh_account', '# of new rh accounts inserted')
ADVISOR_QUEUE = mqueue.MQReader([mqueue.ADVISOR_RESULTS_TOPIC])
REMEDIATIONS_PRODUCER = mqueue.MQWriter(mqueue.REMEDIATION_UPDATES_TOPIC)
RULE_BLACKLIST = ['CVE_2017_5715_cpu_virt|VIRT_CVE_2017_5715_CPU_3_ONLYKERNEL']
RULES_CACHE = {}
CVES_CACHE = {}
async def terminate(_, loop):
"""Trigger shutdown."""
LOGGER.info('Signal received, stopping kafka consumers.')
await ADVISOR_QUEUE.stop()
await REMEDIATIONS_PRODUCER.stop()
loop.stop()
INVALID_IDENTITY = Counter('ve_advisor_listener_invalid_identity',
'# of skipped uploads because of invalid identity')
MISSING_INSIGHTS_ENTITLEMENT = Counter(
've_advisor_listener_non_insights_entitlement',
'# of skipped uploads because of entitlement check')
DATABASE_ERROR = Counter('ve_advisor_listener_database_error',
'# of database errors')
DELETED_UPLOADED = Counter('ve_advisor_listener_deleted_uploaded',
'# of systems uploaded after being deleted')
NEW_RH_ACCOUNT = Counter('ve_advisor_listener_upl_new_rh_account',
'# of new rh accounts inserted')
INVALID_INSIGHTS_ACC = Counter('ve_advisor_listener_invalid_insights_acc',
'# of non-insights messages')
ADVISOR_QUEUE = mqueue.MQReader([mqueue.ADVISOR_RESULTS_TOPIC])
REMEDIATIONS_PRODUCER = mqueue.MQWriter(mqueue.REMEDIATION_UPDATES_TOPIC)
PAYLOAD_TRACKER_PRODUCER = mqueue.MQWriter(mqueue.PAYLOAD_TRACKER_TOPIC)
RULE_BLACKLIST = ['CVE_2017_5715_cpu_virt|VIRT_CVE_2017_5715_CPU_3_ONLYKERNEL']
RULES_CACHE = {}
CVES_CACHE = {}
REQUIRED_MESSAGE_FIELDS = {"input": [{"host": [("insights_id", False)]}]}
async def terminate(_, loop):
"""Trigger shutdown."""
LOGGER.info('Signal received, stopping kafka consumers.')
await ADVISOR_QUEUE.stop()
await REMEDIATIONS_PRODUCER.stop()
# counts
VMAAS_COUNT = Counter('ve_evaluator_vmaas_calls',
'Number of VMaaS-evaluations attempted')
INV_ID_NOT_FOUND = Counter(
've_evaluator_inventory_not_found',
'Number of times inventory-id not in SystemPlatform')
UNKNOWN_MSG = Counter('ve_evaluator_unknown_msg',
'Number of unrecognized messages delivered from queue')
UNKNOWN_TOPIC = Counter(
've_evaluator_unknown_topic',
'Number of times message delivered from unsupported topic')
MESSAGE_PARSE_ERROR = Counter('ve_evaluator_message_parse_error',
'# of message parse errors')
CONSUMER_QUEUE = mqueue.MQReader(kafka_evaluator_topic)
PAYLOAD_TRACKER_PRODUCER = mqueue.MQWriter(mqueue.PAYLOAD_TRACKER_TOPIC)
async def terminate(_, loop):
"""Trigger shutdown."""
LOGGER.info("Signal received, stopping kafka consumers.")
await CONSUMER_QUEUE.stop()
await PAYLOAD_TRACKER_PRODUCER.stop()
loop.stop()
class QueueEvaluator:
""" This class contains logic for the processing vulnerabilities using VMaaS.
"""
def __init__(self):
LOGGER.info("Using BOOTSTRAP_SERVERS: %s", mqueue.BOOTSTRAP_SERVERS)
# number of worker threads
WORKER_THREADS = int(os.getenv('WORKER_THREADS', '30'))
MAX_QUEUE_SIZE = int(os.getenv('MAX_QUEUE_SIZE', '30'))
# prometheus probes
# times
VMAAS_EVAL_TIME = Histogram('ve_evaluator_vmaas_evaluation_seconds', 'Time spent checking a system for vmaas hits')
# counts
VMAAS_COUNT = Counter('ve_evaluator_vmaas_calls', 'Number of VMaaS-evaluations attempted')
INV_ID_NOT_FOUND = Counter('ve_evaluator_inventory_not_found', 'Number of times inventory-id not in SystemPlatform')
UNKNOWN_MSG = Counter('ve_evaluator_unknown_msg', 'Number of unrecognized messages delivered from queue')
UNKNOWN_TOPIC = Counter('ve_evaluator_unknown_topic', 'Number of times message delivered from unsupported topic')
MESSAGE_PARSE_ERROR = Counter('ve_evaluator_message_parse_error', '# of message parse errors')
CONSUMER_QUEUE = mqueue.MQReader(kafka_evaluator_topic)
WEBHOOKS_QUEUE = mqueue.MQWriter(mqueue.WEBHOOKS_TOPIC)
PAYLOAD_TRACKER_PRODUCER = mqueue.MQWriter(mqueue.PAYLOAD_TRACKER_TOPIC)
async def terminate(_, loop):
"""Trigger shutdown."""
LOGGER.info("Signal received, stopping kafka consumers.")
await CONSUMER_QUEUE.stop()
await WEBHOOKS_QUEUE.stop()
await PAYLOAD_TRACKER_PRODUCER.stop()
loop.stop()
class QueueEvaluator:
""" This class contains logic for the processing vulnerabilities using VMaaS.
"""
prometheus_port = os.getenv('PROMETHEUS_PORT', '8085') # pylint: disable=invalid-name
# number of worker threads
WORKER_THREADS = int(os.getenv('WORKER_THREADS', '30'))
MAX_QUEUE_SIZE = int(os.getenv('MAX_QUEUE_SIZE', '30'))
# prometheus probes
# times
VMAAS_EVAL_TIME = Histogram('ve_evaluator_vmaas_evaluation_seconds', 'Time spent checking a system for vmaas hits')
# counts
VMAAS_COUNT = Counter('ve_evaluator_vmaas_calls', 'Number of VMaaS-evaluations attempted')
INV_ID_NOT_FOUND = Counter('ve_evaluator_inventory_not_found', 'Number of times inventory-id not in SystemPlatform')
UNKNOWN_MSG = Counter('ve_evaluator_unknown_msg', 'Number of unrecognized messages delivered from queue')
UNKNOWN_TOPIC = Counter('ve_evaluator_unknown_topic', 'Number of times message delivered from unsupported topic')
CONSUMER_QUEUE = mqueue.MQReader(kafka_evaluator_topic)
WEBHOOKS_QUEUE = mqueue.MQWriter(mqueue.WEBHOOKS_TOPIC)
async def terminate(_, loop):
"""Trigger shutdown."""
LOGGER.info("Signal received, stopping kafka consumers.")
await CONSUMER_QUEUE.stop()
await WEBHOOKS_QUEUE.stop()
loop.stop()
class QueueEvaluator:
""" This class contains logic for the processing vulnerabilities using VMaaS.
"""
def __init__(self):
