def process_upload(upload_data, loop=None):
"""Parse system upload msg and store it, ASSUMING vmaas-json has changed."""
vmaas_request, repo_list = parse_inventory_data(upload_data)
sent = False
if vmaas_request:
import_status = db_import_system(upload_data, vmaas_request, repo_list)
# only give evaluator work if the system's vmaas-call has changed since the last time we did this
if ImportStatus.CHANGED in import_status or DISABLE_OPTIMISATION:
new_upload_msg = {
"type": "upload_new_file",
"host": {
"id": upload_data["host"]["id"],
"account": upload_data["host"]["account"]
},
"platform_metadata": {
"request_id":
upload_data['platform_metadata'].get("request_id")
},
"timestamp": upload_data["timestamp"]
}
EVALUATOR_QUEUE.send(new_upload_msg, loop=loop)
LOGGER.info('Sent message to topic %s: %s', mqueue.EVALUATOR_TOPIC,
json.dumps(new_upload_msg).encode("utf8"))
sent = True
elif ImportStatus.FAILED not in import_status:
UNCHANGED_SYSTEM.inc()
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER,
upload_data,
'success',
status_msg='unchanged system and not evaluated',
loop=loop)
return sent
def process_message(message):
"""Message procession logic"""
try:
msg_dict = json.loads(message.value.decode('utf-8'))
FailedCache.process_failed_cache(FailedCache.upload_cache, executor, self.process_upload_or_re_evaluate, loop)
except json.decoder.JSONDecodeError:
MESSAGE_PARSE_ERROR.inc()
LOGGER.exception("Unable to parse message: ")
return
if message.topic in kafka_evaluator_topic:
if 'type' not in msg_dict:
LOGGER.error("Received message is missing type field: %s", msg_dict)
return
if msg_dict['type'] in ['upload_new_file', 're-evaluate_system']:
process_func = self.process_upload_or_re_evaluate
if msg_dict['type'] == 'upload_new_file':
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'processing',
status_msg='Scheduled for evaluation', loop=loop)
else:
UNKNOWN_MSG.inc()
LOGGER.error("Received unknown message type: %s", msg_dict['type'])
return
future = executor.submit(process_func, msg_dict, loop=loop)
future.add_done_callback(on_thread_done)
else:
UNKNOWN_TOPIC.inc()
LOGGER.error("Received message on unsupported topic: %s", message.topic)
def process_upload_or_re_evaluate(self, msg_dict: dict, loop=None):
"""
Process function to upload new file or re-evaluate system
"""
with DatabasePoolConnection() as conn:
with conn.cursor() as cur:
try:
LOGGER.info("Received message type: %s", msg_dict['type'])
# Lock the system for processing
cur.execute("""SELECT id, inventory_id, vmaas_json, rh_account_id,
opt_out FROM system_platform
WHERE inventory_id = %s
FOR UPDATE""", (msg_dict['host']['id'],))
system_platform = cur.fetchone()
if system_platform is not None:
self.evaluate_vmaas(system_platform, cur, loop=loop)
conn.commit()
if msg_dict['type'] == 'upload_new_file':
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'success', loop=loop)
else:
INV_ID_NOT_FOUND.inc()
LOGGER.error("System with inventory_id not found in DB: %s", msg_dict['host']['id'])
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'error',
status_msg='System with inventory_id not found in DB: %s' % msg_dict['host']['id'],
loop=loop)
except DatabaseError:
LOGGER.exception("Unable to store data: ")
FailedCache.push(FailedCache.upload_cache, msg_dict)
LOGGER.info("Remembered failed upload: %s", str(msg_dict))
conn.rollback()
async def db_import_system(msg_dict: dict, rule_hits: dict, advisor_json: str):
"""Import results from advisor into DB"""
status = ImportStatus.FAILED
# TODO: insert system into database if it's 1st upload, shall we update last seen?
system_data = {
'rh_account':
msg_dict['input']['host']['account'],
'display_name':
msg_dict['input']['host']['display_name'],
'inventory_id':
msg_dict['input']['host']['id'],
'stale_timestamp':
msg_dict['input']['host']['stale_timestamp'],
'stale_warning_timestamp':
msg_dict['input']['host']['stale_warning_timestamp'],
'culled_timestamp':
msg_dict['input']['host']['culled_timestamp']
}
async with DB_POOL.acquire() as conn:
try:
async with conn.transaction():
rh_account_id, system_id, import_status = await db_import_system_platform(
conn, system_data, advisor_json)
if import_status is None:
raise DeletedSystemException()
status |= import_status
if ImportStatus.CHANGED in status:
await db_import_rule_hits(conn, rh_account_id,
system_data['inventory_id'],
system_id, rule_hits)
status -= ImportStatus.FAILED
except DeletedSystemException:
LOGGER.debug("Skip recently deleted system: %s",
system_data['inventory_id'])
except Exception: # pylint: disable=broad-except
status = ImportStatus.FAILED
DATABASE_ERROR.inc()
LOGGER.exception("Error importing system: ")
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER,
msg_dict['input'],
'error',
'Error importing system to vulnerability',
service=PAYLOAD_TRACKER_SERVICE,
loop=asyncio.get_running_loop())
return status
async def process_upload_or_re_evaluate(msg_dict: dict):
"""
Process function to upload new file or re-evaluate system
"""
async with DB_POOL.acquire() as conn:
try:
async with conn.transaction():
LOGGER.info("Received message type: %s", msg_dict['type'])
# Lock the system for processing
system_platform = await conn.fetchrow(
"""SELECT id,
inventory_id,
vmaas_json,
rh_account_id,
opt_out,
stale
FROM system_platform
WHERE inventory_id = $1
AND when_deleted IS NULL
FOR UPDATE""",
msg_dict['host']['id'])
if system_platform is not None:
await evaluate_vmaas(system_platform, conn)
if msg_dict['type'] == 'upload_new_file':
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER,
msg_dict,
'success',
loop=MAIN_LOOP)
else:
INV_ID_NOT_FOUND.inc()
LOGGER.error(
"System with inventory_id not found in DB: %s",
msg_dict['host']['id'])
if msg_dict['type'] == 'upload_new_file':
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER,
msg_dict,
'error',
status_msg=
'System with inventory_id not found in DB: %s' %
msg_dict['host']['id'],
loop=MAIN_LOOP)
# pylint: disable=broad-except
except Exception:
LOGGER.exception("Unable to store data: ")
FailedCache.push(FailedCache.upload_cache, msg_dict)
LOGGER.info("Remembered failed upload: %s", str(msg_dict))
def process_upload(upload_data, loop=None):
"""Parse system upload msg and store it, ASSUMING vmaas-json has changed."""
sent = False
if not validate_system_inventory(upload_data["host"]["id"],
upload_data["timestamp"]):
LOGGER.info(
"Skipping upload, due to system not in inventory anymore, inventory_id: %s",
upload_data["host"]["id"])
DELETED_UPLOADED.inc()
return sent
vmaas_request, repo_list = parse_inventory_data(upload_data)
if vmaas_request:
import_status = db_import_system(upload_data, vmaas_request, repo_list)
# only give evaluator work if the system's vmaas-call has changed since the last time we did this
if ImportStatus.CHANGED in import_status or CFG.disable_optimisation:
new_upload_msg = {
"type": "upload_new_file",
"host": {
"id": upload_data["host"]["id"],
"account": upload_data["host"]["account"]
},
"platform_metadata": {
"request_id":
upload_data['platform_metadata'].get("request_id")
},
"timestamp": upload_data["timestamp"]
}
EVALUATOR_QUEUE.send(new_upload_msg, loop=loop)
LOGGER.info('Sent message to topic %s: %s',
CFG.evaluator_upload_topic,
json.dumps(new_upload_msg).encode("utf8"))
sent = True
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER,
upload_data,
"processing_success",
status_msg=
"system successfully uploaded, sending for evaluation",
loop=loop)
elif ImportStatus.FAILED not in import_status:
UNCHANGED_SYSTEM.inc()
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER,
upload_data,
'success',
status_msg='unchanged system and not evaluated',
loop=loop)
return sent
async def process_message(message):
"""Message procession logic"""
try:
msg_dict = json.loads(message.value.decode('utf-8'))
# Can't use FailedCache.process_failed_cache here because it's tied
# to ThreadExecutor. So do it the asyncio way
if FailedCache.upload_cache:
cache = FailedCache.upload_cache
LOGGER.info("Start processing %d failed uploads", len(cache))
for msg in cache:
LOGGER.info("Processing failed upload: %s", str(msg))
await process_upload_or_re_evaluate(msg)
LOGGER.info("Cleared failed cache")
FailedCache.clear_cache(cache)
except json.decoder.JSONDecodeError:
MESSAGE_PARSE_ERROR.inc()
LOGGER.exception("Unable to parse message: ")
return
if message.topic in CFG.evaluator_topics:
if 'type' not in msg_dict:
LOGGER.error("Received message is missing type field: %s",
msg_dict)
return
if msg_dict['type'] in ['upload_new_file', 're-evaluate_system']:
await process_upload_or_re_evaluate(msg_dict)
if msg_dict['type'] == 'upload_new_file':
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER,
msg_dict,
'processing',
status_msg='Scheduled for evaluation',
loop=MAIN_LOOP)
else:
UNKNOWN_MSG.inc()
LOGGER.error("Received unknown message type: %s", msg_dict['type'])
else:
UNKNOWN_TOPIC.inc()
LOGGER.error("Received message on unsupported topic: %s",
message.topic)
def process_message(msg): # pylint: disable=too-many-return-statements,too-many-branches
"""Message processing logic"""
PROCESS_MESSAGES.inc()
LOGGER.info('Received message from topic %s: %s', msg.topic, msg.value)
try:
msg_dict = json.loads(msg.value.decode("utf8"))
except json.decoder.JSONDecodeError:
MESSAGE_PARSE_ERROR.inc()
LOGGER.exception("Unable to parse message: ")
return
FailedCache.process_failed_cache(FailedCache.upload_cache, executor,
process_upload, loop)
FailedCache.process_failed_cache(FailedCache.delete_cache, executor,
process_delete, loop)
if msg.topic == mqueue.UPLOAD_TOPIC:
if not validate_msg(msg_dict, "upload",
REQUIRED_UPLOAD_MESSAGE_FIELDS):
return
# send message to payload tracker
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER,
msg_dict,
'received',
loop=loop)
# proces only archives from smart_management accounts
identity = get_identity(
msg_dict["platform_metadata"]["b64_identity"])
if identity is None:
INVALID_IDENTITY.inc()
error_msg = "Skipped upload due to invalid identity header."
LOGGER.warning(error_msg)
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER,
msg_dict,
'error',
status_msg=error_msg,
loop=loop)
return
if not is_entitled_smart_management(identity,
allow_missing_section=True):
MISSING_SMART_MANAGEMENT.inc()
error_msg = "Skipped upload due to missing smart_management entitlement."
LOGGER.debug(error_msg)
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER,
msg_dict,
'error',
status_msg=error_msg,
loop=loop)
return
process_func = process_upload
elif msg.topic == mqueue.EVENTS_TOPIC:
if not validate_msg(msg_dict, "event",
REQUIRED_EVENT_MESSAGE_FIELDS):
return
if msg_dict['type'] == 'delete':
process_func = process_delete
else:
UNKNOWN_EVENT_TYPE.inc()
LOGGER.error("Received unknown event type: %s",
msg_dict['type'])
return
else:
UNKNOWN_TOPIC.inc()
LOGGER.error("Received message on unsupported topic: %s",
msg.topic)
return
future = executor.submit(process_func, msg_dict, loop=loop)
future.add_done_callback(on_thread_done)
async def process_message(msg):
"""Message processing logic"""
PROCESS_MESSAGES.inc()
LOGGER.debug('Message from topic %s, body: %s', msg.topic, msg.value)
try:
msg_dict = json.loads(msg.value.decode('utf8'))
except json.decoder.JSONDecodeError:
MESSAGE_PARSE_ERROR.inc()
LOGGER.exception('Unable to parse message: ')
return
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER,
msg_dict['input'],
'processing',
'Starting advisor evaluation',
service=PAYLOAD_TRACKER_SERVICE,
loop=MAIN_LOOP)
if not validate_kafka_msg(msg_dict, REQUIRED_MESSAGE_FIELDS):
INVALID_INSIGHTS_ACC.inc()
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER,
msg_dict['input'],
'error',
'Skipped advisor result due to message coming from non-insights account.',
service=PAYLOAD_TRACKER_SERVICE,
loop=MAIN_LOOP)
LOGGER.debug(
'Skipped advisor result due to coming from non-insights account.')
return
identity = get_identity(
msg_dict['input']['platform_metadata']['b64_identity'])
if identity is None:
INVALID_IDENTITY.inc()
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER,
msg_dict['input'],
'error',
'Skipped advisor result due to invalid identity header.',
service=PAYLOAD_TRACKER_SERVICE,
loop=MAIN_LOOP)
LOGGER.debug('Skipped advisor result due to invalid identity header.')
return
if not is_entitled_insights(identity, allow_missing_section=True):
MISSING_INSIGHTS_ENTITLEMENT.inc()
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER,
msg_dict['input'],
'error',
'Skipped advisor result due to missing insights entitlement.',
service=PAYLOAD_TRACKER_SERVICE,
loop=MAIN_LOOP)
LOGGER.debug(
'Skipped advisor result due to missing insights entitlement.')
return
if not validate_system_inventory(msg_dict["input"]["host"]["id"],
msg_dict["input"]["timestamp"]):
DELETED_SYSTEM_FROM_INVENTORY.inc()
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER,
msg_dict['input'],
'error',
'Skipped advisor result due to system not valid in inventory anymore.',
service=PAYLOAD_TRACKER_SERVICE,
loop=MAIN_LOOP)
LOGGER.info(
'Skipped advisor result due to system not valid in inventory anymore.'
)
return
advisor_json, rule_hits = await parse_inventory_data(msg_dict)
if advisor_json:
LOGGER.info("Evaluating rule hits for inventory_id: %s",
msg_dict['input']['host']['id'])
status = await db_import_system(msg_dict, rule_hits, advisor_json)
if ImportStatus.CHANGED in status:
LOGGER.debug("Finished evaluating rule hits for inventory_id: %s",
msg_dict['input']['host']['id'])
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER,
msg_dict['input'],
'success',
'System successfully uploaded and evaluated',
service=PAYLOAD_TRACKER_SERVICE,
loop=MAIN_LOOP)
elif ImportStatus.FAILED not in status:
LOGGER.info(
"Skipping evaluating rule hits for inventory_id %s due to unchanged system",
msg_dict['input']['host']['id'])
UNCHANGED_SYSTEM.inc()
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER,
msg_dict['input'],
'success',
'Unchanged system and not evaluated',
service=PAYLOAD_TRACKER_SERVICE,
loop=MAIN_LOOP)
def process_message(msg): # pylint: disable=too-many-return-statements,too-many-branches
"""Message processing logic"""
PROCESS_MESSAGES.inc()
LOGGER.debug('Received message from topic %s: %s', msg.topic, msg.value)
try:
msg_dict = json.loads(msg.value.decode("utf8"))
except json.decoder.JSONDecodeError:
MESSAGE_PARSE_ERROR.inc()
LOGGER.exception("Unable to parse message: ")
return
FailedCache.process_failed_cache(FailedCache.upload_cache,
ListenerCtx.executor, process_upload,
ListenerCtx.loop)
FailedCache.process_failed_cache(FailedCache.delete_cache,
ListenerCtx.executor, process_delete,
ListenerCtx.loop)
if msg.topic == CFG.events_topic:
if msg_dict.get("type", "") in ["created", "updated"]:
if not validate_kafka_msg(msg_dict,
REQUIRED_CREATED_UPDATED_MESSAGE_FIELDS):
SKIPPED_MESSAGES.inc()
return
if msg_dict.get("platform_metadata"):
if not validate_kafka_msg(msg_dict,
REQUIRED_UPLOAD_MESSAGE_FIELDS):
SKIPPED_MESSAGES.inc()
return
LOGGER.info(
"Received created/updated msg, inventory_id: %s, type: %s",
msg_dict["host"]["id"], msg_dict["type"])
# send message to payload tracker
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER,
msg_dict,
'received',
loop=ListenerCtx.loop)
# process only system uploads from insights entitled accounts
identity = get_identity(
msg_dict["platform_metadata"]["b64_identity"])
if identity is None:
INVALID_IDENTITY.inc()
error_msg = "Skipped upload due to invalid identity header."
LOGGER.warning(error_msg)
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER,
msg_dict,
'error',
status_msg=error_msg,
loop=ListenerCtx.loop)
return
if not is_entitled_insights(identity,
allow_missing_section=True):
MISSING_INSIGHTS_ENTITLEMENT.inc()
error_msg = "Skipped upload due to missing insights entitlement."
LOGGER.debug(error_msg)
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER,
msg_dict,
'error',
status_msg=error_msg,
loop=ListenerCtx.loop)
return
process_func = process_upload
else:
# display name change message doesn't have platform_metadata section, cannot validate identity and track payload,
# support only display name change
LOGGER.info("Received update event msg, inventory_id: %s",
msg_dict["host"]["id"])
process_func = process_update
elif msg_dict.get("type", "") == "delete":
if not validate_kafka_msg(msg_dict,
REQUIRED_DELETE_MESSAGE_FIELDS):
SKIPPED_MESSAGES.inc()
return
LOGGER.info("Received delete msg, inventory_id: %s",
msg_dict["id"])
process_func = process_delete
else:
UNKNOWN_EVENT_TYPE.inc()
LOGGER.error("Received unknown event type: %s",
msg_dict.get('type', 'missing event type'))
return
else:
UNKNOWN_TOPIC.inc()
LOGGER.error("Received message on unsupported topic: %s", msg.topic)
return
future = ListenerCtx.executor.submit(process_func,
msg_dict,
loop=ListenerCtx.loop)
future.add_done_callback(on_thread_done)
def process_message(msg):
"""Message processing logic"""
PROCESS_MESSAGES.inc()
LOGGER.debug('Message from topic %s, body: %s', msg.topic, msg.value)
try:
msg_dict = json.loads(msg.value.decode('utf8'))
except json.decoder.JSONDecodeError:
MESSAGE_PARSE_ERROR.inc()
LOGGER.exception('Unable to parse message: ')
return
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER,
msg_dict['input'], 'processing',
'Starting advisor evaluation')
if not validate_kafka_msg(msg_dict, REQUIRED_MESSAGE_FIELDS):
INVALID_INSIGHTS_ACC.inc()
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error',
'Skipped advisor result due to message coming from non-insights account.'
)
LOGGER.debug(
'Skipped advisor result due to coming from non-insights account.'
)
identity = get_identity(
msg_dict['input']['platform_metadata']['b64_identity'])
if identity is None:
INVALID_IDENTITY.inc()
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error',
'Skipped advisor result due to invalid identity header.')
LOGGER.debug(
'Skipped advisor result due to invalid identity header.')
return
if not is_entitled_insights(identity, allow_missing_section=True):
MISSING_INSIGHTS_ENTITLEMENT.inc()
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error',
'Skipped advisor result due to missing insights entitlement.')
LOGGER.debug(
'Skipped advisor result due to missing insights entitlement.')
return
# TODO: insert system into database if it's 1st upload, shall we update last seen?
system_data = {
'rh_account':
msg_dict['input']['host']['account'],
'display_name':
msg_dict['input']['host']['display_name'],
'inventory_id':
msg_dict['input']['host']['id'],
'stale_timestamp':
msg_dict['input']['host']['stale_timestamp'],
'stale_warning_timestamp':
msg_dict['input']['host']['stale_warning_timestamp'],
'culled_timestamp':
msg_dict['input']['host']['culled_timestamp']
}
LOGGER.info("Evaluating rule hits for inventory_id: %s",
system_data["inventory_id"])
rule_hits = {}
reports = msg_dict['results']['reports']
for report in reports:
if 'cves' in report['details']:
rule = report['rule_id']
if rule in RULE_BLACKLIST:
# TODO: remove this once CVE_2017_5753_4_cpu_kernel and CVE_2017_5715_cpu_virt are merged
continue
if rule not in RULES_CACHE:
db_import_rule(rule,
list(report['details']['cves'].keys()))
for cve in report['details']['cves']:
if cve not in CVES_CACHE:
db_import_cve(cve)
if not report['details']['cves'][
cve]: # False in the CVE dict indicates failed rule
rule_hits[CVES_CACHE[cve]] = {
'id': RULES_CACHE[rule],
'details': json.dumps(report['details']),
'cve_name': cve
}
elif report['details']['cves'][cve]:
rule_hits[CVES_CACHE[cve]] = {
'id': RULES_CACHE[rule],
'mitigation_reason': report['details']['cves'][cve]
}
try:
success = db_import_system(system_data, rule_hits, loop)
except DatabaseError as exc:
success = False
# The exception should not get lost
raise exc
finally:
LOGGER.debug("Finished evaluating rule hits for inventory_id: %s",
system_data["inventory_id"])
if success:
send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER,
msg_dict['input'], 'success')
else:
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER, msg_dict['input'], 'error',
'Error importing system to vulnerability')
