def run(self):
LOG.info(f"{self.prefix} - Processor is running. Awaiting msgs.")
for msg in iter(self):
if msg.error():
LOG.error("%s - Consumer error: %s", self.prefix, msg.error())
kafka_failures.labels(reporter=self.reporter).inc()
raise KafkaException(msg.error())
try:
msg = json.loads(msg.value().decode("utf-8"))
self.handle_msg(msg)
except json.decoder.JSONDecodeError:
kafka_failures.labels(reporter=self.reporter).inc()
LOG.error(
'Unable to decode kafka message: %s - %s',
msg.value(), self.prefix
)
except Exception as err:
processor_requests_failures.labels(
reporter=self.reporter,
account_number=msg['input']['host']['account']
).inc()
LOG.error(
'An error occurred during message processing: %s - %s',
repr(err),
self.prefix
)
finally:
self.consumer.commit()
def run(self):
"""Initialize Consumer."""
for msg in iter(self):
if msg.error():
LOG.error("%s - Consumer error: %s", self.prefix, msg.error())
kafka_failures.labels(reporter=self.reporter).inc()
raise KafkaException(msg.error())
account = None
host_id = None
try:
msg = json.loads(msg.value().decode("utf-8"))
event_type = msg['type']
if event_type == 'delete':
account = msg['account']
host_id = msg['id']
else:
account = msg['host']['account']
host_id = msg['host']['id']
if event_type in self.event_type_map.keys():
handler = self.event_type_map[event_type]
handler(msg)
else:
LOG.info('%s - Event Handling is not found for event %s',
self.prefix, event_type)
except json.decoder.JSONDecodeError:
kafka_failures.labels(reporter=self.reporter).inc()
LOG.error('%s - Unable to decode kafka message: %s',
self.prefix, msg.value())
except Exception as err:
processor_requests_failures.labels(
reporter=self.reporter, account_number=account).inc()
LOG.error(
'%s - An error occurred during message processing: %s in the system %s created from account: %s',
self.prefix, repr(err), host_id, account)
finally:
self.consumer.commit()
LOG.warning("Stopping inventory consumer")
self.consumer.close()
def get_performance_profile(report_url, account_number, custom_prefix=prefix):
with _download_and_extract_report(report_url,
account_number,
custom_prefix=custom_prefix) as archive:
try:
LOG.debug(
"%s - Extracting performance profile from the report present at %s.\n",
custom_prefix, report_url)
broker = run(performance_profile, root=archive.tmp_dir)
result = broker[performance_profile]
del result["type"]
LOG.debug(
"%s - Extracted performance profile from the report successfully present at %s.\n",
custom_prefix, report_url)
return result
except Exception as e:
processor_requests_failures.labels(
reporter='INVENTORY EVENTS',
account_number=account_number).inc()
LOG.error(
"%s - Failed to extract performance_profile from the report present at %s. ERROR - %s\n",
custom_prefix, report_url, e)
def process_system_details(self, msg):
""" Store new system information (stale, stale_warning timestamp) and return internal DB id"""
host = msg['host']
with app.app_context():
try:
account = get_or_create(db.session,
RhAccount,
'account',
account=host['account'])
system_fields = {
"account_id": account.id,
"inventory_id": host['id'],
"display_name": host['display_name'],
"fqdn": host['fqdn'],
"cloud_provider": host['system_profile']['cloud_provider'],
"stale_timestamp": host['stale_timestamp'],
"operating_system":
host['system_profile']['operating_system'],
}
system = get_or_create(db.session, System, 'inventory_id',
**system_fields)
# Commit changes
db.session.commit()
processor_requests_success.labels(
reporter=self.reporter,
account_number=host['account']).inc()
LOG.info(
"%s - Refreshed system %s (%s) belonging to account: %s (%s).",
self.prefix, system.inventory_id, system.id,
account.account, account.id)
except Exception as err:
processor_requests_failures.labels(
reporter=self.reporter,
account_number=host['account']).inc()
LOG.error(
"%s - Unable to add host %s to DB belonging to account: %s - %s",
self.prefix, host['fqdn'], host['account'], err)
def process_report(self, host, reports, utilization_info, performance_record):
"""create/update system and performance_profile based on reports data."""
with app.app_context():
try:
account = get_or_create(
db.session, RhAccount, 'account',
account=host['account']
)
if len(reports) == 0:
rec_count = len(reports)
state_key = OPTIMIZED_SYSTEM_KEY
LOG.info(
"%s - No ROS rule hits found for system with inventory id: %s. Hence, marking the state as %s.",
self.prefix, host['id'], SYSTEM_STATES[state_key])
else:
state_key = reports[0].get('key')
rec_count = 0 if state_key == 'NO_PCP_DATA' else len(reports)
LOG.info(
"%s - Marking the state of system with inventory id: %s as %s.",
self.prefix, host['id'], SYSTEM_STATES[state_key])
system = get_or_create(
db.session, System, 'inventory_id',
account_id=account.id,
inventory_id=host['id'],
display_name=host['display_name'],
fqdn=host['fqdn'],
rule_hit_details=reports,
number_of_recommendations=-1 if state_key == 'NO_PCP_DATA' else rec_count,
state=SYSTEM_STATES[state_key],
instance_type=performance_record.get('instance_type'),
region=performance_record.get('region')
)
LOG.info(
f"{self.prefix} - System created/updated successfully: {host['id']}"
)
set_default_utilization = False
# For Optimized state, reports would be empty, but utilization_info would be present
if reports and reports[0].get('key') == 'NO_PCP_DATA':
set_default_utilization = True
if set_default_utilization is False:
performance_utilization = {
'memory': int(utilization_info['mem_utilization']),
'cpu': int(utilization_info['cpu_utilization']),
'io': cast_iops_as_float(utilization_info['io_utilization'])
}
# max_io will be used to sort systems endpoint response instead of io
performance_utilization.update(
{'max_io': max(performance_utilization['io'].values())}
)
else:
LOG.debug(f"{self.prefix} - Setting default utilization for performance profile")
performance_utilization = {
'memory': -1,
'cpu': -1,
'max_io': -1.0,
'io': {}
}
# Following are saved on respective system record
del performance_record['instance_type']
del performance_record['region']
get_or_create(
db.session, PerformanceProfile, ['system_id', 'report_date'],
system_id=system.id,
performance_record=performance_record,
performance_utilization=performance_utilization,
report_date=datetime.datetime.utcnow().date()
)
LOG.info(
f"{self.prefix} - Performance profile created/updated successfully for the system: {host['id']}"
)
db.session.commit()
processor_requests_success.labels(
reporter=self.reporter, account_number=host['account']
).inc()
LOG.info("%s - Refreshed system %s (%s) belonging to account: %s (%s).",
self.prefix, system.inventory_id, system.id, account.account, account.id)
except Exception as err:
processor_requests_failures.labels(
reporter=self.reporter, account_number=host['account']
).inc()
LOG.error("%s - Unable to add host %s to DB belonging to account: %s - %s",
self.prefix, host['id'], host['account'], err)
