def run(self): LOG.info(f"{self.prefix} - Processor is running. Awaiting msgs.") for msg in iter(self): if msg.error(): LOG.error("%s - Consumer error: %s", self.prefix, msg.error()) kafka_failures.labels(reporter=self.reporter).inc() raise KafkaException(msg.error()) try: msg = json.loads(msg.value().decode("utf-8")) self.handle_msg(msg) except json.decoder.JSONDecodeError: kafka_failures.labels(reporter=self.reporter).inc() LOG.error( 'Unable to decode kafka message: %s - %s', msg.value(), self.prefix ) except Exception as err: processor_requests_failures.labels( reporter=self.reporter, account_number=msg['input']['host']['account'] ).inc() LOG.error( 'An error occurred during message processing: %s - %s', repr(err), self.prefix ) finally: self.consumer.commit()
def run(self): """Initialize Consumer.""" for msg in iter(self): if msg.error(): LOG.error("%s - Consumer error: %s", self.prefix, msg.error()) kafka_failures.labels(reporter=self.reporter).inc() raise KafkaException(msg.error()) account = None host_id = None try: msg = json.loads(msg.value().decode("utf-8")) event_type = msg['type'] if event_type == 'delete': account = msg['account'] host_id = msg['id'] else: account = msg['host']['account'] host_id = msg['host']['id'] if event_type in self.event_type_map.keys(): handler = self.event_type_map[event_type] handler(msg) else: LOG.info('%s - Event Handling is not found for event %s', self.prefix, event_type) except json.decoder.JSONDecodeError: kafka_failures.labels(reporter=self.reporter).inc() LOG.error('%s - Unable to decode kafka message: %s', self.prefix, msg.value()) except Exception as err: processor_requests_failures.labels( reporter=self.reporter, account_number=account).inc() LOG.error( '%s - An error occurred during message processing: %s in the system %s created from account: %s', self.prefix, repr(err), host_id, account) finally: self.consumer.commit() LOG.warning("Stopping inventory consumer") self.consumer.close()
def get_performance_profile(report_url, account_number, custom_prefix=prefix): with _download_and_extract_report(report_url, account_number, custom_prefix=custom_prefix) as archive: try: LOG.debug( "%s - Extracting performance profile from the report present at %s.\n", custom_prefix, report_url) broker = run(performance_profile, root=archive.tmp_dir) result = broker[performance_profile] del result["type"] LOG.debug( "%s - Extracted performance profile from the report successfully present at %s.\n", custom_prefix, report_url) return result except Exception as e: processor_requests_failures.labels( reporter='INVENTORY EVENTS', account_number=account_number).inc() LOG.error( "%s - Failed to extract performance_profile from the report present at %s. ERROR - %s\n", custom_prefix, report_url, e)
def process_system_details(self, msg): """ Store new system information (stale, stale_warning timestamp) and return internal DB id""" host = msg['host'] with app.app_context(): try: account = get_or_create(db.session, RhAccount, 'account', account=host['account']) system_fields = { "account_id": account.id, "inventory_id": host['id'], "display_name": host['display_name'], "fqdn": host['fqdn'], "cloud_provider": host['system_profile']['cloud_provider'], "stale_timestamp": host['stale_timestamp'], "operating_system": host['system_profile']['operating_system'], } system = get_or_create(db.session, System, 'inventory_id', **system_fields) # Commit changes db.session.commit() processor_requests_success.labels( reporter=self.reporter, account_number=host['account']).inc() LOG.info( "%s - Refreshed system %s (%s) belonging to account: %s (%s).", self.prefix, system.inventory_id, system.id, account.account, account.id) except Exception as err: processor_requests_failures.labels( reporter=self.reporter, account_number=host['account']).inc() LOG.error( "%s - Unable to add host %s to DB belonging to account: %s - %s", self.prefix, host['fqdn'], host['account'], err)
def process_report(self, host, reports, utilization_info, performance_record): """create/update system and performance_profile based on reports data.""" with app.app_context(): try: account = get_or_create( db.session, RhAccount, 'account', account=host['account'] ) if len(reports) == 0: rec_count = len(reports) state_key = OPTIMIZED_SYSTEM_KEY LOG.info( "%s - No ROS rule hits found for system with inventory id: %s. Hence, marking the state as %s.", self.prefix, host['id'], SYSTEM_STATES[state_key]) else: state_key = reports[0].get('key') rec_count = 0 if state_key == 'NO_PCP_DATA' else len(reports) LOG.info( "%s - Marking the state of system with inventory id: %s as %s.", self.prefix, host['id'], SYSTEM_STATES[state_key]) system = get_or_create( db.session, System, 'inventory_id', account_id=account.id, inventory_id=host['id'], display_name=host['display_name'], fqdn=host['fqdn'], rule_hit_details=reports, number_of_recommendations=-1 if state_key == 'NO_PCP_DATA' else rec_count, state=SYSTEM_STATES[state_key], instance_type=performance_record.get('instance_type'), region=performance_record.get('region') ) LOG.info( f"{self.prefix} - System created/updated successfully: {host['id']}" ) set_default_utilization = False # For Optimized state, reports would be empty, but utilization_info would be present if reports and reports[0].get('key') == 'NO_PCP_DATA': set_default_utilization = True if set_default_utilization is False: performance_utilization = { 'memory': int(utilization_info['mem_utilization']), 'cpu': int(utilization_info['cpu_utilization']), 'io': cast_iops_as_float(utilization_info['io_utilization']) } # max_io will be used to sort systems endpoint response instead of io performance_utilization.update( {'max_io': max(performance_utilization['io'].values())} ) else: LOG.debug(f"{self.prefix} - Setting default utilization for performance profile") performance_utilization = { 'memory': -1, 'cpu': -1, 'max_io': -1.0, 'io': {} } # Following are saved on respective system record del performance_record['instance_type'] del performance_record['region'] get_or_create( db.session, PerformanceProfile, ['system_id', 'report_date'], system_id=system.id, performance_record=performance_record, performance_utilization=performance_utilization, report_date=datetime.datetime.utcnow().date() ) LOG.info( f"{self.prefix} - Performance profile created/updated successfully for the system: {host['id']}" ) db.session.commit() processor_requests_success.labels( reporter=self.reporter, account_number=host['account'] ).inc() LOG.info("%s - Refreshed system %s (%s) belonging to account: %s (%s).", self.prefix, system.inventory_id, system.id, account.account, account.id) except Exception as err: processor_requests_failures.labels( reporter=self.reporter, account_number=host['account'] ).inc() LOG.error("%s - Unable to add host %s to DB belonging to account: %s - %s", self.prefix, host['id'], host['account'], err)