def process_report(self, host, reports): """create/update system based on reports data.""" with app.app_context(): account = get_or_create(db.session, RhAccount, 'account', account=host['account']) state_key = reports[0].get('key') if len(reports) == 0: state_key = OPTIMIZED_SYSTEM_KEY LOG.info( 'There is no ros rule hits. ' "Marking state of system with inventory id: %s as %s", host['id'], SYSTEM_STATES[state_key]) system = get_or_create(db.session, System, 'inventory_id', account_id=account.id, inventory_id=host['id'], display_name=host['display_name'], fqdn=host['fqdn'], rule_hit_details=reports, number_of_recommendations=len(reports), state=SYSTEM_STATES[state_key]) db.session.commit() LOG.info( "Refreshed system %s (%s) belonging to account: %s (%s) via engine-result", system.inventory_id, system.id, account.account, account.id)
def test_identity(mocker): request = mock.Mock() request.headers = { "X-RH-IDENTITY": "eyJpZGVudGl0eSI6IHsiYWNjb3VudF9udW1iZXIiOiAiMDAwMDAwMSIsICJ0eXB" "lIjogIlVzZXIiLCAiaW50ZXJuYWwiOiB7Im9yZ19pZCI6ICIwMDAwMDEifX19Cg==" } expected_result = { 'identity': { 'account_number': '0000001', 'type': 'User', 'internal': { 'org_id': '000001' } } } result = utils.identity(request) assert expected_result == result with app.app_context(): empty_header_request = mock.Mock() empty_header_request.headers = {} mocker.patch('ros.lib.utils.abort') utils.identity(empty_header_request) utils.abort.assert_called_once()
def remove_outdated_data(self): with app.app_context(): results = db.session.query(PerformanceProfile).filter( PerformanceProfile.report_date < ( datetime.now(timezone.utc) - timedelta(days=DAYS_UNTIL_STALE))).delete() db.session.commit() if results: LOG.info("Deleted %s performance profiles older than %d days", results, DAYS_UNTIL_STALE) time.sleep(GARBAGE_COLLECTION_INTERVAL)
def test_process_report_under_pressure(engine_result_message, engine_consumer, db_setup, performance_record): engine_result_message = engine_result_message("insights-engine-result-under-pressure.json") host = engine_result_message["input"]["host"] ros_reports = [engine_result_message["results"]["reports"][7]] system_metadata = engine_result_message["results"]["system"]["metadata"] _performance_record = copy.copy(performance_record) engine_consumer.process_report(host, ros_reports, system_metadata, performance_record) system_record = db_get_host(host['id']) assert str(system_record.inventory_id) == host['id'] with app.app_context(): assert system_record.instance_type == _performance_record['instance_type'] assert system_record.region == _performance_record['region'] assert system_record.state == SYSTEM_STATES['INSTANCE_OPTIMIZED_UNDER_PRESSURE'] assert db.session.query(PerformanceProfile).filter_by(system_id=system_record.id).\ first().performance_record == performance_record
def test_process_report_no_pcp(engine_result_message, engine_consumer, db_setup, performance_record): engine_result_message = engine_result_message("insights-engine-result-no-pcp.json") host = engine_result_message["input"]["host"] ros_reports = [engine_result_message["results"]["reports"][7]] system_metadata = engine_result_message["results"]["system"]["metadata"] _performance_record = copy.copy(performance_record) engine_consumer.process_report(host, ros_reports, system_metadata, performance_record) data = db_get_host(host['id']) assert str(data.inventory_id) == host['id'] with app.app_context(): assert data.instance_type == _performance_record['instance_type'] assert data.region == _performance_record['region'] assert data.state == SYSTEM_STATES['NO_PCP_DATA'] assert db.session.query(PerformanceProfile).filter_by(system_id=data.id).first().performance_record ==\ performance_record
def host_delete_event(self, msg): """Process delete message.""" self.prefix = "PROCESSING DELETE EVENT" host_id = msg['id'] insights_id = msg['insights_id'] with app.app_context(): LOG.info( 'Deleting performance profile records with insights_id %s - %s', insights_id, self.prefix) rows_deleted = db.session.query( System.id).filter(System.inventory_id == host_id).delete() if rows_deleted > 0: LOG.info('Deleted host from inventory with id: %s - %s', host_id, self.prefix) db.session.commit()
def test_process_report_optimized(engine_result_message, engine_consumer, db_setup, performance_record): engine_result_message = engine_result_message("insights-engine-result-optimized.json") host = engine_result_message["input"]["host"] ros_reports = [] system_metadata = engine_result_message["results"]["system"]["metadata"] _performance_record = copy.copy(performance_record) engine_consumer.process_report(host, ros_reports, system_metadata, performance_record) system_record = db_get_host(host['id']) assert str(system_record.inventory_id) == host['id'] with app.app_context(): assert system_record.rule_hit_details == ros_reports assert system_record.instance_type == _performance_record['instance_type'] assert system_record.region == _performance_record['region'] assert system_record.state == SYSTEM_STATES['OPTIMIZED'] assert db.session.query(PerformanceProfile).filter_by(system_id=system_record.id).\ first().performance_record == performance_record
def test_process_report_no_pcp(engine_result_message, engine_consumer, db_setup, performance_record): engine_result_message = engine_result_message("insights-engine-result-no-pcp.json") host = engine_result_message["input"]["host"] ros_reports = [engine_result_message["results"]["reports"][7]] system_metadata = engine_result_message["results"]["system"]["metadata"] _performance_record = copy.copy(performance_record) engine_consumer.process_report(host, ros_reports, system_metadata, performance_record) system_record = db_get_host(host['id']) performance_utilization = db.session.query(PerformanceProfile).\ filter_by(system_id=system_record.id).first().performance_utilization sample_performance_util_no_pcp = {'cpu': -1, 'memory': -1, 'max_io': -1.0, 'io': {}} assert str(system_record.inventory_id) == host['id'] with app.app_context(): assert system_record.instance_type == _performance_record['instance_type'] assert system_record.region == _performance_record['region'] assert system_record.state == SYSTEM_STATES['NO_PCP_DATA'] assert performance_utilization == sample_performance_util_no_pcp
def test_host_create_events(inventory_event_consumer, inventory_event_message, db_setup, mocker): mocker.patch.object( inventory_event_consumer, 'process_system_details', side_effect=inventory_event_consumer.process_system_details, autospec=True) inventory_event_message[ 'type'] = 'created' # Setup to meet test case conditions inventory_event_consumer.host_create_update_events(inventory_event_message) inventory_event_consumer.process_system_details.assert_called_once() inventory_event_consumer.process_system_details( msg=inventory_event_message) with app.app_context(): assert db_get_host(inventory_event_message['host']['id']).display_name == \ inventory_event_message['host']['display_name'] assert type(db_get_host( inventory_event_message['host']['id'])).__name__ == 'System'
def host_delete_event(self, msg): """Process delete message.""" self.prefix = "INVENTORY DELETE EVENT" host_id = msg['id'] insights_id = msg['insights_id'] with app.app_context(): LOG.info( '%s - Deleting performance profile records with insights_id %s', self.prefix, insights_id) rows_deleted = db.session.query( System.id).filter(System.inventory_id == host_id).delete() db.session.commit() if rows_deleted > 0: processor_requests_success.labels( reporter=self.reporter, account_number=msg['account']).inc() LOG.info('%s - Deleted host with inventory id: %s', self.prefix, host_id)
def database(): DB_USER = os.getenv("ROS_DB_USER", "postgres") DB_PASSWORD = os.getenv("ROS_DB_PASS", "postgres") DB_HOST = os.getenv("ROS_DB_HOST", "localhost") DB_PORT = os.getenv("ROS_DB_PORT", "15432") DB_NAME = os.getenv("DB_NAME", "ros_db_test") DB_URI = f"postgresql://{DB_USER}:{DB_PASSWORD}"\ f"@{DB_HOST}:{DB_PORT}/{DB_NAME}" if not database_exists(DB_URI): create_database(DB_URI) app.config['SQLALCHEMY_DATABASE_URI'] = DB_URI app.testing = True with app.app_context(): db.create_all() yield db.session.remove() drop_database(DB_URI)
def process_system_details(self, msg): """ Store new system information (stale, stale_warning timestamp) and return internal DB id""" host = msg['host'] with app.app_context(): try: account = get_or_create(db.session, RhAccount, 'account', account=host['account']) system_fields = { "account_id": account.id, "inventory_id": host['id'], "display_name": host['display_name'], "fqdn": host['fqdn'], "cloud_provider": host['system_profile']['cloud_provider'], "stale_timestamp": host['stale_timestamp'], "operating_system": host['system_profile']['operating_system'], } system = get_or_create(db.session, System, 'inventory_id', **system_fields) # Commit changes db.session.commit() processor_requests_success.labels( reporter=self.reporter, account_number=host['account']).inc() LOG.info( "%s - Refreshed system %s (%s) belonging to account: %s (%s).", self.prefix, system.inventory_id, system.id, account.account, account.id) except Exception as err: processor_requests_failures.labels( reporter=self.reporter, account_number=host['account']).inc() LOG.error( "%s - Unable to add host %s to DB belonging to account: %s - %s", self.prefix, host['fqdn'], host['account'], err)
def process_system_details(self, msg): """ Store new system information (stale, stale_warning timestamp) and return internal DB id""" host = msg['host'] performance_record = get_performance_profile( msg['platform_metadata']['url']) if performance_record: performance_utilization = self._calculate_performance_utilization( performance_record, host) with app.app_context(): account = get_or_create(db.session, RhAccount, 'account', account=host['account']) system = get_or_create( db.session, System, 'inventory_id', account_id=account.id, inventory_id=host['id'], display_name=host['display_name'], fqdn=host['fqdn'], cloud_provider=host['system_profile']['cloud_provider'], instance_type=performance_record.get('instance_type'), stale_timestamp=host['stale_timestamp']) get_or_create(db.session, PerformanceProfile, ['system_id', 'report_date'], system_id=system.id, performance_record=performance_record, performance_utilization=performance_utilization, report_date=datetime.datetime.utcnow().date()) # Commit changes db.session.commit() LOG.info( "Refreshed system %s (%s) belonging to account: %s (%s) via report-processor", system.inventory_id, system.id, account.account, account.id)
def test_host_update_events(inventory_event_consumer, inventory_event_message, db_setup, mocker): mocker.patch.object( inventory_event_consumer, 'process_system_details', side_effect=inventory_event_consumer.process_system_details, autospec=True) # Setup to meet test case conditions inventory_event_message['type'] = 'created' inventory_event_consumer.host_create_update_events( inventory_event_message) # creating system for test inventory_event_message['type'] = 'updated' inventory_event_message['platform_metadata'] = None updated_display_name = 'Test - Display Name Update' # Test case change inventory_event_message['host']['display_name'] = updated_display_name inventory_event_consumer.host_create_update_events(inventory_event_message) inventory_event_consumer.process_system_details.call_count = 2 inventory_event_consumer.process_system_details( msg=inventory_event_message) with app.app_context(): updated_system = db_get_host(inventory_event_message['host']['id']) assert updated_system.display_name == updated_display_name
def test_process_system_details(inventory_event_consumer, inventory_event_message, db_setup): inventory_event_consumer.process_system_details(inventory_event_message) with app.app_context(): host = db_get_host(inventory_event_message['host']['id']) assert str(host.inventory_id) == inventory_event_message['host']['id']
def process_report(self, host, reports, utilization_info, performance_record): """create/update system and performance_profile based on reports data.""" with app.app_context(): try: account = get_or_create( db.session, RhAccount, 'account', account=host['account'] ) if len(reports) == 0: rec_count = len(reports) state_key = OPTIMIZED_SYSTEM_KEY LOG.info( "%s - No ROS rule hits found for system with inventory id: %s. Hence, marking the state as %s.", self.prefix, host['id'], SYSTEM_STATES[state_key]) else: state_key = reports[0].get('key') rec_count = 0 if state_key == 'NO_PCP_DATA' else len(reports) LOG.info( "%s - Marking the state of system with inventory id: %s as %s.", self.prefix, host['id'], SYSTEM_STATES[state_key]) system = get_or_create( db.session, System, 'inventory_id', account_id=account.id, inventory_id=host['id'], display_name=host['display_name'], fqdn=host['fqdn'], rule_hit_details=reports, number_of_recommendations=-1 if state_key == 'NO_PCP_DATA' else rec_count, state=SYSTEM_STATES[state_key], instance_type=performance_record.get('instance_type'), region=performance_record.get('region') ) LOG.info( f"{self.prefix} - System created/updated successfully: {host['id']}" ) set_default_utilization = False # For Optimized state, reports would be empty, but utilization_info would be present if reports and reports[0].get('key') == 'NO_PCP_DATA': set_default_utilization = True if set_default_utilization is False: performance_utilization = { 'memory': int(utilization_info['mem_utilization']), 'cpu': int(utilization_info['cpu_utilization']), 'io': cast_iops_as_float(utilization_info['io_utilization']) } # max_io will be used to sort systems endpoint response instead of io performance_utilization.update( {'max_io': max(performance_utilization['io'].values())} ) else: LOG.debug(f"{self.prefix} - Setting default utilization for performance profile") performance_utilization = { 'memory': -1, 'cpu': -1, 'max_io': -1.0, 'io': {} } # Following are saved on respective system record del performance_record['instance_type'] del performance_record['region'] get_or_create( db.session, PerformanceProfile, ['system_id', 'report_date'], system_id=system.id, performance_record=performance_record, performance_utilization=performance_utilization, report_date=datetime.datetime.utcnow().date() ) LOG.info( f"{self.prefix} - Performance profile created/updated successfully for the system: {host['id']}" ) db.session.commit() processor_requests_success.labels( reporter=self.reporter, account_number=host['account'] ).inc() LOG.info("%s - Refreshed system %s (%s) belonging to account: %s (%s).", self.prefix, system.inventory_id, system.id, account.account, account.id) except Exception as err: processor_requests_failures.labels( reporter=self.reporter, account_number=host['account'] ).inc() LOG.error("%s - Unable to add host %s to DB belonging to account: %s - %s", self.prefix, host['id'], host['account'], err)