def process_report(self, host, reports):
"""create/update system based on reports data."""
with app.app_context():
account = get_or_create(db.session,
RhAccount,
'account',
account=host['account'])
state_key = reports[0].get('key')
if len(reports) == 0:
state_key = OPTIMIZED_SYSTEM_KEY
LOG.info(
'There is no ros rule hits. '
"Marking state of system with inventory id: %s as %s",
host['id'], SYSTEM_STATES[state_key])
system = get_or_create(db.session,
System,
'inventory_id',
account_id=account.id,
inventory_id=host['id'],
display_name=host['display_name'],
fqdn=host['fqdn'],
rule_hit_details=reports,
number_of_recommendations=len(reports),
state=SYSTEM_STATES[state_key])
db.session.commit()
LOG.info(
"Refreshed system %s (%s) belonging to account: %s (%s) via engine-result",
system.inventory_id, system.id, account.account, account.id)
def test_identity(mocker):
request = mock.Mock()
request.headers = {
"X-RH-IDENTITY":
"eyJpZGVudGl0eSI6IHsiYWNjb3VudF9udW1iZXIiOiAiMDAwMDAwMSIsICJ0eXB"
"lIjogIlVzZXIiLCAiaW50ZXJuYWwiOiB7Im9yZ19pZCI6ICIwMDAwMDEifX19Cg=="
}
expected_result = {
'identity': {
'account_number': '0000001',
'type': 'User',
'internal': {
'org_id': '000001'
}
}
}
result = utils.identity(request)
assert expected_result == result
with app.app_context():
empty_header_request = mock.Mock()
empty_header_request.headers = {}
mocker.patch('ros.lib.utils.abort')
utils.identity(empty_header_request)
utils.abort.assert_called_once()
def remove_outdated_data(self):
with app.app_context():
results = db.session.query(PerformanceProfile).filter(
PerformanceProfile.report_date < (
datetime.now(timezone.utc) -
timedelta(days=DAYS_UNTIL_STALE))).delete()
db.session.commit()
if results:
LOG.info("Deleted %s performance profiles older than %d days",
results, DAYS_UNTIL_STALE)
time.sleep(GARBAGE_COLLECTION_INTERVAL)
def test_process_report_under_pressure(engine_result_message, engine_consumer, db_setup, performance_record):
engine_result_message = engine_result_message("insights-engine-result-under-pressure.json")
host = engine_result_message["input"]["host"]
ros_reports = [engine_result_message["results"]["reports"][7]]
system_metadata = engine_result_message["results"]["system"]["metadata"]
_performance_record = copy.copy(performance_record)
engine_consumer.process_report(host, ros_reports, system_metadata, performance_record)
system_record = db_get_host(host['id'])
assert str(system_record.inventory_id) == host['id']
with app.app_context():
assert system_record.instance_type == _performance_record['instance_type']
assert system_record.region == _performance_record['region']
assert system_record.state == SYSTEM_STATES['INSTANCE_OPTIMIZED_UNDER_PRESSURE']
assert db.session.query(PerformanceProfile).filter_by(system_id=system_record.id).\
first().performance_record == performance_record
def test_process_report_no_pcp(engine_result_message, engine_consumer, db_setup, performance_record):
engine_result_message = engine_result_message("insights-engine-result-no-pcp.json")
host = engine_result_message["input"]["host"]
ros_reports = [engine_result_message["results"]["reports"][7]]
system_metadata = engine_result_message["results"]["system"]["metadata"]
_performance_record = copy.copy(performance_record)
engine_consumer.process_report(host, ros_reports, system_metadata, performance_record)
data = db_get_host(host['id'])
assert str(data.inventory_id) == host['id']
with app.app_context():
assert data.instance_type == _performance_record['instance_type']
assert data.region == _performance_record['region']
assert data.state == SYSTEM_STATES['NO_PCP_DATA']
assert db.session.query(PerformanceProfile).filter_by(system_id=data.id).first().performance_record ==\
performance_record
def host_delete_event(self, msg):
"""Process delete message."""
self.prefix = "PROCESSING DELETE EVENT"
host_id = msg['id']
insights_id = msg['insights_id']
with app.app_context():
LOG.info(
'Deleting performance profile records with insights_id %s - %s',
insights_id, self.prefix)
rows_deleted = db.session.query(
System.id).filter(System.inventory_id == host_id).delete()
if rows_deleted > 0:
LOG.info('Deleted host from inventory with id: %s - %s',
host_id, self.prefix)
db.session.commit()
def test_process_report_optimized(engine_result_message, engine_consumer, db_setup, performance_record):
engine_result_message = engine_result_message("insights-engine-result-optimized.json")
host = engine_result_message["input"]["host"]
ros_reports = []
system_metadata = engine_result_message["results"]["system"]["metadata"]
_performance_record = copy.copy(performance_record)
engine_consumer.process_report(host, ros_reports, system_metadata, performance_record)
system_record = db_get_host(host['id'])
assert str(system_record.inventory_id) == host['id']
with app.app_context():
assert system_record.rule_hit_details == ros_reports
assert system_record.instance_type == _performance_record['instance_type']
assert system_record.region == _performance_record['region']
assert system_record.state == SYSTEM_STATES['OPTIMIZED']
assert db.session.query(PerformanceProfile).filter_by(system_id=system_record.id).\
first().performance_record == performance_record
def test_process_report_no_pcp(engine_result_message, engine_consumer, db_setup, performance_record):
engine_result_message = engine_result_message("insights-engine-result-no-pcp.json")
host = engine_result_message["input"]["host"]
ros_reports = [engine_result_message["results"]["reports"][7]]
system_metadata = engine_result_message["results"]["system"]["metadata"]
_performance_record = copy.copy(performance_record)
engine_consumer.process_report(host, ros_reports, system_metadata, performance_record)
system_record = db_get_host(host['id'])
performance_utilization = db.session.query(PerformanceProfile).\
filter_by(system_id=system_record.id).first().performance_utilization
sample_performance_util_no_pcp = {'cpu': -1, 'memory': -1, 'max_io': -1.0, 'io': {}}
assert str(system_record.inventory_id) == host['id']
with app.app_context():
assert system_record.instance_type == _performance_record['instance_type']
assert system_record.region == _performance_record['region']
assert system_record.state == SYSTEM_STATES['NO_PCP_DATA']
assert performance_utilization == sample_performance_util_no_pcp
def test_host_create_events(inventory_event_consumer, inventory_event_message,
db_setup, mocker):
mocker.patch.object(
inventory_event_consumer,
'process_system_details',
side_effect=inventory_event_consumer.process_system_details,
autospec=True)
inventory_event_message[
'type'] = 'created' # Setup to meet test case conditions
inventory_event_consumer.host_create_update_events(inventory_event_message)
inventory_event_consumer.process_system_details.assert_called_once()
inventory_event_consumer.process_system_details(
msg=inventory_event_message)
with app.app_context():
assert db_get_host(inventory_event_message['host']['id']).display_name == \
inventory_event_message['host']['display_name']
assert type(db_get_host(
inventory_event_message['host']['id'])).__name__ == 'System'
def host_delete_event(self, msg):
"""Process delete message."""
self.prefix = "INVENTORY DELETE EVENT"
host_id = msg['id']
insights_id = msg['insights_id']
with app.app_context():
LOG.info(
'%s - Deleting performance profile records with insights_id %s',
self.prefix, insights_id)
rows_deleted = db.session.query(
System.id).filter(System.inventory_id == host_id).delete()
db.session.commit()
if rows_deleted > 0:
processor_requests_success.labels(
reporter=self.reporter,
account_number=msg['account']).inc()
LOG.info('%s - Deleted host with inventory id: %s',
self.prefix, host_id)
def database():
DB_USER = os.getenv("ROS_DB_USER", "postgres")
DB_PASSWORD = os.getenv("ROS_DB_PASS", "postgres")
DB_HOST = os.getenv("ROS_DB_HOST", "localhost")
DB_PORT = os.getenv("ROS_DB_PORT", "15432")
DB_NAME = os.getenv("DB_NAME", "ros_db_test")
DB_URI = f"postgresql://{DB_USER}:{DB_PASSWORD}"\
f"@{DB_HOST}:{DB_PORT}/{DB_NAME}"
if not database_exists(DB_URI):
create_database(DB_URI)
app.config['SQLALCHEMY_DATABASE_URI'] = DB_URI
app.testing = True
with app.app_context():
db.create_all()
yield
db.session.remove()
drop_database(DB_URI)
def process_system_details(self, msg):
""" Store new system information (stale, stale_warning timestamp) and return internal DB id"""
host = msg['host']
with app.app_context():
try:
account = get_or_create(db.session,
RhAccount,
'account',
account=host['account'])
system_fields = {
"account_id": account.id,
"inventory_id": host['id'],
"display_name": host['display_name'],
"fqdn": host['fqdn'],
"cloud_provider": host['system_profile']['cloud_provider'],
"stale_timestamp": host['stale_timestamp'],
"operating_system":
host['system_profile']['operating_system'],
}
system = get_or_create(db.session, System, 'inventory_id',
**system_fields)
# Commit changes
db.session.commit()
processor_requests_success.labels(
reporter=self.reporter,
account_number=host['account']).inc()
LOG.info(
"%s - Refreshed system %s (%s) belonging to account: %s (%s).",
self.prefix, system.inventory_id, system.id,
account.account, account.id)
except Exception as err:
processor_requests_failures.labels(
reporter=self.reporter,
account_number=host['account']).inc()
LOG.error(
"%s - Unable to add host %s to DB belonging to account: %s - %s",
self.prefix, host['fqdn'], host['account'], err)
def process_system_details(self, msg):
""" Store new system information (stale, stale_warning timestamp) and return internal DB id"""
host = msg['host']
performance_record = get_performance_profile(
msg['platform_metadata']['url'])
if performance_record:
performance_utilization = self._calculate_performance_utilization(
performance_record, host)
with app.app_context():
account = get_or_create(db.session,
RhAccount,
'account',
account=host['account'])
system = get_or_create(
db.session,
System,
'inventory_id',
account_id=account.id,
inventory_id=host['id'],
display_name=host['display_name'],
fqdn=host['fqdn'],
cloud_provider=host['system_profile']['cloud_provider'],
instance_type=performance_record.get('instance_type'),
stale_timestamp=host['stale_timestamp'])
get_or_create(db.session,
PerformanceProfile, ['system_id', 'report_date'],
system_id=system.id,
performance_record=performance_record,
performance_utilization=performance_utilization,
report_date=datetime.datetime.utcnow().date())
# Commit changes
db.session.commit()
LOG.info(
"Refreshed system %s (%s) belonging to account: %s (%s) via report-processor",
system.inventory_id, system.id, account.account,
account.id)
def test_host_update_events(inventory_event_consumer, inventory_event_message,
db_setup, mocker):
mocker.patch.object(
inventory_event_consumer,
'process_system_details',
side_effect=inventory_event_consumer.process_system_details,
autospec=True)
# Setup to meet test case conditions
inventory_event_message['type'] = 'created'
inventory_event_consumer.host_create_update_events(
inventory_event_message) # creating system for test
inventory_event_message['type'] = 'updated'
inventory_event_message['platform_metadata'] = None
updated_display_name = 'Test - Display Name Update' # Test case change
inventory_event_message['host']['display_name'] = updated_display_name
inventory_event_consumer.host_create_update_events(inventory_event_message)
inventory_event_consumer.process_system_details.call_count = 2
inventory_event_consumer.process_system_details(
msg=inventory_event_message)
with app.app_context():
updated_system = db_get_host(inventory_event_message['host']['id'])
assert updated_system.display_name == updated_display_name
def test_process_system_details(inventory_event_consumer,
inventory_event_message, db_setup):
inventory_event_consumer.process_system_details(inventory_event_message)
with app.app_context():
host = db_get_host(inventory_event_message['host']['id'])
assert str(host.inventory_id) == inventory_event_message['host']['id']
def process_report(self, host, reports, utilization_info, performance_record):
"""create/update system and performance_profile based on reports data."""
with app.app_context():
try:
account = get_or_create(
db.session, RhAccount, 'account',
account=host['account']
)
if len(reports) == 0:
rec_count = len(reports)
state_key = OPTIMIZED_SYSTEM_KEY
LOG.info(
"%s - No ROS rule hits found for system with inventory id: %s. Hence, marking the state as %s.",
self.prefix, host['id'], SYSTEM_STATES[state_key])
else:
state_key = reports[0].get('key')
rec_count = 0 if state_key == 'NO_PCP_DATA' else len(reports)
LOG.info(
"%s - Marking the state of system with inventory id: %s as %s.",
self.prefix, host['id'], SYSTEM_STATES[state_key])
system = get_or_create(
db.session, System, 'inventory_id',
account_id=account.id,
inventory_id=host['id'],
display_name=host['display_name'],
fqdn=host['fqdn'],
rule_hit_details=reports,
number_of_recommendations=-1 if state_key == 'NO_PCP_DATA' else rec_count,
state=SYSTEM_STATES[state_key],
instance_type=performance_record.get('instance_type'),
region=performance_record.get('region')
)
LOG.info(
f"{self.prefix} - System created/updated successfully: {host['id']}"
)
set_default_utilization = False
# For Optimized state, reports would be empty, but utilization_info would be present
if reports and reports[0].get('key') == 'NO_PCP_DATA':
set_default_utilization = True
if set_default_utilization is False:
performance_utilization = {
'memory': int(utilization_info['mem_utilization']),
'cpu': int(utilization_info['cpu_utilization']),
'io': cast_iops_as_float(utilization_info['io_utilization'])
}
# max_io will be used to sort systems endpoint response instead of io
performance_utilization.update(
{'max_io': max(performance_utilization['io'].values())}
)
else:
LOG.debug(f"{self.prefix} - Setting default utilization for performance profile")
performance_utilization = {
'memory': -1,
'cpu': -1,
'max_io': -1.0,
'io': {}
}
# Following are saved on respective system record
del performance_record['instance_type']
del performance_record['region']
get_or_create(
db.session, PerformanceProfile, ['system_id', 'report_date'],
system_id=system.id,
performance_record=performance_record,
performance_utilization=performance_utilization,
report_date=datetime.datetime.utcnow().date()
)
LOG.info(
f"{self.prefix} - Performance profile created/updated successfully for the system: {host['id']}"
)
db.session.commit()
processor_requests_success.labels(
reporter=self.reporter, account_number=host['account']
).inc()
LOG.info("%s - Refreshed system %s (%s) belonging to account: %s (%s).",
self.prefix, system.inventory_id, system.id, account.account, account.id)
except Exception as err:
processor_requests_failures.labels(
reporter=self.reporter, account_number=host['account']
).inc()
LOG.error("%s - Unable to add host %s to DB belonging to account: %s - %s",
self.prefix, host['id'], host['account'], err)
