def _validate_set_state(self, topology, instances):
self.assertThat(topology, IsNotEmpty(), 'The topology graph is empty')
host = g_utils.all_matches(
topology,
vitrage_type=NOVA_HOST_DATASOURCE,
id=self._get_hostname(),
vitrage_state=OperationalResourceState.ERROR,
vitrage_aggregated_state=OperationalResourceState.ERROR)
vm1 = g_utils.all_matches(
topology,
vitrage_type=NOVA_INSTANCE_DATASOURCE,
id=instances[0].id,
vitrage_state=OperationalResourceState.SUBOPTIMAL,
vitrage_aggregated_state=OperationalResourceState.SUBOPTIMAL)
vm2 = g_utils.all_matches(
topology,
vitrage_type=NOVA_INSTANCE_DATASOURCE,
id=instances[1].id,
vitrage_state=OperationalResourceState.SUBOPTIMAL,
vitrage_aggregated_state=OperationalResourceState.SUBOPTIMAL)
self.assertThat(host, matchers.HasLength(1))
self.assertThat(vm1, matchers.HasLength(1))
self.assertThat(vm2, matchers.HasLength(1))
def _validate_deduce_alarms(self, alarms, instances):
"""Validate alarm existence """
self.assertThat(alarms, IsNotEmpty(), 'The alarms list is empty')
LOG.debug("The alarms list is : %s", alarms)
# Find the vitrage_id of the deduced alarms using their original id.
vitrage_resources = self.vitrage_client.resource.list(
all_tenants=False)
vitrage_instance_0_id = g_utils.first_match(vitrage_resources,
id=instances[0].id)
vitrage_instance_1_id = g_utils.first_match(vitrage_resources,
id=instances[1].id)
# Find the deduced alarms based on their properties
deduce_alarms_1 = g_utils.all_matches(
alarms,
vitrage_type=VITRAGE_DATASOURCE,
name=VITRAGE_ALARM_NAME,
vitrage_resource_type=NOVA_INSTANCE_DATASOURCE,
vitrage_resource_id=vitrage_instance_0_id[VProps.VITRAGE_ID])
deduce_alarms_2 = g_utils.all_matches(
alarms,
vitrage_type=VITRAGE_DATASOURCE,
name=VITRAGE_ALARM_NAME,
vitrage_resource_type=NOVA_INSTANCE_DATASOURCE,
vitrage_resource_id=vitrage_instance_1_id[VProps.VITRAGE_ID])
self.assertThat(alarms, matchers.HasLength(3),
"Expected 3 alarms - 1 on host and 2 deduced")
self.assertThat(deduce_alarms_1, matchers.HasLength(1),
"Deduced alarm not found")
self.assertThat(deduce_alarms_2, matchers.HasLength(1),
"Deduced alarm not found")
def _validate_passed_templates_length(self, api_templates, cli_templates):
api_passes_templates = g_utils.all_matches(
api_templates,
**{'status details': self.OK_MSG})
cli_passes_templates = cli_templates.count(' ' + self.OK_MSG + ' ')
self.assertThat(api_passes_templates,
matchers.HasLength(cli_passes_templates))
def _check_template_instance_3rd_degree_scenarios(self):
alarm_count = self.vitrage_client.alarm.count(all_tenants=True)
self.assertEqual(CONF.root_cause_analysis_service.instances_per_host,
alarm_count['SEVERE'],
'Each instance should have one SEVERE deduced alarm')
self.assertEqual(
CONF.root_cause_analysis_service.instances_per_host,
alarm_count['CRITICAL'],
'Each instance should have one CRITICAL deduced alarm')
expected_rca = [{VertexProperties.VITRAGE_TYPE: 'zabbix'}] * CONF.\
root_cause_analysis_service.zabbix_alarms_per_host
expected_rca.extend([{'name': DEDUCED_1}, {'name': DEDUCED_2}])
def check_rca(alarm):
rca = self.vitrage_client.rca.get(alarm['vitrage_id'],
all_tenants=True)
try:
self._check_rca(rca, expected_rca, alarm)
return True
except Exception:
LOG.exception('check_rca failed')
return False
# 10 threads calling rca api
alarms = self.vitrage_client.alarm.list(all_tenants=True,
vitrage_id='all')
deduced_alarms = g_utils.all_matches(alarms,
vitrage_type='vitrage',
name=DEDUCED_2)
workers = futures.ThreadPoolExecutor(max_workers=10)
workers_result = [r for r in workers.map(check_rca, deduced_alarms)]
self.assertTrue(all(workers_result))
def _check_deduced(self, deduced_count, deduced_props, resource_id):
alarms = self.vitrage_client.alarm.list(vitrage_id=resource_id,
all_tenants=True)
deduces = g_utils.all_matches(alarms, **deduced_props)
self.assertThat(
deduces, matchers.HasLength(deduced_count),
'Expected %s deduces\n - \n%s\n - \n%s' %
(deduced_count, alarms, deduces))
def _validate_rca(self, rca):
self.assertThat(rca, IsNotEmpty, 'The rca is empty')
LOG.debug("The rca alarms list is : %s", rca)
resource_alarm = g_utils.all_matches(rca,
vitrage_type=AODH_DATASOURCE,
name=RCA_ALARM_NAME)
deduce_alarms = g_utils.all_matches(
rca,
vitrage_type=VITRAGE_DATASOURCE,
name=VITRAGE_ALARM_NAME,
severity=OperationalAlarmSeverity.WARNING)
self.assertThat(rca, matchers.HasLength(3))
self.assertThat(resource_alarm, matchers.HasLength(1))
self.assertThat(deduce_alarms, matchers.HasLength(2))
def delete_all_instances(**kwargs):
instances = TempestClients.nova().servers.list()
instances_to_delete = g_utils.all_matches(instances, **kwargs)
for item in instances_to_delete:
try:
TempestClients.nova().servers.force_delete(item)
except Exception:
LOG.exception('Failed to force delete instance %s', item.id)
wait_for_status(30,
check_deleted_instances,
ids=[instance.id for instance in instances_to_delete])
time.sleep(2)
def _create_alarm(self, resource_id, alarm_name, unic=False):
aodh_utils.create_aodh_alarm(resource_id=resource_id,
name=alarm_name,
unic=unic)
list_alarms = self.vitrage_client.alarm.list(vitrage_id='all',
all_tenants=True)
expected_alarm = g_utils.all_matches(list_alarms,
resource_id=resource_id,
vitrage_type=AODH_DATASOURCE)
if not expected_alarm:
return None
return expected_alarm[0]
def _compare_alarms_lists(self, api_alarms, cli_alarms, resource_type,
resource_id):
"""Validate alarm existence """
self.assertThat(api_alarms, IsNotEmpty(),
'The alarms list taken from api is empty')
self.assertThat(cli_alarms, IsNotEmpty(),
'The alarms list taken from cli is empty')
LOG.info("The alarms list taken from cli is : " + str(cli_alarms))
LOG.info("The alarms list taken by api is : " +
str(json.dumps(api_alarms)))
cli_items = cli_alarms.splitlines()
api_by_type = g_utils.all_matches(api_alarms,
vitrage_type=resource_type)
cli_by_type = cli_alarms.count(' ' + resource_type + ' ')
api_by_id = g_utils.all_matches(api_alarms, resource_id=resource_id)
cli_by_id = cli_alarms.count(resource_id)
self.assertEqual(len(cli_items), len(api_alarms) + 4)
self.assertThat(api_by_type, matchers.HasLength(cli_by_type))
self.assertThat(api_by_id, matchers.HasLength(cli_by_id))
def _check_template_instance_3rd_degree_scenarios(self):
try:
alarm_count = TempestClients.vitrage().alarm.count(
all_tenants=True)
self.assertEqual(
self.conf.mock_graph_datasource.instances_per_host,
alarm_count['SEVERE'],
'Each instance should have one SEVERE deduced alarm')
self.assertEqual(
self.conf.mock_graph_datasource.instances_per_host,
alarm_count['CRITICAL'],
'Each instance should have one CRITICAL deduced alarm')
expected_rca = [{VertexProperties.VITRAGE_TYPE: 'zabbix'}] * self.\
conf.mock_graph_datasource.zabbix_alarms_per_host
expected_rca.extend([{'name': DEDUCED_1}, {'name': DEDUCED_2}])
def check_rca(alarm):
rca = TempestClients.vitrage().rca.get(alarm['vitrage_id'],
all_tenants=True)
try:
self._check_rca(rca, expected_rca, alarm)
return True
except Exception as e:
LOG.exception('check_rca failed', e)
return False
# 10 threads calling rca api
alarms = TempestClients.vitrage().alarm.list(all_tenants=True,
vitrage_id='all')
deduced_alarms = g_utils.all_matches(alarms,
vitrage_type='vitrage',
name=DEDUCED_2)
workers = futures.ThreadPoolExecutor(max_workers=10)
workers_result = [
r for r in workers.map(check_rca, deduced_alarms)
]
self.assertTrue(all(workers_result))
except Exception as e:
v_utils.delete_template(name=TEMPLATE_NAME)
self._handle_exception(e)
raise
def _get_num_default_ports(cls):
ports = TempestClients.neutron().list_ports()['ports']
nova_ports = general_utils.all_matches(ports,
device_owner='compute:nova')
LOG.debug('ports: %s, nova_ports: %s', ports, nova_ports)
return len(nova_ports)
def _port_to_network_edges(self, networks, ports):
counter = 0
for net in networks:
counter += len(g_utils.all_matches(ports, network_id=net['id']))
return counter
def test_overlapping_action_add_causal_relationship(self):
try:
# ---- Do first & second ----
self._trigger_do_action(TRIGGER_ALARM_1)
self._trigger_do_action(TRIGGER_ALARM_2)
alarms = self.vitrage_client.alarm.list(
vitrage_id=self.orig_host.get(VProps.VITRAGE_ID),
all_tenants=True)
deduced = g_utils.first_match(alarms, **DEDUCED_PROPS)
trigger1 = g_utils.first_match(alarms, **TRIGGER_ALARM_1_PROPS)
trigger2 = g_utils.first_match(alarms, **TRIGGER_ALARM_2_PROPS)
# Get Rca for the deduced
rca = self.vitrage_client.rca.get(deduced[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger1, trigger2], DEDUCED_PROPS)
# Get Rca for trigger 1
rca = self.vitrage_client.rca.get(trigger1[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger1], TRIGGER_ALARM_1_PROPS)
# Get Rca for trigger 2
rca = self.vitrage_client.rca.get(trigger2[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger2], TRIGGER_ALARM_2_PROPS)
# ---- Undo - first ----
self._trigger_undo_action(TRIGGER_ALARM_1)
alarms = self.vitrage_client.alarm.list(
vitrage_id=self.orig_host.get(VProps.VITRAGE_ID),
all_tenants=True)
deduced = g_utils.first_match(alarms, **DEDUCED_PROPS)
trigger2 = g_utils.first_match(alarms, **TRIGGER_ALARM_2_PROPS)
# Get Rca for the deduced
rca = self.vitrage_client.rca.get(deduced[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger2], DEDUCED_PROPS)
# Get Rca for trigger 2
rca = self.vitrage_client.rca.get(trigger2[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger2], TRIGGER_ALARM_2_PROPS)
# ---- Undo - second ----
self._trigger_undo_action(TRIGGER_ALARM_2)
alarms = self.vitrage_client.alarm.list(
vitrage_id=self.orig_host.get(VProps.VITRAGE_ID),
all_tenants=True)
self.assertThat(
g_utils.all_matches(alarms, **TRIGGER_ALARM_1_PROPS),
IsEmpty(),
'trigger alarm 1 should have been removed')
self.assertThat(
g_utils.all_matches(alarms, **TRIGGER_ALARM_2_PROPS),
IsEmpty(),
'trigger alarm 2 should have been removed')
self.assertThat(
g_utils.all_matches(alarms, **DEDUCED_PROPS),
IsEmpty(),
'deduced alarm should have been removed')
finally:
self._trigger_undo_action(TRIGGER_ALARM_1)
self._trigger_undo_action(TRIGGER_ALARM_2)
