def test_action_add_causal_relationship(self):
try:
# Do
self._trigger_do_action(TRIGGER_ALARM_2)
alarms = TempestClients.vitrage().alarm.list(
vitrage_id=self.orig_host.get(VProps.VITRAGE_ID),
all_tenants=True)
self.assertEqual(True, len(alarms) >= 2, 'alarms %s' % str(alarms))
deduced = g_utils.first_match(alarms, **DEDUCED_PROPS)
trigger = g_utils.first_match(alarms, **TRIGGER_ALARM_2_PROPS)
# Get Rca for the deduced
rca = TempestClients.vitrage().rca.get(deduced[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger], DEDUCED_PROPS)
# Get Rca for the trigger
rca = TempestClients.vitrage().rca.get(trigger[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger], TRIGGER_ALARM_2_PROPS)
except Exception as e:
self._handle_exception(e)
raise
finally:
self._trigger_undo_action(TRIGGER_ALARM_2)
def test_overlapping_action_mark_down(self):
try:
host_name = self.orig_host.get(VProps.NAME)
# Do - first
self._trigger_do_action(TRIGGER_ALARM_3)
nova_service = TempestClients.nova().services.list(
host=host_name, binary='nova-compute')[0]
self.assertEqual("down", str(nova_service.state))
# Do - second
self._trigger_do_action(TRIGGER_ALARM_4)
nova_service = TempestClients.nova().services.list(
host=host_name, binary='nova-compute')[0]
self.assertEqual("down", str(nova_service.state))
# Undo - first
self._trigger_undo_action(TRIGGER_ALARM_3)
nova_service = TempestClients.nova().services.list(
host=host_name, binary='nova-compute')[0]
self.assertEqual("down", str(nova_service.state))
# Undo - second
self._trigger_undo_action(TRIGGER_ALARM_4)
nova_service = TempestClients.nova().services.list(
host=host_name, binary='nova-compute')[0]
self.assertEqual("up", str(nova_service.state))
except Exception as e:
self._handle_exception(e)
raise
finally:
self._trigger_undo_action(TRIGGER_ALARM_3)
self._trigger_undo_action(TRIGGER_ALARM_4)
# nova.host datasource may take up to snapshot_intreval to update
time.sleep(130)
def test_validate_notifier(self):
"""validate_notifier test
There tests validates work of aodh alarm notifier -
all created vitrage alarms appears in ceilometer
alarms-list.
IMPORTANT: enable notifiers=aodh in vitrage.conf file
"""
try:
nova_utils.create_instances(num_instances=2,
set_public_network=True)
self._create_alarm(
resource_id=self._get_hostname(),
alarm_name=RCA_ALARM_NAME)
vitrage_alarms = TempestClients.vitrage().alarm.list(
vitrage_id='all', all_tenants=True)
aodh_alarms = TempestClients.aodh().alarm.list()
self._validate_notifier(alarms=aodh_alarms,
vitrage_alarms=vitrage_alarms)
except Exception as e:
self._handle_exception(e)
raise
finally:
self._clean_all()
def _check_num_volumes(num_volumes=0, state=''):
if len(TempestClients.cinder().volumes.list()) != num_volumes:
return False
return all(volume.__dict__['status'].upper() == state.upper()
and len(volume.__dict__['attachments']) == 1
for volume in TempestClients.cinder().volumes.list())
def create_aodh_alarm(resource_id=None, name=None, unic=True):
if not name:
name = '%s-%s' % ('test_', random.randrange(0, 100000, 1))
elif unic:
name = '%s-%s' % (name, random.randrange(0, 100000, 1))
aodh_request = _aodh_request(resource_id=resource_id, name=name)
TempestClients.aodh().alarm.create(aodh_request)
time.sleep(45)
def create_volume_and_attach(name, size, instance_id, mount_point):
volume = TempestClients.cinder().volumes.create(name=name, size=size)
time.sleep(2)
TempestClients.cinder().volumes.attach(volume=volume,
instance_uuid=instance_id,
mountpoint=mount_point)
wait_for_status(30, _check_num_volumes, num_volumes=1, state='in-use')
time.sleep(2)
return volume
def delete_all_stacks():
stacks = TempestClients.heat().stacks.list()
for stack in stacks:
try:
TempestClients.heat().stacks.delete(stack.to_dict()['id'])
except Exception:
pass
wait_for_status(30, _check_num_stacks, num_stacks=0)
time.sleep(4)
def setUpClass(cls):
super(BaseVitrageTempest, cls).setUpClass()
cls.conf = service.prepare_service([])
TempestClients.class_init(cls.conf)
cls.vitrage_client = TempestClients.vitrage()
cls.num_default_networks = \
len(TempestClients.neutron().list_networks()['networks'])
cls.num_default_ports = 0
cls.num_default_entities = 3
cls.num_default_edges = 2
def delete_all_instances(**kwargs):
instances = TempestClients.nova().servers.list()
instances_to_delete = g_utils.all_matches(instances, **kwargs)
for item in instances_to_delete:
try:
TempestClients.nova().servers.force_delete(item)
except Exception:
pass
wait_for_status(30,
_check_num_instances,
num_instances=len(instances) - len(instances_to_delete))
time.sleep(2)
def generate_fake_host_alarm(hostname, event_type, enabled=True):
details = {
'hostname': hostname,
'source': 'fake_tempest_monitor',
'cause': 'another alarm',
'severity': 'critical',
'status': DOWN if enabled else UP,
'monitor_id': 'fake tempest monitor id',
'monitor_event_id': '111',
}
event_time = datetime.now()
event_time_iso = event_time.isoformat()
TempestClients.vitrage().event.post(event_time_iso, event_type, details)
def _validate_deduce_alarms(self, alarms, instances):
"""Validate alarm existence """
self.assertNotEqual(len(alarms), 0, 'The alarms list is empty')
LOG.info("The alarms list is : " + str(json.dumps(alarms)))
# Find the vitrage_id of the deduced alarms using their original id.
vitrage_resources = TempestClients.vitrage().resource.list(
all_tenants=False)
vitrage_instance_0_id = g_utils.first_match(vitrage_resources,
id=instances[0].id)
vitrage_instance_1_id = g_utils.first_match(vitrage_resources,
id=instances[1].id)
# Find the deduced alarms based on their properties
deduce_alarms_1 = g_utils.all_matches(
alarms,
vitrage_type=VITRAGE_DATASOURCE,
name=VITRAGE_ALARM_NAME,
vitrage_resource_type=NOVA_INSTANCE_DATASOURCE,
vitrage_resource_id=vitrage_instance_0_id[VProps.VITRAGE_ID])
deduce_alarms_2 = g_utils.all_matches(
alarms,
vitrage_type=VITRAGE_DATASOURCE,
name=VITRAGE_ALARM_NAME,
vitrage_resource_type=NOVA_INSTANCE_DATASOURCE,
vitrage_resource_id=vitrage_instance_1_id[VProps.VITRAGE_ID])
self.assertEqual(3, len(alarms), "Expected 3 alarms - 1 on host and "
"2 deduced")
self.assertEqual(1, len(deduce_alarms_1), "Deduced alarm not found")
self.assertEqual(1, len(deduce_alarms_2), "Deduced alarm not found")
def create_stacks(num_stacks, nested, template_file):
tpl_files, template = template_utils.process_template_path(
template_file,
object_request=http.authenticated_fetcher(TempestClients.heat()))
for i in range(num_stacks):
stack_name = 'stack_%s' % i + ('_nested' if nested else '')
TempestClients.heat().stacks.create(stack_name=stack_name,
template=template,
files=tpl_files,
parameters={})
wait_for_status(45,
_check_num_stacks,
num_stacks=num_stacks,
state='CREATE_COMPLETE')
time.sleep(2)
def _check_deduced(self, deduced_count, deduced_props, resource_id):
alarms = TempestClients.vitrage().alarm.list(vitrage_id=resource_id,
all_tenants=True)
deduces = g_utils.all_matches(alarms, **deduced_props)
self.assertEqual(
deduced_count, len(deduces),
'Expected %s deduces\n - \n%s\n - \n%s' %
(str(deduced_count), str(alarms), str(deduces)))
def _check_num_stacks(num_stacks, state=''):
stacks_list = \
[stack.to_dict() for stack in TempestClients.heat().stacks.list()
if 'FAILED' not in stack.to_dict()['stack_status']]
if len(stacks_list) != num_stacks:
return False
return all(stack['stack_status'].upper() == state.upper()
for stack in stacks_list)
def test_action_mark_down_instance(self):
vm_id = ""
try:
vm_id = nova_utils.create_instances(set_public_network=True)[0].id
# Do
self._trigger_do_action(TRIGGER_ALARM_3)
nova_instance = TempestClients.nova().servers.get(vm_id)
self.assertEqual("ERROR", str(nova_instance.status))
# Undo
self._trigger_undo_action(TRIGGER_ALARM_3)
nova_instance = TempestClients.nova().servers.get(vm_id)
self.assertEqual("ACTIVE", str(nova_instance.status))
except Exception as e:
self._handle_exception(e)
raise
finally:
pass
self._trigger_undo_action(TRIGGER_ALARM_3)
nova_utils.delete_all_instances(id=vm_id)
def delete_all_volumes():
volumes = TempestClients.cinder().volumes.list()
for volume in volumes:
try:
TempestClients.cinder().volumes.detach(volume)
TempestClients.cinder().volumes.force_delete(volume)
except Exception:
TempestClients.cinder().volumes.force_delete(volume)
wait_for_status(30, _check_num_volumes, num_volumes=0)
time.sleep(2)
def test_compare_cli_vs_api_alarms(self):
"""Wrapper that returns a test graph."""
try:
instances = nova_utils.create_instances(num_instances=1,
set_public_network=True)
self.assertNotEqual(len(instances), 0,
'The instances list is empty')
aodh_utils.create_aodh_alarm(resource_id=instances[0].id,
name='tempest_aodh_test')
api_alarms = TempestClients.vitrage().alarm.list(vitrage_id='all',
all_tenants=True)
cli_alarms = utils.run_vitrage_command('vitrage alarm list',
self.conf)
self._compare_alarms_lists(api_alarms, cli_alarms, AODH_DATASOURCE,
instances[0].id)
except Exception as e:
self._handle_exception(e)
raise
finally:
aodh_utils.delete_all_aodh_alarms()
nova_utils.delete_all_instances()
def create_instances(num_instances=1, set_public_network=False, name='vm'):
nics = []
flavor = get_first_flavor()
image = glance_utils.get_first_image()
if set_public_network:
public_net = neutron_utils.get_public_network()
if public_net:
nics = [{'net-id': public_net['id']}]
resources = [
TempestClients.nova().servers.create(name='%s-%s' % (name, index),
flavor=flavor,
image=image,
nics=nics)
for index in range(num_instances)
]
wait_for_status(30,
_check_num_instances,
num_instances=num_instances,
state='active')
time.sleep(2)
return resources
def delete_all_aodh_alarms():
alarms = TempestClients.aodh().alarm.list()
for alarm in alarms:
TempestClients.aodh().alarm.delete(alarm[AodhProps.ALARM_ID])
time.sleep(120)
def delete_all_ceilometer_alarms():
alarms = TempestClients.ceilometer().alarms.list()
for alarm in alarms:
TempestClients.ceilometer().alarms.delete(alarm.alarm_id)
time.sleep(120)
def _print_entity_graph(self):
api_graph = TempestClients.vitrage().topology.get(all_tenants=True)
graph = self._create_graph_from_graph_dictionary(api_graph)
LOG.info('Entity Graph: \n%s', graph.json_output_graph())
def get_first_instance(**kwargs):
instances = TempestClients.vitrage().resource.list(
NOVA_INSTANCE_DATASOURCE, all_tenants=True)
return g_utils.first_match(instances, **kwargs)
def get_first_image():
return TempestClients.glance().images.list().next()
def _check_num_instances(num_instances=0, state=''):
if len(TempestClients.nova().servers.list()) != num_instances:
return False
return all(instance.__dict__['status'].upper() == state.upper()
for instance in TempestClients.nova().servers.list())
def get_first_flavor():
return TempestClients.nova().flavors.list()[0]
def get_first_host(**kwargs):
hosts = TempestClients.vitrage().resource.list(NOVA_HOST_DATASOURCE,
all_tenants=True)
return g_utils.first_match(hosts, **kwargs)
def _check_num_alarms(self, num_alarms=0, state=''):
if len(TempestClients.aodh().alarm.list()) != num_alarms:
return False
return all(alarm[AodhProps.STATE].upper() == state.upper()
for alarm in TempestClients.aodh().alarm.list())
def _find_instance_resource_id(self):
servers = TempestClients.nova().servers.list()
return servers[0].id
def _check_num_vitrage_alarms(self, num_alarms):
vitrage_alarms = TempestClients.vitrage().alarm.list(vitrage_id='all',
all_tenants=True)
if len(vitrage_alarms) == num_alarms:
return True
return False
def test_overlapping_action_add_causal_relationship(self):
try:
# ---- Do first & second ----
self._trigger_do_action(TRIGGER_ALARM_1)
self._trigger_do_action(TRIGGER_ALARM_2)
alarms = TempestClients.vitrage().alarm.list(
vitrage_id=self.orig_host.get(VProps.VITRAGE_ID),
all_tenants=True)
deduced = g_utils.first_match(alarms, **DEDUCED_PROPS)
trigger1 = g_utils.first_match(alarms, **TRIGGER_ALARM_1_PROPS)
trigger2 = g_utils.first_match(alarms, **TRIGGER_ALARM_2_PROPS)
# Get Rca for the deduced
rca = TempestClients.vitrage().rca.get(deduced[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger1, trigger2], DEDUCED_PROPS)
# Get Rca for trigger 1
rca = TempestClients.vitrage().rca.get(trigger1[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger1], TRIGGER_ALARM_1_PROPS)
# Get Rca for trigger 2
rca = TempestClients.vitrage().rca.get(trigger2[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger2], TRIGGER_ALARM_2_PROPS)
# ---- Undo - first ----
self._trigger_undo_action(TRIGGER_ALARM_1)
alarms = TempestClients.vitrage().alarm.list(
vitrage_id=self.orig_host.get(VProps.VITRAGE_ID),
all_tenants=True)
deduced = g_utils.first_match(alarms, **DEDUCED_PROPS)
trigger2 = g_utils.first_match(alarms, **TRIGGER_ALARM_2_PROPS)
# Get Rca for the deduced
rca = TempestClients.vitrage().rca.get(deduced[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger2], DEDUCED_PROPS)
# Get Rca for trigger 2
rca = TempestClients.vitrage().rca.get(trigger2[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger2], TRIGGER_ALARM_2_PROPS)
# ---- Undo - second ----
self._trigger_undo_action(TRIGGER_ALARM_2)
alarms = TempestClients.vitrage().alarm.list(
vitrage_id=self.orig_host.get(VProps.VITRAGE_ID),
all_tenants=True)
self.assertEqual(
0, len(g_utils.all_matches(alarms, **TRIGGER_ALARM_1_PROPS)),
'trigger alarm 1 should have been removed')
self.assertEqual(
0, len(g_utils.all_matches(alarms, **TRIGGER_ALARM_2_PROPS)),
'trigger alarm 2 should have been removed')
self.assertEqual(0,
len(g_utils.all_matches(alarms, **DEDUCED_PROPS)),
'deduced alarm should have been removed')
except Exception as e:
self._handle_exception(e)
raise
finally:
self._trigger_undo_action(TRIGGER_ALARM_1)
self._trigger_undo_action(TRIGGER_ALARM_2)