def test_with_no_match(self):
"""Test to check that filters with no match do not send event """
try:
# Add webhook
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS,
regex_filter=FILTER_NO_MATCH,
)
# Raise alarm
self._trigger_do_action(TRIGGER_ALARM_1)
# Check event not received
self.assertThat(self.mock_server.requests, IsEmpty(),
'event should not have passed filter')
# Raise another alarm
self._trigger_do_action(TRIGGER_ALARM_2)
# Check second event not received
self.assertThat(self.mock_server.requests, IsEmpty(),
'event should not have passed filter')
finally:
self._trigger_undo_action(TRIGGER_ALARM_1)
self._trigger_undo_action(TRIGGER_ALARM_2)
def test_multiple_webhooks(self):
"""Test to check filter by type and with no filter (with 2 separate
webhooks)
"""
try:
# Add webhook
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS,
regex_filter=TYPE_FILTER,
)
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS
)
# Raise alarm
self._trigger_do_action(TRIGGER_ALARM_1)
# Check event received
self.assertThat(self.mock_server.requests, matchers.HasLength(2),
'event not posted to all webhooks')
# Raise another alarm
self._trigger_do_action(TRIGGER_ALARM_2)
# Check second event received
self.assertThat(self.mock_server.requests, matchers.HasLength(4),
'event not posted to all webhooks')
finally:
self._trigger_undo_action(TRIGGER_ALARM_1)
self._trigger_undo_action(TRIGGER_ALARM_2)
def test_basic_event(self):
try:
# Add webhook with filter matching alarm
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS,
regex_filter=NAME_FILTER,
headers=HEADERS_PROPS
)
# Raise alarm
self._trigger_do_action(TRIGGER_ALARM_1)
# Check event received
self.assertThat(self.mock_server.requests, matchers.HasLength(1),
'Wrong number of notifications for raise alarm')
# Undo
self._trigger_undo_action(TRIGGER_ALARM_1)
# Check event undo received
self.assertThat(self.mock_server.requests, matchers.HasLength(2),
'Wrong number of notifications for clear alarm')
finally:
self._trigger_undo_action(TRIGGER_ALARM_1)
def test_with_no_filter(self):
"""Test to see that a webhook with no filter receives all
notifications
"""
try:
# Add webhook
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS,
regex_filter=NAME_FILTER,
)
# Raise alarm
self._trigger_do_action(TRIGGER_ALARM_1)
# Check event received
self.assertThat(self.mock_server.requests, matchers.HasLength(1),
'Wrong number of notifications for raise alarm')
# Raise another alarm
self._trigger_do_action(TRIGGER_ALARM_2)
# Check second event received
self.assertThat(self.mock_server.requests, matchers.HasLength(2),
'Wrong number of notifications for clear alarm')
finally:
self._trigger_undo_action(TRIGGER_ALARM_1)
self._trigger_undo_action(TRIGGER_ALARM_2)
def test_db_init(self):
try:
v_utils.add_template(TEMPLATE_NAME)
time.sleep(SLEEP)
# 1. check template works well
self._check_template_instance_3rd_degree_scenarios()
# 2. check fast fail-over - start from database
topo1 = TempestClients.vitrage().topology.get(all_tenants=True)
v_utils.restart_graph()
time.sleep(MAX_FAIL_OVER_TIME)
for i in range(5):
self._check_template_instance_3rd_degree_scenarios()
topo2 = TempestClients.vitrage().topology.get(all_tenants=True)
self.assert_graph_equal(
topo1, topo2, 'comparing graph items iteration ' + str(i))
time.sleep(self.conf.datasources.snapshots_interval)
v_utils.delete_template(name=TEMPLATE_NAME)
time.sleep(SLEEP)
self._check_template_instance_3rd_degree_scenarios_deleted()
except Exception as e:
self._handle_exception(e)
if v_utils.get_first_template(name=TEMPLATE_NAME):
v_utils.delete_template(name=TEMPLATE_NAME)
time.sleep(SLEEP)
raise
def test_add_webhook(self):
webhooks = TempestClients.vitrage().webhook.list()
self.assertThat(
webhooks, matchers.HasLength(self.pre_test_webhook_count),
'Amount of webhooks should be '
'the same as before the test')
created_webhook = TempestClients.vitrage().webhook.add(
url="https://www.test.com",
regex_filter=REGEX_PROPS,
headers=HEADERS_PROPS)
self.assertIsNone(created_webhook.get('ERROR'), 'webhook not '
'created')
self.assertEqual(created_webhook[HEADERS], HEADERS_PROPS,
'headers not created correctly')
self.assertEqual(created_webhook[REGEX_FILTER], REGEX_PROPS,
'regex not created correctly')
self.assertEqual(created_webhook[URL], "https://www.test.com",
'URL not created correctly')
webhooks = TempestClients.vitrage().webhook.list()
self.assertThat(webhooks,
matchers.HasLength(self.pre_test_webhook_count + 1))
TempestClients.vitrage().webhook.delete(created_webhook['id'])
def test_for_deduced_alarm(self):
try:
# Add webhook with filter for the deduced alarm
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS,
regex_filter=NAME_FILTER_FOR_DEDUCED,
headers=HEADERS_PROPS
)
# Raise the trigger alarm
self._trigger_do_action(TRIGGER_ALARM_WITH_DEDUCED)
# Check event received - expected one for the deduced alarm
# (the trigger alarm does not pass the filter). This test verifies
# that the webhook is called only once for the deduced alarm.
time.sleep(1)
self.assertThat(self.mock_server.requests, matchers.HasLength(1),
'Wrong number of notifications for deduced alarm')
# Undo
self._trigger_undo_action(TRIGGER_ALARM_WITH_DEDUCED)
# Check event undo received
time.sleep(1)
self.assertThat(self.mock_server.requests, matchers.HasLength(2),
'Wrong number of notifications '
'for clear deduced alarm')
finally:
self._trigger_undo_action(TRIGGER_ALARM_WITH_DEDUCED)
def test_action_add_causal_relationship(self):
try:
# Do
self._trigger_do_action(TRIGGER_ALARM_2)
alarms = TempestClients.vitrage().alarm.list(
vitrage_id=self.orig_host.get(VProps.VITRAGE_ID),
all_tenants=True)
self.assertTrue(len(alarms) >= 2, 'alarms %s' % str(alarms))
deduced = g_utils.first_match(alarms, **DEDUCED_PROPS)
trigger = g_utils.first_match(alarms, **TRIGGER_ALARM_2_PROPS)
# Get Rca for the deduced
rca = TempestClients.vitrage().rca.get(deduced[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger], DEDUCED_PROPS)
# Get Rca for the trigger
rca = TempestClients.vitrage().rca.get(trigger[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger], TRIGGER_ALARM_2_PROPS)
except Exception as e:
self._handle_exception(e)
raise
finally:
self._trigger_undo_action(TRIGGER_ALARM_2)
def get_first_host(**kwargs):
try:
hosts = TempestClients.vitrage().resource.list(
NOVA_HOST_DATASOURCE, all_tenants=True)
except Exception as e:
LOG.exception("get_first_host failed with %s", e)
hosts = TempestClients.vitrage().resource.list(
NOVA_HOST_DATASOURCE, all_tenants=True)
return g_utils.first_match(hosts, **kwargs)
def delete_template(uuid=None, **kwargs):
if not uuid:
template = get_first_template(**kwargs)
if template:
uuid = template['uuid']
else:
return
TempestClients.vitrage().template.delete(uuid)
wait_for_status(
100,
lambda _id: True if not get_first_template(uuid=_id) else False,
_id=uuid)
def generate_fake_host_alarm(hostname, event_type, enabled=True):
details = {
'hostname': hostname,
'source': 'fake_tempest_monitor',
'cause': 'another alarm',
'severity': 'critical',
'status': DOWN if enabled else UP,
'monitor_id': 'fake tempest monitor id',
'monitor_event_id': '111',
}
event_time = datetime.now()
event_time_iso = event_time.isoformat()
TempestClients.vitrage().event.post(event_time_iso, event_type, details)
def _validate_deduce_alarms(self, alarms, instances):
"""Validate alarm existence """
self.assertThat(alarms, IsNotEmpty(), 'The alarms list is empty')
LOG.info("The alarms list is : " + str(json.dumps(alarms)))
# Find the vitrage_id of the deduced alarms using their original id.
vitrage_resources = TempestClients.vitrage().resource.list(
all_tenants=False)
vitrage_instance_0_id = g_utils.first_match(vitrage_resources,
id=instances[0].id)
vitrage_instance_1_id = g_utils.first_match(vitrage_resources,
id=instances[1].id)
# Find the deduced alarms based on their properties
deduce_alarms_1 = g_utils.all_matches(
alarms,
vitrage_type=VITRAGE_DATASOURCE,
name=VITRAGE_ALARM_NAME,
vitrage_resource_type=NOVA_INSTANCE_DATASOURCE,
vitrage_resource_id=vitrage_instance_0_id[VProps.VITRAGE_ID])
deduce_alarms_2 = g_utils.all_matches(
alarms,
vitrage_type=VITRAGE_DATASOURCE,
name=VITRAGE_ALARM_NAME,
vitrage_resource_type=NOVA_INSTANCE_DATASOURCE,
vitrage_resource_id=vitrage_instance_1_id[VProps.VITRAGE_ID])
self.assertThat(alarms, matchers.HasLength(3),
"Expected 3 alarms - 1 on host and 2 deduced")
self.assertThat(deduce_alarms_1, matchers.HasLength(1),
"Deduced alarm not found")
self.assertThat(deduce_alarms_2, matchers.HasLength(1),
"Deduced alarm not found")
def _check_template_instance_3rd_degree_scenarios_deleted(self):
alarm_count = TempestClients.vitrage().alarm.count(all_tenants=True)
self.assertEqual(0, alarm_count['SEVERE'],
'found SEVERE deduced alarms after template delete')
self.assertEqual(
0, alarm_count['CRITICAL'],
'found CRITICAL deduced alarms after template delete')
def _check_deduced(self, deduced_count, deduced_props, resource_id):
alarms = TempestClients.vitrage().alarm.list(vitrage_id=resource_id,
all_tenants=True)
deduces = g_utils.all_matches(alarms, **deduced_props)
self.assertThat(
deduces, matchers.HasLength(deduced_count),
'Expected %s deduces\n - \n%s\n - \n%s' %
(str(deduced_count), str(alarms), str(deduces)))
def check_rca(alarm):
rca = TempestClients.vitrage().rca.get(alarm['vitrage_id'],
all_tenants=True)
try:
self._check_rca(rca, expected_rca, alarm)
return True
except Exception as e:
LOG.exception('check_rca failed', e)
return False
def _check_template_instance_3rd_degree_scenarios(self):
try:
alarm_count = TempestClients.vitrage().alarm.count(
all_tenants=True)
self.assertEqual(
self.conf.mock_graph_datasource.instances_per_host,
alarm_count['SEVERE'],
'Each instance should have one SEVERE deduced alarm')
self.assertEqual(
self.conf.mock_graph_datasource.instances_per_host,
alarm_count['CRITICAL'],
'Each instance should have one CRITICAL deduced alarm')
expected_rca = [{VertexProperties.VITRAGE_TYPE: 'zabbix'}] * self.\
conf.mock_graph_datasource.zabbix_alarms_per_host
expected_rca.extend([{'name': DEDUCED_1}, {'name': DEDUCED_2}])
def check_rca(alarm):
rca = TempestClients.vitrage().rca.get(alarm['vitrage_id'],
all_tenants=True)
try:
self._check_rca(rca, expected_rca, alarm)
return True
except Exception as e:
LOG.exception('check_rca failed', e)
return False
# 10 threads calling rca api
alarms = TempestClients.vitrage().alarm.list(all_tenants=True,
vitrage_id='all')
deduced_alarms = g_utils.all_matches(alarms,
vitrage_type='vitrage',
name=DEDUCED_2)
workers = futures.ThreadPoolExecutor(max_workers=10)
workers_result = [
r for r in workers.map(check_rca, deduced_alarms)
]
self.assertTrue(all(workers_result))
except Exception as e:
v_utils.delete_template(name=TEMPLATE_NAME)
self._handle_exception(e)
raise
def test_delete_webhook(self):
webhooks = TempestClients.vitrage().webhook.list()
self.assertThat(
webhooks, matchers.HasLength(self.pre_test_webhook_count),
'Amount of webhooks should '
'be the same as before the test')
created_webhook = TempestClients.vitrage().webhook.add(
url="https://www.test.com",
regex_filter=REGEX_PROPS,
headers=HEADERS_PROPS)
created_webhook = TempestClients.vitrage().webhook.delete(
id=created_webhook['id'])
self.assertIsNotNone(created_webhook.get('SUCCESS'),
'failed to delete')
self.assertThat(webhooks,
matchers.HasLength(self.pre_test_webhook_count),
'No webhooks should exist after deletion')
def _print_entity_graph(self):
api_graph = TempestClients.vitrage().topology.get(all_tenants=True)
graph = self._create_graph_from_graph_dictionary(api_graph)
node_link_data = json_graph.node_link_data(graph._g)
for index, node in enumerate(node_link_data['nodes']):
if VProps.ID in graph._g.node[node[VProps.ID]]:
node[VProps.ID] = graph._g.node[node[VProps.ID]][VProps.ID]
node[VProps.GRAPH_INDEX] = index
LOG.info('Entity Graph: \n%s', json.dumps(node_link_data))
def test_list_webhook(self):
webhooks = TempestClients.vitrage().webhook.list()
self.assertThat(
webhooks, matchers.HasLength(self.pre_test_webhook_count),
'Amount of webhooks should be '
'the same as before the test')
created_webhook = TempestClients.vitrage().webhook.add(
url="https://www.test.com",
regex_filter=REGEX_PROPS,
headers=HEADERS_PROPS)
webhooks = TempestClients.vitrage().webhook.list()
self.assertThat(webhooks,
matchers.HasLength(self.pre_test_webhook_count + 1))
self.assertEqual(created_webhook[HEADERS], webhooks[0][HEADERS])
self.assertEqual(created_webhook['id'], webhooks[0]['id'])
self.assertEqual(created_webhook[REGEX_FILTER],
webhooks[0][REGEX_FILTER])
TempestClients.vitrage().webhook.delete(created_webhook['id'])
def add_template(filename='',
folder='templates/api',
template_type=TemplateTypes.STANDARD,
status=TemplateStatus.ACTIVE):
full_path = g_utils.tempest_resources_dir() + '/' + folder + '/' + filename
t = TempestClients.vitrage().template.add(full_path, template_type)
if t and t[0]:
wait_for_status(
100,
get_first_template,
uuid=t[0]['uuid'], status=status)
return t[0]
return None
def test_show_webhook(self):
webhooks = TempestClients.vitrage().webhook.list()
self.assertThat(
webhooks, matchers.HasLength(self.pre_test_webhook_count),
'Amount of webhooks should be '
'the same as before the test')
created_webhook = TempestClients.vitrage().webhook.add(
url="https://www.test.com",
regex_filter=REGEX_PROPS,
headers=HEADERS_PROPS)
show_webhook = TempestClients.vitrage().webhook.show(
created_webhook['id'])
self.assertIsNotNone(show_webhook, 'webhook not listed')
self.assertEqual(created_webhook[HEADERS], show_webhook[HEADERS],
'headers mismatch')
self.assertEqual(created_webhook[REGEX_FILTER],
show_webhook[REGEX_FILTER], 'regex mismatch')
self.assertEqual(created_webhook[URL], show_webhook[URL],
'URL mismatch')
TempestClients.vitrage().webhook.delete(created_webhook['id'])
def setUpClass(cls):
super(BaseVitrageTempest, cls).setUpClass()
warnings.filterwarnings(action="ignore",
message="unclosed",
category=ResourceWarning)
TempestClients.class_init(cls.os_primary.credentials)
cls.vitrage_client = TempestClients.vitrage()
cls.vitrage_client_for_demo_user = \
TempestClients.vitrage_client_for_user()
cls.num_default_networks = \
len(TempestClients.neutron().list_networks()['networks'])
cls.num_default_ports = cls._get_num_default_ports()
cls.num_default_entities = 3
cls.num_default_edges = 2
cls.num_demo_tenant_networks = cls._calc_num_tenant_networks()
def test_high_availability_events(self):
"""The purpose of the test is to check that events are stored
That is, during different stages in vitrage-graph lifetime:
before graph read from db (during init)
after graph read from db (during init)
during get_all
after get_all
"""
try:
# adding a template just to create more load (to slow things down)
v_utils.add_template(TEMPLATE_NAME)
time.sleep(SLEEP)
self.keep_sending_events = True
self.num_of_sent_events = 0
doctor_events_thread = self._async_doctor_events()
time.sleep(10)
v_utils.stop_graph()
time.sleep(10)
v_utils.restart_graph()
v_utils.delete_template(name=TEMPLATE_NAME)
# sleep to allow get_all to start and finish at least once:
time.sleep(4 * self.conf.datasources.snapshots_interval)
v_utils.restart_graph()
self.keep_sending_events = False
time.sleep(MAX_FAIL_OVER_TIME)
doctor_events_thread.join(timeout=10)
alarm_count = TempestClients.vitrage().alarm.count(
all_tenants=True)
self.assertTrue(self.num_of_sent_events > 0,
'Test did not create events')
self.assertEqual(self.num_of_sent_events, alarm_count['CRITICAL'],
'CRITICAL doctor events expected')
except Exception as e:
self._handle_exception(e)
raise
finally:
self._remove_doctor_events()
if v_utils.get_first_template(name=TEMPLATE_NAME):
v_utils.delete_template(name=TEMPLATE_NAME)
time.sleep(SLEEP)
def setUpClass(cls):
super(BaseVitrageTempest, cls).setUpClass()
warnings.filterwarnings(action="ignore",
message="unclosed",
category=ResourceWarning)
cls.conf = service.prepare_service([])
TempestClients.class_init(cls.conf)
cls.vitrage_client = TempestClients.vitrage()
cls.vitrage_client_for_demo_user = \
TempestClients.vitrage_client_for_user(
cls.DEMO_USERNAME, cls.USER_DOMAIN_ID,
cls.DEMO_PROJECT_NAME, cls.PROJECT_DOMAIN_ID)
cls.num_default_networks = \
len(TempestClients.neutron().list_networks()['networks'])
cls.num_default_ports = cls._get_num_default_ports()
cls.num_default_entities = 3
cls.num_default_edges = 2
cls.num_demo_tenant_networks = cls._calc_num_demo_tenant_networks()
def test_compare_cli_vs_api_alarms(self):
"""Wrapper that returns a test graph."""
try:
instances = nova_utils.create_instances(num_instances=1,
set_public_network=True)
self.assertThat(instances, IsNotEmpty(),
'The instances list is empty')
aodh_utils.create_aodh_alarm(resource_id=instances[0].id,
name='tempest_aodh_test')
api_alarms = TempestClients.vitrage().alarm.list(vitrage_id='all',
all_tenants=True)
cli_alarms = utils.run_vitrage_command('vitrage alarm list',
self.conf)
self._compare_alarms_lists(api_alarms, cli_alarms, AODH_DATASOURCE,
instances[0].id)
except Exception as e:
self._handle_exception(e)
raise
finally:
aodh_utils.delete_all_aodh_alarms()
nova_utils.delete_all_instances()
def test_validate_notifier(self):
"""validate_notifier test
There tests validates work of aodh alarm notifier -
all created vitrage alarms appears in ceilometer
alarms-list.
IMPORTANT: enable notifiers=aodh in vitrage.conf file
"""
try:
nova_utils.create_instances(num_instances=2,
set_public_network=True)
self._create_alarm(resource_id=self._get_hostname(),
alarm_name=RCA_ALARM_NAME)
vitrage_alarms = TempestClients.vitrage().alarm.list(
vitrage_id='all', all_tenants=True)
aodh_alarms = TempestClients.aodh().alarm.list()
self._validate_notifier(alarms=aodh_alarms,
vitrage_alarms=vitrage_alarms)
except Exception as e:
self._handle_exception(e)
raise
finally:
self._clean_all()
def _delete_webhooks():
webhooks = TempestClients.vitrage().webhook.list()
for webhook in webhooks:
TempestClients.vitrage().webhook.delete(webhook['id'])
def get_first_template(**kwargs):
templates = TempestClients.vitrage().template.list()
return g_utils.first_match(templates, **kwargs)
def get_first_instance(**kwargs):
instances = TempestClients.vitrage().resource.list(
NOVA_INSTANCE_DATASOURCE, all_tenants=True)
return g_utils.first_match(instances, **kwargs)
def test_payload_format(self):
try:
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS,
headers=HEADERS_PROPS
)
# Raise the trigger alarm
self._trigger_do_action(TRIGGER_ALARM_1)
# pre check that correct amount of notifications sent
self.assertThat(self.mock_server.requests, matchers.HasLength(1),
'Wrong number of notifications for alarm')
self.assertThat(messages, matchers.HasLength(1),
'Wrong number of messages for alarm')
alarm = ast.literal_eval(messages[0])
# check that only specified fields are sent for the alarm,
# payload and resource
passed_filter = utils.filter_data(alarm,
MAIN_FILTER,
match_filter=False)
self.assertThat(passed_filter,
IsEmpty(),
"Wrong main fields sent")
payload = alarm.get(PAYLOAD)
if payload:
passed_filter = utils.filter_data(payload,
DOCTOR_ALARM_FILTER,
match_filter=False)
self.assertThat(passed_filter,
IsEmpty(),
"Wrong alarm fields sent")
sent_fields = utils.filter_data(payload, DOCTOR_ALARM_FILTER)
self.assertEqual(DOCTOR_ALARM_FILTER, sent_fields,
"Some alarm fields not sent")
resource = payload.get(VProps.RESOURCE)
if resource:
passed_filter = utils.filter_data(resource,
RESOURCE_FILTER,
match_filter=False)
self.assertThat(passed_filter,
IsEmpty(),
"Wrong resource fields sent")
sent_fields = utils.filter_data(resource, RESOURCE_FILTER)
self.assertEqual(RESOURCE_FILTER, sent_fields,
"Some resource fields not sent")
finally:
self._trigger_undo_action(TRIGGER_ALARM_1)