def test_with_no_filter(self):
"""Test to see that a webhook with no filter receives all
notifications
"""
try:
# Add webhook
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS,
regex_filter=NAME_FILTER,
)
# Raise alarm
self._trigger_do_action(TRIGGER_ALARM_1)
# Check event received
self.assertThat(self.mock_server.requests, matchers.HasLength(1),
'Wrong number of notifications for raise alarm')
# Raise another alarm
self._trigger_do_action(TRIGGER_ALARM_2)
# Check second event received
self.assertThat(self.mock_server.requests, matchers.HasLength(2),
'Wrong number of notifications for clear alarm')
finally:
self._trigger_undo_action(TRIGGER_ALARM_1)
self._trigger_undo_action(TRIGGER_ALARM_2)
def test_for_deduced_alarm(self):
try:
# Add webhook with filter for the deduced alarm
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS,
regex_filter=NAME_FILTER_FOR_DEDUCED,
headers=HEADERS_PROPS
)
# Raise the trigger alarm
self._trigger_do_action(TRIGGER_ALARM_WITH_DEDUCED)
# Check event received - expected one for the deduced alarm
# (the trigger alarm does not pass the filter). This test verifies
# that the webhook is called only once for the deduced alarm.
time.sleep(1)
self.assertThat(self.mock_server.requests, matchers.HasLength(1),
'Wrong number of notifications for deduced alarm')
# Undo
self._trigger_undo_action(TRIGGER_ALARM_WITH_DEDUCED)
# Check event undo received
time.sleep(1)
self.assertThat(self.mock_server.requests, matchers.HasLength(2),
'Wrong number of notifications '
'for clear deduced alarm')
finally:
self._trigger_undo_action(TRIGGER_ALARM_WITH_DEDUCED)
def test_basic_event(self):
try:
# Add webhook with filter matching alarm
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS,
regex_filter=NAME_FILTER,
headers=HEADERS_PROPS
)
# Raise alarm
self._trigger_do_action(TRIGGER_ALARM_1)
# Check event received
self.assertThat(self.mock_server.requests, matchers.HasLength(1),
'Wrong number of notifications for raise alarm')
# Undo
self._trigger_undo_action(TRIGGER_ALARM_1)
# Check event undo received
self.assertThat(self.mock_server.requests, matchers.HasLength(2),
'Wrong number of notifications for clear alarm')
finally:
self._trigger_undo_action(TRIGGER_ALARM_1)
def test_multiple_webhooks(self):
"""Test to check filter by type and with no filter (with 2 separate
webhooks)
"""
try:
# Add webhook
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS,
regex_filter=TYPE_FILTER,
)
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS
)
# Raise alarm
self._trigger_do_action(TRIGGER_ALARM_1)
# Check event received
self.assertThat(self.mock_server.requests, matchers.HasLength(2),
'event not posted to all webhooks')
# Raise another alarm
self._trigger_do_action(TRIGGER_ALARM_2)
# Check second event received
self.assertThat(self.mock_server.requests, matchers.HasLength(4),
'event not posted to all webhooks')
finally:
self._trigger_undo_action(TRIGGER_ALARM_1)
self._trigger_undo_action(TRIGGER_ALARM_2)
def test_overlapping_action_mark_down(self):
try:
host_name = self.orig_host.get(VProps.NAME)
# Do - first
self._trigger_do_action(TRIGGER_ALARM_3)
nova_service = TempestClients.nova().services.list(
host=host_name, binary='nova-compute')[0]
self.assertEqual("down", nova_service.state)
# Do - second
self._trigger_do_action(TRIGGER_ALARM_4)
nova_service = TempestClients.nova().services.list(
host=host_name, binary='nova-compute')[0]
self.assertEqual("down", nova_service.state)
# Undo - first
self._trigger_undo_action(TRIGGER_ALARM_3)
nova_service = TempestClients.nova().services.list(
host=host_name, binary='nova-compute')[0]
self.assertEqual("down", nova_service.state)
# Undo - second
self._trigger_undo_action(TRIGGER_ALARM_4)
nova_service = TempestClients.nova().services.list(
host=host_name, binary='nova-compute')[0]
self.assertEqual("up", nova_service.state)
finally:
self._trigger_undo_action(TRIGGER_ALARM_3)
self._trigger_undo_action(TRIGGER_ALARM_4)
# nova.host datasource may take up to snapshot_interval to update
time.sleep(130)
def test_action_add_causal_relationship(self):
try:
# Do
self._trigger_do_action(TRIGGER_ALARM_2)
alarms = TempestClients.vitrage().alarm.list(
vitrage_id=self.orig_host.get(VProps.VITRAGE_ID),
all_tenants=True)
self.assertTrue(len(alarms) >= 2, 'alarms %s' % str(alarms))
deduced = g_utils.first_match(alarms, **DEDUCED_PROPS)
trigger = g_utils.first_match(alarms, **TRIGGER_ALARM_2_PROPS)
# Get Rca for the deduced
rca = TempestClients.vitrage().rca.get(deduced[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger], DEDUCED_PROPS)
# Get Rca for the trigger
rca = TempestClients.vitrage().rca.get(trigger[VProps.VITRAGE_ID],
all_tenants=True)
self._check_rca(rca, [deduced, trigger], TRIGGER_ALARM_2_PROPS)
except Exception as e:
self._handle_exception(e)
raise
finally:
self._trigger_undo_action(TRIGGER_ALARM_2)
def test_add_webhook(self):
webhooks = TempestClients.vitrage().webhook.list()
self.assertThat(
webhooks, matchers.HasLength(self.pre_test_webhook_count),
'Amount of webhooks should be '
'the same as before the test')
created_webhook = TempestClients.vitrage().webhook.add(
url="https://www.test.com",
regex_filter=REGEX_PROPS,
headers=HEADERS_PROPS)
self.assertIsNone(created_webhook.get('ERROR'), 'webhook not '
'created')
self.assertEqual(created_webhook[HEADERS], HEADERS_PROPS,
'headers not created correctly')
self.assertEqual(created_webhook[REGEX_FILTER], REGEX_PROPS,
'regex not created correctly')
self.assertEqual(created_webhook[URL], "https://www.test.com",
'URL not created correctly')
webhooks = TempestClients.vitrage().webhook.list()
self.assertThat(webhooks,
matchers.HasLength(self.pre_test_webhook_count + 1))
TempestClients.vitrage().webhook.delete(created_webhook['id'])
def _check_num_volumes(num_volumes=0, state=''):
if len(TempestClients.cinder().volumes.list()) != num_volumes:
return False
return all(vars(volume)['status'].upper() == state.upper() and
len(vars(volume)['attachments']) == 1
for volume in TempestClients.cinder().volumes.list())
def test_with_no_match(self):
"""Test to check that filters with no match do not send event """
try:
# Add webhook
TempestClients.vitrage().webhook.add(
url=self.URL_PROPS,
regex_filter=FILTER_NO_MATCH,
)
# Raise alarm
self._trigger_do_action(TRIGGER_ALARM_1)
# Check event not received
self.assertThat(self.mock_server.requests, IsEmpty(),
'event should not have passed filter')
# Raise another alarm
self._trigger_do_action(TRIGGER_ALARM_2)
# Check second event not received
self.assertThat(self.mock_server.requests, IsEmpty(),
'event should not have passed filter')
finally:
self._trigger_undo_action(TRIGGER_ALARM_1)
self._trigger_undo_action(TRIGGER_ALARM_2)
def test_db_init(self):
try:
v_utils.add_template(TEMPLATE_NAME)
time.sleep(SLEEP)
# 1. check template works well
self._check_template_instance_3rd_degree_scenarios()
# 2. check fast fail-over - start from database
topo1 = TempestClients.vitrage().topology.get(all_tenants=True)
v_utils.restart_graph()
time.sleep(MAX_FAIL_OVER_TIME)
for i in range(5):
self._check_template_instance_3rd_degree_scenarios()
topo2 = TempestClients.vitrage().topology.get(all_tenants=True)
self.assert_graph_equal(
topo1, topo2, 'comparing graph items iteration ' + str(i))
time.sleep(self.conf.datasources.snapshots_interval)
v_utils.delete_template(name=TEMPLATE_NAME)
time.sleep(SLEEP)
self._check_template_instance_3rd_degree_scenarios_deleted()
except Exception as e:
self._handle_exception(e)
if v_utils.get_first_template(name=TEMPLATE_NAME):
v_utils.delete_template(name=TEMPLATE_NAME)
time.sleep(SLEEP)
raise
def create_aodh_alarm(resource_id=None, name=None, unic=True):
if not name:
name = '%s-%s' % ('test_', random.randrange(0, 100000, 1))
elif unic:
name = '%s-%s' % (name, random.randrange(0, 100000, 1))
aodh_request = _aodh_request('event', resource_id=resource_id, name=name)
TempestClients.aodh().alarm.create(aodh_request)
time.sleep(20)
def create_ceilometer_alarm(resource_id=None, name=None, unic=True):
if not name:
name = '%s-%s' % ('test_', random.randrange(0, 100000, 1))
elif unic:
name = '%s-%s' % (name, random.randrange(0, 100000, 1))
aodh_request = _aodh_request(resource_id=resource_id, name=name)
TempestClients.ceilometer().alarms.create(**aodh_request)
time.sleep(45)
def get_first_host(**kwargs):
try:
hosts = TempestClients.vitrage().resource.list(
NOVA_HOST_DATASOURCE, all_tenants=True)
except Exception as e:
LOG.exception("get_first_host failed with %s", e)
hosts = TempestClients.vitrage().resource.list(
NOVA_HOST_DATASOURCE, all_tenants=True)
return g_utils.first_match(hosts, **kwargs)
def delete_all_stacks():
stacks = TempestClients.heat().stacks.list()
for stack in stacks:
try:
TempestClients.heat().stacks.delete(stack.to_dict()['id'])
except Exception:
pass
wait_for_status(30, _check_num_stacks, num_stacks=0)
time.sleep(4)
def create_volume_and_attach(name, size, instance_id, mount_point):
volume = TempestClients.cinder().volumes.create(name=name,
size=size)
time.sleep(2)
TempestClients.cinder().volumes.attach(volume=volume,
instance_uuid=instance_id,
mountpoint=mount_point)
wait_for_status(30, _check_num_volumes, num_volumes=1, state='in-use')
time.sleep(2)
return volume
def delete_all_instances(**kwargs):
instances = TempestClients.nova().servers.list()
instances_to_delete = g_utils.all_matches(instances, **kwargs)
for item in instances_to_delete:
try:
TempestClients.nova().servers.force_delete(item)
except Exception:
LOG.exception('Failed to force delete instance %s', item.id)
wait_for_status(30,
check_deleted_instances,
ids=[instance.id for instance in instances_to_delete])
time.sleep(2)
def delete_template(uuid=None, **kwargs):
if not uuid:
template = get_first_template(**kwargs)
if template:
uuid = template['uuid']
else:
return
TempestClients.vitrage().template.delete(uuid)
wait_for_status(
100,
lambda _id: True if not get_first_template(uuid=_id) else False,
_id=uuid)
def generate_fake_host_alarm(hostname, event_type, enabled=True):
details = {
'hostname': hostname,
'source': 'fake_tempest_monitor',
'cause': 'another alarm',
'severity': 'critical',
'status': DOWN if enabled else UP,
'monitor_id': 'fake tempest monitor id',
'monitor_event_id': '111',
}
event_time = datetime.now()
event_time_iso = event_time.isoformat()
TempestClients.vitrage().event.post(event_time_iso, event_type, details)
def _do_test_action_mark_down_instance(self, trigger_name):
vm_id = ""
try:
vm_id = nova_utils.create_instances(set_public_network=True)[0].id
# Do
self._trigger_do_action(trigger_name)
nova_instance = TempestClients.nova().servers.get(vm_id)
self.assertEqual("ERROR", nova_instance.status)
# Undo
self._trigger_undo_action(trigger_name)
nova_instance = TempestClients.nova().servers.get(vm_id)
self.assertEqual("ACTIVE", nova_instance.status)
finally:
self._trigger_undo_action(trigger_name)
nova_utils.delete_all_instances(id=vm_id)
def create_stacks(num_stacks, nested, template_file, image):
tpl_files, template = template_utils.process_template_path(
template_file,
object_request=http.authenticated_fetcher(TempestClients.heat()))
for i in range(num_stacks):
stack_name = 'stack_%s' % i + ('_nested' if nested else '')
TempestClients.heat().stacks.create(stack_name=stack_name,
template=template,
files=tpl_files,
parameters={'image': image})
wait_for_status(45,
_check_num_stacks,
num_stacks=num_stacks,
state='CREATE_COMPLETE')
time.sleep(2)
def _check_template_instance_3rd_degree_scenarios_deleted(self):
alarm_count = TempestClients.vitrage().alarm.count(all_tenants=True)
self.assertEqual(0, alarm_count['SEVERE'],
'found SEVERE deduced alarms after template delete')
self.assertEqual(
0, alarm_count['CRITICAL'],
'found CRITICAL deduced alarms after template delete')
def setUpClass(cls):
super(BaseVitrageTempest, cls).setUpClass()
warnings.filterwarnings(action="ignore",
message="unclosed",
category=ResourceWarning)
TempestClients.class_init(cls.os_primary.credentials)
cls.vitrage_client = TempestClients.vitrage()
cls.vitrage_client_for_demo_user = \
TempestClients.vitrage_client_for_user()
cls.num_default_networks = \
len(TempestClients.neutron().list_networks()['networks'])
cls.num_default_ports = cls._get_num_default_ports()
cls.num_default_entities = 3
cls.num_default_edges = 2
cls.num_demo_tenant_networks = cls._calc_num_tenant_networks()
def _validate_deduce_alarms(self, alarms, instances):
"""Validate alarm existence """
self.assertThat(alarms, IsNotEmpty(), 'The alarms list is empty')
LOG.info("The alarms list is : " + str(json.dumps(alarms)))
# Find the vitrage_id of the deduced alarms using their original id.
vitrage_resources = TempestClients.vitrage().resource.list(
all_tenants=False)
vitrage_instance_0_id = g_utils.first_match(vitrage_resources,
id=instances[0].id)
vitrage_instance_1_id = g_utils.first_match(vitrage_resources,
id=instances[1].id)
# Find the deduced alarms based on their properties
deduce_alarms_1 = g_utils.all_matches(
alarms,
vitrage_type=VITRAGE_DATASOURCE,
name=VITRAGE_ALARM_NAME,
vitrage_resource_type=NOVA_INSTANCE_DATASOURCE,
vitrage_resource_id=vitrage_instance_0_id[VProps.VITRAGE_ID])
deduce_alarms_2 = g_utils.all_matches(
alarms,
vitrage_type=VITRAGE_DATASOURCE,
name=VITRAGE_ALARM_NAME,
vitrage_resource_type=NOVA_INSTANCE_DATASOURCE,
vitrage_resource_id=vitrage_instance_1_id[VProps.VITRAGE_ID])
self.assertThat(alarms, matchers.HasLength(3),
"Expected 3 alarms - 1 on host and 2 deduced")
self.assertThat(deduce_alarms_1, matchers.HasLength(1),
"Deduced alarm not found")
self.assertThat(deduce_alarms_2, matchers.HasLength(1),
"Deduced alarm not found")
def _check_deduced(self, deduced_count, deduced_props, resource_id):
alarms = TempestClients.vitrage().alarm.list(vitrage_id=resource_id,
all_tenants=True)
deduces = g_utils.all_matches(alarms, **deduced_props)
self.assertThat(
deduces, matchers.HasLength(deduced_count),
'Expected %s deduces\n - \n%s\n - \n%s' %
(str(deduced_count), str(alarms), str(deduces)))
def _check_num_stacks(num_stacks, state=''):
stacks_list = \
[stack.to_dict() for stack in TempestClients.heat().stacks.list()
if 'FAILED' not in stack.to_dict()['stack_status']]
if len(stacks_list) != num_stacks:
return False
return all(stack['stack_status'].upper() == state.upper()
for stack in stacks_list)
def check_rca(alarm):
rca = TempestClients.vitrage().rca.get(alarm['vitrage_id'],
all_tenants=True)
try:
self._check_rca(rca, expected_rca, alarm)
return True
except Exception as e:
LOG.exception('check_rca failed', e)
return False
def create_aodh_metrics_threshold_alarm(resource_id=None, name=None):
if not name:
name = '%s-%s' % ('test_', random.randrange(0, 100000, 1))
met = TempestClients.gnocchi().metric.create(resource_id=resource_id,
name='test',
archive_policy_name='high')
metric = met[AodhProps.ID]
rule_opts = dict(threshold='100',
aggregation_method='mean',
comparison_operator='lt',
metrics=[metric])
rule = {AodhProps.METRICS_THRESHOLD_RULE: rule_opts}
aodh_request = _aodh_request(AodhProps.METRICS_THRESHOLD,
resource_id=resource_id,
name=name,
rule=rule)
TempestClients.aodh().alarm.create(aodh_request)
time.sleep(10)
def _check_template_instance_3rd_degree_scenarios(self):
try:
alarm_count = TempestClients.vitrage().alarm.count(
all_tenants=True)
self.assertEqual(
self.conf.mock_graph_datasource.instances_per_host,
alarm_count['SEVERE'],
'Each instance should have one SEVERE deduced alarm')
self.assertEqual(
self.conf.mock_graph_datasource.instances_per_host,
alarm_count['CRITICAL'],
'Each instance should have one CRITICAL deduced alarm')
expected_rca = [{VertexProperties.VITRAGE_TYPE: 'zabbix'}] * self.\
conf.mock_graph_datasource.zabbix_alarms_per_host
expected_rca.extend([{'name': DEDUCED_1}, {'name': DEDUCED_2}])
def check_rca(alarm):
rca = TempestClients.vitrage().rca.get(alarm['vitrage_id'],
all_tenants=True)
try:
self._check_rca(rca, expected_rca, alarm)
return True
except Exception as e:
LOG.exception('check_rca failed', e)
return False
# 10 threads calling rca api
alarms = TempestClients.vitrage().alarm.list(all_tenants=True,
vitrage_id='all')
deduced_alarms = g_utils.all_matches(alarms,
vitrage_type='vitrage',
name=DEDUCED_2)
workers = futures.ThreadPoolExecutor(max_workers=10)
workers_result = [
r for r in workers.map(check_rca, deduced_alarms)
]
self.assertTrue(all(workers_result))
except Exception as e:
v_utils.delete_template(name=TEMPLATE_NAME)
self._handle_exception(e)
raise
def _do_test_action_mark_down_host(self, trigger_name):
try:
host_name = self.orig_host.get(VProps.NAME)
# Do
self._trigger_do_action(trigger_name)
nova_service = TempestClients.nova().services.list(
host=host_name, binary='nova-compute')[0]
self.assertEqual("down", nova_service.state)
# Undo
self._trigger_undo_action(trigger_name)
nova_service = TempestClients.nova().services.list(
host=host_name, binary='nova-compute')[0]
self.assertEqual("up", nova_service.state)
finally:
self._trigger_undo_action(trigger_name)
# nova.host datasource may take up to snapshot_intreval to update
time.sleep(130)
def setUpClass(cls):
super(TestMistralNotifier, cls).setUpClass()
cls.mistral_client = TempestClients.mistral()
cls._templates = []
cls._templates.append(v_utils.add_template('v1_execute_mistral.yaml'))
cls._templates.append(v_utils.add_template('v2_execute_mistral.yaml'))
cls._templates.append(v_utils.add_template('v3_execute_mistral.yaml'))
# Create a Mistral workflow
cls.mistral_client.workflows.create(WF_DEFINITION)
