def test_with_no_match(self): """Test to check that filters with no match do not send event """ try: # Add webhook TempestClients.vitrage().webhook.add( url=self.URL_PROPS, regex_filter=FILTER_NO_MATCH, ) # Raise alarm self._trigger_do_action(TRIGGER_ALARM_1) # Check event not received self.assertThat(self.mock_server.requests, IsEmpty(), 'event should not have passed filter') # Raise another alarm self._trigger_do_action(TRIGGER_ALARM_2) # Check second event not received self.assertThat(self.mock_server.requests, IsEmpty(), 'event should not have passed filter') finally: self._trigger_undo_action(TRIGGER_ALARM_1) self._trigger_undo_action(TRIGGER_ALARM_2)
def test_multiple_webhooks(self): """Test to check filter by type and with no filter (with 2 separate webhooks) """ try: # Add webhook TempestClients.vitrage().webhook.add( url=self.URL_PROPS, regex_filter=TYPE_FILTER, ) TempestClients.vitrage().webhook.add( url=self.URL_PROPS ) # Raise alarm self._trigger_do_action(TRIGGER_ALARM_1) # Check event received self.assertThat(self.mock_server.requests, matchers.HasLength(2), 'event not posted to all webhooks') # Raise another alarm self._trigger_do_action(TRIGGER_ALARM_2) # Check second event received self.assertThat(self.mock_server.requests, matchers.HasLength(4), 'event not posted to all webhooks') finally: self._trigger_undo_action(TRIGGER_ALARM_1) self._trigger_undo_action(TRIGGER_ALARM_2)
def test_basic_event(self): try: # Add webhook with filter matching alarm TempestClients.vitrage().webhook.add( url=self.URL_PROPS, regex_filter=NAME_FILTER, headers=HEADERS_PROPS ) # Raise alarm self._trigger_do_action(TRIGGER_ALARM_1) # Check event received self.assertThat(self.mock_server.requests, matchers.HasLength(1), 'Wrong number of notifications for raise alarm') # Undo self._trigger_undo_action(TRIGGER_ALARM_1) # Check event undo received self.assertThat(self.mock_server.requests, matchers.HasLength(2), 'Wrong number of notifications for clear alarm') finally: self._trigger_undo_action(TRIGGER_ALARM_1)
def test_with_no_filter(self): """Test to see that a webhook with no filter receives all notifications """ try: # Add webhook TempestClients.vitrage().webhook.add( url=self.URL_PROPS, regex_filter=NAME_FILTER, ) # Raise alarm self._trigger_do_action(TRIGGER_ALARM_1) # Check event received self.assertThat(self.mock_server.requests, matchers.HasLength(1), 'Wrong number of notifications for raise alarm') # Raise another alarm self._trigger_do_action(TRIGGER_ALARM_2) # Check second event received self.assertThat(self.mock_server.requests, matchers.HasLength(2), 'Wrong number of notifications for clear alarm') finally: self._trigger_undo_action(TRIGGER_ALARM_1) self._trigger_undo_action(TRIGGER_ALARM_2)
def test_db_init(self): try: v_utils.add_template(TEMPLATE_NAME) time.sleep(SLEEP) # 1. check template works well self._check_template_instance_3rd_degree_scenarios() # 2. check fast fail-over - start from database topo1 = TempestClients.vitrage().topology.get(all_tenants=True) v_utils.restart_graph() time.sleep(MAX_FAIL_OVER_TIME) for i in range(5): self._check_template_instance_3rd_degree_scenarios() topo2 = TempestClients.vitrage().topology.get(all_tenants=True) self.assert_graph_equal( topo1, topo2, 'comparing graph items iteration ' + str(i)) time.sleep(self.conf.datasources.snapshots_interval) v_utils.delete_template(name=TEMPLATE_NAME) time.sleep(SLEEP) self._check_template_instance_3rd_degree_scenarios_deleted() except Exception as e: self._handle_exception(e) if v_utils.get_first_template(name=TEMPLATE_NAME): v_utils.delete_template(name=TEMPLATE_NAME) time.sleep(SLEEP) raise
def test_add_webhook(self): webhooks = TempestClients.vitrage().webhook.list() self.assertThat( webhooks, matchers.HasLength(self.pre_test_webhook_count), 'Amount of webhooks should be ' 'the same as before the test') created_webhook = TempestClients.vitrage().webhook.add( url="https://www.test.com", regex_filter=REGEX_PROPS, headers=HEADERS_PROPS) self.assertIsNone(created_webhook.get('ERROR'), 'webhook not ' 'created') self.assertEqual(created_webhook[HEADERS], HEADERS_PROPS, 'headers not created correctly') self.assertEqual(created_webhook[REGEX_FILTER], REGEX_PROPS, 'regex not created correctly') self.assertEqual(created_webhook[URL], "https://www.test.com", 'URL not created correctly') webhooks = TempestClients.vitrage().webhook.list() self.assertThat(webhooks, matchers.HasLength(self.pre_test_webhook_count + 1)) TempestClients.vitrage().webhook.delete(created_webhook['id'])
def test_for_deduced_alarm(self): try: # Add webhook with filter for the deduced alarm TempestClients.vitrage().webhook.add( url=self.URL_PROPS, regex_filter=NAME_FILTER_FOR_DEDUCED, headers=HEADERS_PROPS ) # Raise the trigger alarm self._trigger_do_action(TRIGGER_ALARM_WITH_DEDUCED) # Check event received - expected one for the deduced alarm # (the trigger alarm does not pass the filter). This test verifies # that the webhook is called only once for the deduced alarm. time.sleep(1) self.assertThat(self.mock_server.requests, matchers.HasLength(1), 'Wrong number of notifications for deduced alarm') # Undo self._trigger_undo_action(TRIGGER_ALARM_WITH_DEDUCED) # Check event undo received time.sleep(1) self.assertThat(self.mock_server.requests, matchers.HasLength(2), 'Wrong number of notifications ' 'for clear deduced alarm') finally: self._trigger_undo_action(TRIGGER_ALARM_WITH_DEDUCED)
def test_action_add_causal_relationship(self): try: # Do self._trigger_do_action(TRIGGER_ALARM_2) alarms = TempestClients.vitrage().alarm.list( vitrage_id=self.orig_host.get(VProps.VITRAGE_ID), all_tenants=True) self.assertTrue(len(alarms) >= 2, 'alarms %s' % str(alarms)) deduced = g_utils.first_match(alarms, **DEDUCED_PROPS) trigger = g_utils.first_match(alarms, **TRIGGER_ALARM_2_PROPS) # Get Rca for the deduced rca = TempestClients.vitrage().rca.get(deduced[VProps.VITRAGE_ID], all_tenants=True) self._check_rca(rca, [deduced, trigger], DEDUCED_PROPS) # Get Rca for the trigger rca = TempestClients.vitrage().rca.get(trigger[VProps.VITRAGE_ID], all_tenants=True) self._check_rca(rca, [deduced, trigger], TRIGGER_ALARM_2_PROPS) except Exception as e: self._handle_exception(e) raise finally: self._trigger_undo_action(TRIGGER_ALARM_2)
def get_first_host(**kwargs): try: hosts = TempestClients.vitrage().resource.list( NOVA_HOST_DATASOURCE, all_tenants=True) except Exception as e: LOG.exception("get_first_host failed with %s", e) hosts = TempestClients.vitrage().resource.list( NOVA_HOST_DATASOURCE, all_tenants=True) return g_utils.first_match(hosts, **kwargs)
def delete_template(uuid=None, **kwargs): if not uuid: template = get_first_template(**kwargs) if template: uuid = template['uuid'] else: return TempestClients.vitrage().template.delete(uuid) wait_for_status( 100, lambda _id: True if not get_first_template(uuid=_id) else False, _id=uuid)
def generate_fake_host_alarm(hostname, event_type, enabled=True): details = { 'hostname': hostname, 'source': 'fake_tempest_monitor', 'cause': 'another alarm', 'severity': 'critical', 'status': DOWN if enabled else UP, 'monitor_id': 'fake tempest monitor id', 'monitor_event_id': '111', } event_time = datetime.now() event_time_iso = event_time.isoformat() TempestClients.vitrage().event.post(event_time_iso, event_type, details)
def _validate_deduce_alarms(self, alarms, instances): """Validate alarm existence """ self.assertThat(alarms, IsNotEmpty(), 'The alarms list is empty') LOG.info("The alarms list is : " + str(json.dumps(alarms))) # Find the vitrage_id of the deduced alarms using their original id. vitrage_resources = TempestClients.vitrage().resource.list( all_tenants=False) vitrage_instance_0_id = g_utils.first_match(vitrage_resources, id=instances[0].id) vitrage_instance_1_id = g_utils.first_match(vitrage_resources, id=instances[1].id) # Find the deduced alarms based on their properties deduce_alarms_1 = g_utils.all_matches( alarms, vitrage_type=VITRAGE_DATASOURCE, name=VITRAGE_ALARM_NAME, vitrage_resource_type=NOVA_INSTANCE_DATASOURCE, vitrage_resource_id=vitrage_instance_0_id[VProps.VITRAGE_ID]) deduce_alarms_2 = g_utils.all_matches( alarms, vitrage_type=VITRAGE_DATASOURCE, name=VITRAGE_ALARM_NAME, vitrage_resource_type=NOVA_INSTANCE_DATASOURCE, vitrage_resource_id=vitrage_instance_1_id[VProps.VITRAGE_ID]) self.assertThat(alarms, matchers.HasLength(3), "Expected 3 alarms - 1 on host and 2 deduced") self.assertThat(deduce_alarms_1, matchers.HasLength(1), "Deduced alarm not found") self.assertThat(deduce_alarms_2, matchers.HasLength(1), "Deduced alarm not found")
def _check_template_instance_3rd_degree_scenarios_deleted(self): alarm_count = TempestClients.vitrage().alarm.count(all_tenants=True) self.assertEqual(0, alarm_count['SEVERE'], 'found SEVERE deduced alarms after template delete') self.assertEqual( 0, alarm_count['CRITICAL'], 'found CRITICAL deduced alarms after template delete')
def _check_deduced(self, deduced_count, deduced_props, resource_id): alarms = TempestClients.vitrage().alarm.list(vitrage_id=resource_id, all_tenants=True) deduces = g_utils.all_matches(alarms, **deduced_props) self.assertThat( deduces, matchers.HasLength(deduced_count), 'Expected %s deduces\n - \n%s\n - \n%s' % (str(deduced_count), str(alarms), str(deduces)))
def check_rca(alarm): rca = TempestClients.vitrage().rca.get(alarm['vitrage_id'], all_tenants=True) try: self._check_rca(rca, expected_rca, alarm) return True except Exception as e: LOG.exception('check_rca failed', e) return False
def _check_template_instance_3rd_degree_scenarios(self): try: alarm_count = TempestClients.vitrage().alarm.count( all_tenants=True) self.assertEqual( self.conf.mock_graph_datasource.instances_per_host, alarm_count['SEVERE'], 'Each instance should have one SEVERE deduced alarm') self.assertEqual( self.conf.mock_graph_datasource.instances_per_host, alarm_count['CRITICAL'], 'Each instance should have one CRITICAL deduced alarm') expected_rca = [{VertexProperties.VITRAGE_TYPE: 'zabbix'}] * self.\ conf.mock_graph_datasource.zabbix_alarms_per_host expected_rca.extend([{'name': DEDUCED_1}, {'name': DEDUCED_2}]) def check_rca(alarm): rca = TempestClients.vitrage().rca.get(alarm['vitrage_id'], all_tenants=True) try: self._check_rca(rca, expected_rca, alarm) return True except Exception as e: LOG.exception('check_rca failed', e) return False # 10 threads calling rca api alarms = TempestClients.vitrage().alarm.list(all_tenants=True, vitrage_id='all') deduced_alarms = g_utils.all_matches(alarms, vitrage_type='vitrage', name=DEDUCED_2) workers = futures.ThreadPoolExecutor(max_workers=10) workers_result = [ r for r in workers.map(check_rca, deduced_alarms) ] self.assertTrue(all(workers_result)) except Exception as e: v_utils.delete_template(name=TEMPLATE_NAME) self._handle_exception(e) raise
def test_delete_webhook(self): webhooks = TempestClients.vitrage().webhook.list() self.assertThat( webhooks, matchers.HasLength(self.pre_test_webhook_count), 'Amount of webhooks should ' 'be the same as before the test') created_webhook = TempestClients.vitrage().webhook.add( url="https://www.test.com", regex_filter=REGEX_PROPS, headers=HEADERS_PROPS) created_webhook = TempestClients.vitrage().webhook.delete( id=created_webhook['id']) self.assertIsNotNone(created_webhook.get('SUCCESS'), 'failed to delete') self.assertThat(webhooks, matchers.HasLength(self.pre_test_webhook_count), 'No webhooks should exist after deletion')
def _print_entity_graph(self): api_graph = TempestClients.vitrage().topology.get(all_tenants=True) graph = self._create_graph_from_graph_dictionary(api_graph) node_link_data = json_graph.node_link_data(graph._g) for index, node in enumerate(node_link_data['nodes']): if VProps.ID in graph._g.node[node[VProps.ID]]: node[VProps.ID] = graph._g.node[node[VProps.ID]][VProps.ID] node[VProps.GRAPH_INDEX] = index LOG.info('Entity Graph: \n%s', json.dumps(node_link_data))
def test_list_webhook(self): webhooks = TempestClients.vitrage().webhook.list() self.assertThat( webhooks, matchers.HasLength(self.pre_test_webhook_count), 'Amount of webhooks should be ' 'the same as before the test') created_webhook = TempestClients.vitrage().webhook.add( url="https://www.test.com", regex_filter=REGEX_PROPS, headers=HEADERS_PROPS) webhooks = TempestClients.vitrage().webhook.list() self.assertThat(webhooks, matchers.HasLength(self.pre_test_webhook_count + 1)) self.assertEqual(created_webhook[HEADERS], webhooks[0][HEADERS]) self.assertEqual(created_webhook['id'], webhooks[0]['id']) self.assertEqual(created_webhook[REGEX_FILTER], webhooks[0][REGEX_FILTER]) TempestClients.vitrage().webhook.delete(created_webhook['id'])
def add_template(filename='', folder='templates/api', template_type=TemplateTypes.STANDARD, status=TemplateStatus.ACTIVE): full_path = g_utils.tempest_resources_dir() + '/' + folder + '/' + filename t = TempestClients.vitrage().template.add(full_path, template_type) if t and t[0]: wait_for_status( 100, get_first_template, uuid=t[0]['uuid'], status=status) return t[0] return None
def test_show_webhook(self): webhooks = TempestClients.vitrage().webhook.list() self.assertThat( webhooks, matchers.HasLength(self.pre_test_webhook_count), 'Amount of webhooks should be ' 'the same as before the test') created_webhook = TempestClients.vitrage().webhook.add( url="https://www.test.com", regex_filter=REGEX_PROPS, headers=HEADERS_PROPS) show_webhook = TempestClients.vitrage().webhook.show( created_webhook['id']) self.assertIsNotNone(show_webhook, 'webhook not listed') self.assertEqual(created_webhook[HEADERS], show_webhook[HEADERS], 'headers mismatch') self.assertEqual(created_webhook[REGEX_FILTER], show_webhook[REGEX_FILTER], 'regex mismatch') self.assertEqual(created_webhook[URL], show_webhook[URL], 'URL mismatch') TempestClients.vitrage().webhook.delete(created_webhook['id'])
def setUpClass(cls): super(BaseVitrageTempest, cls).setUpClass() warnings.filterwarnings(action="ignore", message="unclosed", category=ResourceWarning) TempestClients.class_init(cls.os_primary.credentials) cls.vitrage_client = TempestClients.vitrage() cls.vitrage_client_for_demo_user = \ TempestClients.vitrage_client_for_user() cls.num_default_networks = \ len(TempestClients.neutron().list_networks()['networks']) cls.num_default_ports = cls._get_num_default_ports() cls.num_default_entities = 3 cls.num_default_edges = 2 cls.num_demo_tenant_networks = cls._calc_num_tenant_networks()
def test_high_availability_events(self): """The purpose of the test is to check that events are stored That is, during different stages in vitrage-graph lifetime: before graph read from db (during init) after graph read from db (during init) during get_all after get_all """ try: # adding a template just to create more load (to slow things down) v_utils.add_template(TEMPLATE_NAME) time.sleep(SLEEP) self.keep_sending_events = True self.num_of_sent_events = 0 doctor_events_thread = self._async_doctor_events() time.sleep(10) v_utils.stop_graph() time.sleep(10) v_utils.restart_graph() v_utils.delete_template(name=TEMPLATE_NAME) # sleep to allow get_all to start and finish at least once: time.sleep(4 * self.conf.datasources.snapshots_interval) v_utils.restart_graph() self.keep_sending_events = False time.sleep(MAX_FAIL_OVER_TIME) doctor_events_thread.join(timeout=10) alarm_count = TempestClients.vitrage().alarm.count( all_tenants=True) self.assertTrue(self.num_of_sent_events > 0, 'Test did not create events') self.assertEqual(self.num_of_sent_events, alarm_count['CRITICAL'], 'CRITICAL doctor events expected') except Exception as e: self._handle_exception(e) raise finally: self._remove_doctor_events() if v_utils.get_first_template(name=TEMPLATE_NAME): v_utils.delete_template(name=TEMPLATE_NAME) time.sleep(SLEEP)
def setUpClass(cls): super(BaseVitrageTempest, cls).setUpClass() warnings.filterwarnings(action="ignore", message="unclosed", category=ResourceWarning) cls.conf = service.prepare_service([]) TempestClients.class_init(cls.conf) cls.vitrage_client = TempestClients.vitrage() cls.vitrage_client_for_demo_user = \ TempestClients.vitrage_client_for_user( cls.DEMO_USERNAME, cls.USER_DOMAIN_ID, cls.DEMO_PROJECT_NAME, cls.PROJECT_DOMAIN_ID) cls.num_default_networks = \ len(TempestClients.neutron().list_networks()['networks']) cls.num_default_ports = cls._get_num_default_ports() cls.num_default_entities = 3 cls.num_default_edges = 2 cls.num_demo_tenant_networks = cls._calc_num_demo_tenant_networks()
def test_compare_cli_vs_api_alarms(self): """Wrapper that returns a test graph.""" try: instances = nova_utils.create_instances(num_instances=1, set_public_network=True) self.assertThat(instances, IsNotEmpty(), 'The instances list is empty') aodh_utils.create_aodh_alarm(resource_id=instances[0].id, name='tempest_aodh_test') api_alarms = TempestClients.vitrage().alarm.list(vitrage_id='all', all_tenants=True) cli_alarms = utils.run_vitrage_command('vitrage alarm list', self.conf) self._compare_alarms_lists(api_alarms, cli_alarms, AODH_DATASOURCE, instances[0].id) except Exception as e: self._handle_exception(e) raise finally: aodh_utils.delete_all_aodh_alarms() nova_utils.delete_all_instances()
def test_validate_notifier(self): """validate_notifier test There tests validates work of aodh alarm notifier - all created vitrage alarms appears in ceilometer alarms-list. IMPORTANT: enable notifiers=aodh in vitrage.conf file """ try: nova_utils.create_instances(num_instances=2, set_public_network=True) self._create_alarm(resource_id=self._get_hostname(), alarm_name=RCA_ALARM_NAME) vitrage_alarms = TempestClients.vitrage().alarm.list( vitrage_id='all', all_tenants=True) aodh_alarms = TempestClients.aodh().alarm.list() self._validate_notifier(alarms=aodh_alarms, vitrage_alarms=vitrage_alarms) except Exception as e: self._handle_exception(e) raise finally: self._clean_all()
def _delete_webhooks(): webhooks = TempestClients.vitrage().webhook.list() for webhook in webhooks: TempestClients.vitrage().webhook.delete(webhook['id'])
def get_first_template(**kwargs): templates = TempestClients.vitrage().template.list() return g_utils.first_match(templates, **kwargs)
def get_first_instance(**kwargs): instances = TempestClients.vitrage().resource.list( NOVA_INSTANCE_DATASOURCE, all_tenants=True) return g_utils.first_match(instances, **kwargs)
def test_payload_format(self): try: TempestClients.vitrage().webhook.add( url=self.URL_PROPS, headers=HEADERS_PROPS ) # Raise the trigger alarm self._trigger_do_action(TRIGGER_ALARM_1) # pre check that correct amount of notifications sent self.assertThat(self.mock_server.requests, matchers.HasLength(1), 'Wrong number of notifications for alarm') self.assertThat(messages, matchers.HasLength(1), 'Wrong number of messages for alarm') alarm = ast.literal_eval(messages[0]) # check that only specified fields are sent for the alarm, # payload and resource passed_filter = utils.filter_data(alarm, MAIN_FILTER, match_filter=False) self.assertThat(passed_filter, IsEmpty(), "Wrong main fields sent") payload = alarm.get(PAYLOAD) if payload: passed_filter = utils.filter_data(payload, DOCTOR_ALARM_FILTER, match_filter=False) self.assertThat(passed_filter, IsEmpty(), "Wrong alarm fields sent") sent_fields = utils.filter_data(payload, DOCTOR_ALARM_FILTER) self.assertEqual(DOCTOR_ALARM_FILTER, sent_fields, "Some alarm fields not sent") resource = payload.get(VProps.RESOURCE) if resource: passed_filter = utils.filter_data(resource, RESOURCE_FILTER, match_filter=False) self.assertThat(passed_filter, IsEmpty(), "Wrong resource fields sent") sent_fields = utils.filter_data(resource, RESOURCE_FILTER) self.assertEqual(RESOURCE_FILTER, sent_fields, "Some resource fields not sent") finally: self._trigger_undo_action(TRIGGER_ALARM_1)