def test_db_init(self):
try:
v_utils.add_template(TEMPLATE_NAME)
time.sleep(SLEEP)
# 1. check template works well
self._check_template_instance_3rd_degree_scenarios()
# 2. check fast fail-over - start from database
topo1 = TempestClients.vitrage().topology.get(all_tenants=True)
v_utils.restart_graph()
time.sleep(MAX_FAIL_OVER_TIME)
for i in range(5):
self._check_template_instance_3rd_degree_scenarios()
topo2 = TempestClients.vitrage().topology.get(all_tenants=True)
self.assert_graph_equal(
topo1, topo2, 'comparing graph items iteration ' + str(i))
time.sleep(self.conf.datasources.snapshots_interval)
v_utils.delete_template(name=TEMPLATE_NAME)
time.sleep(SLEEP)
self._check_template_instance_3rd_degree_scenarios_deleted()
except Exception as e:
self._handle_exception(e)
if v_utils.get_first_template(name=TEMPLATE_NAME):
v_utils.delete_template(name=TEMPLATE_NAME)
time.sleep(SLEEP)
raise
def setUpClass(cls):
super(TestBasicActions, cls).setUpClass()
cls._templates = []
cls._templates.append(
v_utils.add_template("e2e_test_basic_actions.yaml"))
cls._templates.append(
v_utils.add_template("e2e_test_basic_actions_v3.yaml"))
def setUpClass(cls):
super(TestMistralNotifier, cls).setUpClass()
cls.mistral_client = TempestClients.mistral()
cls._templates = []
cls._templates.append(v_utils.add_template('v1_execute_mistral.yaml'))
cls._templates.append(v_utils.add_template('v2_execute_mistral.yaml'))
cls._templates.append(v_utils.add_template('v3_execute_mistral.yaml'))
# Create a Mistral workflow
cls.mistral_client.workflows.create(WF_DEFINITION)
def test_template_show_with_name(self):
"""Compare template content from file to DB"""
# add standard template
template_path = \
g_utils.tempest_resources_dir() + '/templates/api/' \
+ STANDARD_TEMPLATE
v_utils.add_template(STANDARD_TEMPLATE, template_type=TTypes.STANDARD)
name = 'host_high_memory_usage_scenarios'
db_row = v_utils.get_first_template(name=name,
type=TTypes.STANDARD,
status=TemplateStatus.ACTIVE)
payload_from_db = self.vitrage_client.template.show(name)
with open(template_path, 'r') as stream:
payload_from_file = yaml.load(stream, Loader=yaml.BaseLoader)
self.assert_dict_equal(payload_from_file, payload_from_db,
"Template content doesn't match")
v_utils.delete_template(db_row['uuid'])
def test_high_availability_events(self):
"""The purpose of the test is to check that events are stored
That is, during different stages in vitrage-graph lifetime:
before graph read from db (during init)
after graph read from db (during init)
during get_all
after get_all
"""
try:
# adding a template just to create more load (to slow things down)
v_utils.add_template(TEMPLATE_NAME)
time.sleep(SLEEP)
self.keep_sending_events = True
self.num_of_sent_events = 0
doctor_events_thread = self._async_doctor_events()
time.sleep(10)
v_utils.stop_graph()
time.sleep(10)
v_utils.restart_graph()
v_utils.delete_template(name=TEMPLATE_NAME)
# sleep to allow get_all to start and finish at least once:
time.sleep(4 * self.conf.datasources.snapshots_interval)
v_utils.restart_graph()
self.keep_sending_events = False
time.sleep(MAX_FAIL_OVER_TIME)
doctor_events_thread.join(timeout=10)
alarm_count = TempestClients.vitrage().alarm.count(
all_tenants=True)
self.assertTrue(self.num_of_sent_events > 0,
'Test did not create events')
self.assertEqual(self.num_of_sent_events, alarm_count['CRITICAL'],
'CRITICAL doctor events expected')
except Exception as e:
self._handle_exception(e)
raise
finally:
self._remove_doctor_events()
if v_utils.get_first_template(name=TEMPLATE_NAME):
v_utils.delete_template(name=TEMPLATE_NAME)
time.sleep(SLEEP)
def _add_templates():
v_utils.add_template(STANDARD_TEMPLATE, template_type=TTypes.STANDARD)
v_utils.add_template(EQUIVALENCE_TEMPLATE,
template_type=TTypes.EQUIVALENCE)
v_utils.add_template(DEFINITION_TEMPLATE,
template_type=TTypes.DEFINITION)
v_utils.add_template(STANDARD_ERROR,
template_type=TTypes.STANDARD,
status=TemplateStatus.ERROR)
return [
'host_high_memory_usage_scenarios', 'entity equivalence example',
'basic_def_template', 'corrupted_template'
]
def test_high_availability_events(self):
"""The purpose of the test is to check that events are stored
That is, during different stages in vitrage-graph lifetime:
before graph read from db (during init)
after graph read from db (during init)
during get_all
after get_all
"""
try:
# adding a template just to create more load (to slow things down)
v_utils.add_template(TEMPLATE_NAME)
time.sleep(SLEEP)
self.keep_sending_events = True
self.num_of_sent_events = 0
doctor_events_thread = self._async_doctor_events()
time.sleep(10)
v_utils.stop_graph()
time.sleep(10)
v_utils.restart_graph()
time.sleep(MAX_FAIL_OVER_TIME)
v_utils.delete_template(name=TEMPLATE_NAME)
# sleep to allow get_all to start and finish at least once:
time.sleep(4 * CONF.root_cause_analysis_service.snapshots_interval)
v_utils.restart_graph()
self.keep_sending_events = False
time.sleep(MAX_FAIL_OVER_TIME)
doctor_events_thread.join(timeout=10)
alarm_count = self.vitrage_client.alarm.count(all_tenants=True)
self.assertTrue(self.num_of_sent_events > 0,
'Test did not create events')
self.assertAlmostEqual(self.num_of_sent_events,
alarm_count['CRITICAL'],
msg='CRITICAL doctor events expected',
delta=1)
finally:
self._remove_doctor_events()
def test_template_show(self):
"""Compare template content from file to DB"""
try:
# add standard template
template_path = \
g_utils.tempest_resources_dir() + '/templates/api/'\
+ STANDARD_TEMPLATE
v_utils.add_template(STANDARD_TEMPLATE,
template_type=TTypes.STANDARD)
db_row = v_utils.get_first_template(
name='host_high_memory_usage_scenarios',
type=TTypes.STANDARD,
status=TemplateStatus.ACTIVE)
payload_from_db = self.client.template.show(db_row['uuid'])
payload_from_file = file.load_yaml_file(template_path)
self.assertEqual(payload_from_file, payload_from_db,
"Template content doesn't match")
v_utils.delete_template(db_row['uuid'])
except Exception as e:
self._handle_exception(e)
raise
def test_db_init(self):
v_utils.add_template(TEMPLATE_NAME)
time.sleep(SLEEP)
# 1. check template works well
self._check_template_instance_3rd_degree_scenarios()
# 2. check fast fail-over - start from database
topo1 = self.vitrage_client.topology.get(all_tenants=True)
v_utils.restart_graph()
time.sleep(MAX_FAIL_OVER_TIME)
for i in range(5):
self._check_template_instance_3rd_degree_scenarios()
topo2 = self.vitrage_client.topology.get(all_tenants=True)
self._assert_graph_equal(topo1, topo2,
'comparing graph items iteration %s' % i)
time.sleep(CONF.root_cause_analysis_service.snapshots_interval)
v_utils.delete_template(name=TEMPLATE_NAME)
time.sleep(SLEEP)
self._check_template_instance_3rd_degree_scenarios_deleted()
def setUpClass(cls):
super(TestWebhook, cls).setUpClass()
cls._template = v_utils.add_template("e2e_webhooks.yaml")
# Configure mock server.
cls.mock_server_port = _get_free_port()
cls.mock_server = MockHTTPServer(('localhost', cls.mock_server_port),
MockServerRequestHandler)
# Start running mock server in a separate thread.
cls.mock_server_thread = Thread(target=cls.mock_server.serve_forever)
cls.mock_server_thread.setDaemon(True)
cls.mock_server_thread.start()
cls.URL_PROPS = 'http://localhost:%s/' % cls.mock_server_port
def test_evaluator_reload_with_existing_template(self):
"""Test reload new template e2e
1.add the relevant template
2.raise trigger alarm.
3. check action is executed
This checks that new workers execute new template
"""
host_id = self.orig_host.get(VProps.VITRAGE_ID)
self.added_template = v_util.add_template(TEST_TEMPLATE,
folder=FOLDER_PATH)
self._trigger_do_action(TRIGGER_ALARM_1)
self._check_deduced(1, DEDUCED_PROPS, host_id)
def test_template_delete(self):
try:
# add standard template
v_utils.add_template(STANDARD_TEMPLATE,
template_type=TTypes.STANDARD)
db_row = v_utils.get_first_template(
name='host_high_memory_usage_scenarios',
type=TTypes.STANDARD,
status=TemplateStatus.ACTIVE)
self.assertIsNotNone(db_row,
'Template should appear in templates list')
# delete template
uuid = db_row['uuid']
v_utils.delete_template(uuid)
db_row = v_utils.get_first_template(
name='host_high_memory_usage_scenarios', type=TTypes.STANDARD)
self.assertIsNone(db_row, 'Template should not appear in list')
except Exception as e:
self._handle_exception(e)
raise
def test_evaluator_reload_with_multiple_new_template(self):
"""Test reload new template e2e
1. raise trigger alarm (template is not loaded yet)
2. add 2 new templates.
3. check both actions are executed
This checks that the evaluators are reloaded for both templates
and run on all existing vertices.
"""
second_template = None
try:
host_id = self.orig_host.get(VProps.VITRAGE_ID)
self._trigger_do_action(TRIGGER_ALARM_1)
self._trigger_do_action(TRIGGER_ALARM_2)
v_util.add_template(folder=FOLDER_PATH)
self.added_template = v_util.get_first_template(name=INFILE_NAME)
second_template = v_util.get_first_template(name=INFILE_NAME_2)
self._check_deduced(1, DEDUCED_PROPS, host_id)
self._check_deduced(1, DEDUCED_PROPS_2, host_id)
finally:
if second_template:
v_util.delete_template(second_template['uuid'])
self._trigger_undo_action(TRIGGER_ALARM_1)
self._trigger_undo_action(TRIGGER_ALARM_2)
def _add_delete_template_with_name(self):
"""A helper function:
Adds and deletes a template.
Returns its name.
"""
# add a template
v_utils.add_template(STANDARD_TEMPLATE, template_type=TTypes.STANDARD)
db_row = v_utils.get_first_template(
name='host_high_memory_usage_scenarios',
type=TTypes.STANDARD,
status=TemplateStatus.ACTIVE)
self.assertIsNotNone(db_row,
'Template should appear in templates list')
# delete it
name = db_row['name']
v_utils.delete_template_with_name(name)
db_row = v_utils.get_first_template(
name='host_high_memory_usage_scenarios', type=TTypes.STANDARD)
self.assertIsNone(db_row, 'Template should not appear in list')
return name
def test_evaluator_reload_with_new_template(self):
"""Test reload new template e2e
1. raise trigger alarm (template is not loaded yet)
2. add the relevant template
3. check action is executed
This checks that the evaluators are reloaded and run on all existing
vertices.
"""
host_id = self.orig_host.get(VProps.VITRAGE_ID)
self._trigger_do_action(TRIGGER_ALARM_1)
self.added_template = v_util.add_template(TEST_TEMPLATE,
folder=FOLDER_PATH)
time.sleep(2)
self._check_deduced(1, DEDUCED_PROPS, host_id)
def test_evaluator_reload_with_existing_template_v2(self):
"""Test reload new template e2e v2
1.add the relevant template
2.delete the template
2.raise trigger alarm.
3. check no deduced alarm
This checks that template deleted properly and no action executed.
:return:
"""
host_id = self.orig_host.get(VProps.VITRAGE_ID)
self.added_template = v_util.add_template(TEST_TEMPLATE,
folder=FOLDER_PATH)
v_util.delete_template(self.added_template['uuid'])
self.added_template = None
self._trigger_do_action(TRIGGER_ALARM_1)
self._check_deduced(0, DEDUCED_PROPS, host_id)
def test_execute_mistral_more_than_once(self):
executions = self.mistral_client.executions.list()
self.assertIsNotNone(executions,
'Failed to get the list of workflow executions')
num_executions = len(executions)
# Make sure there are at least two instances in the environment
nova_utils.create_instances(num_instances=2, set_public_network=True)
num_instances = len(TempestClients.nova().servers.list())
# Add a template that executes the same Mistral workflow for every
# instance. This should immediately trigger execute_mistral actions.
template = None
try:
template = v_utils.add_template('v3_execute_mistral_twice.yaml')
finally:
if template:
v_utils.delete_template(template['uuid']) # no longer needed
time.sleep(2) # wait for the evaluator to process the new template
# Verify that there is an execution for every instance
executions = self.mistral_client.executions.list()
self.assertIsNotNone(executions,
'Failed to get the list of workflow executions')
msg = "There are %d executions. Expected number of executions: %d " \
"(old number of executions) + %d (number of instances)" % \
(len(executions), num_executions, num_instances)
self.assertThat(executions, HasLength(num_executions + num_instances),
msg)
executed_on_instances = set()
for i in range(num_instances):
# There may be many old executions in the list. The relevant ones
# are at the end. Check the last `num_instances` executions.
execution = \
self.mistral_client.executions.get(executions[-i].id)
execution_input = jsonutils.loads(execution.input)
executed_on_instances.add(execution_input['farewell'])
msg = "There are %d instances in the graph but only %d distinct " \
"executions" % (num_instances, len(executed_on_instances))
self.assertThat(executed_on_instances, HasLength(num_instances), msg)
def test_evaluator_reload_with_new_template_v2(self):
"""Test reload new template e2e v2
1. raise trigger alarm
2. add the relevant template
3. delete the template.
4. check action - should be not active.
This checks that the evaluators are reloaded and run on all existing
vertices.
Checks temporary worker that was added to delete template.
"""
host_id = self.orig_host.get(VProps.VITRAGE_ID)
self._trigger_do_action(TRIGGER_ALARM_1)
self.added_template = v_util.add_template(TEST_TEMPLATE,
folder=FOLDER_PATH)
time.sleep(2)
self._check_deduced(1, DEDUCED_PROPS, host_id)
v_util.delete_template(self.added_template['uuid'])
self.added_template = None
time.sleep(2)
self._check_deduced(0, DEDUCED_PROPS, host_id)
def setUpClass(cls):
super(TestValidate, cls).setUpClass()
cls._template = v_utils.add_template(STANDARD_TEMPLATE)
def setUpClass(cls):
super(TestMistralNotifier, cls).setUpClass()
cls.mistral_client = os_clients.mistral_client(cls.conf)
cls._templates = []
cls._templates.append(v_utils.add_template('v1_execute_mistral.yaml'))
cls._templates.append(v_utils.add_template('v2_execute_mistral.yaml'))
def setUpClass(cls):
super(TestOverlappingActions, cls).setUpClass()
cls._template = v_utils.add_template(
'e2e_test_overlapping_actions.yaml')
def setUpClass(cls):
super(TestRca, cls).setUpClass()
cls._template = v_utils.add_template('host_aodh_alarm_for_rca.yaml')
def setUpClass(cls):
super(TestBasicActions, cls).setUpClass()
cls._template = v_utils.add_template("e2e_test_basic_actions.yaml")
def setUpClass(cls):
super(TestAlarms, cls).setUpClass()
cls._template = v_utils.add_template('nagios_alarm_for_alarms.yaml')
