class ProcessFault(object):
def __init__(self, session, params, env):
self.session = session
self.params = params
self.env = env
self.nodes_service_infos = []
self.service_name = self.params.get('fault_service_name')
self.select_policy = self.params.get('select_policy')
self.select_count = int(self.params.get('select_count'))
self.cloud_manager = CloudManager(self.params, self.env)
def setup(self):
self.nodes_service_infos = self.cloud_manager.get_nodes_service_info(
self.service_name, 'all', self.select_count)
def test(self):
return self.cloud_manager.act_to_process(
self.params.get('fault_action'), self.service_name,
self.params.get('fault_scale'), self.select_policy,
self.select_count)
def teardown(self):
nodes_service_infos_after_act = self.cloud_manager.get_nodes_service_info(
self.service_name, 'all', self.select_count)
ret = False
for node_service_info in self.nodes_service_infos:
for node_service_info_after_act in nodes_service_infos_after_act:
if node_service_info[
'node'].host == node_service_info_after_act[
'node'].host:
if len(node_service_info['service_info']['child_pids']) \
!= len(node_service_info_after_act['service_info']['child_pids']):
LOG.error(
"The killed pids were not automatically restarted!"
)
time.sleep(int(self.params.get('recover_time')))
LOG.info(
"Waiting 60s so that the process can be started automatically!"
)
ret = False
break
else:
ret = True
if not ret:
nodes_service_infos_after_act = self.cloud_manager.get_nodes_service_info(
self.service_name, 'all', self.select_count)
for node_service_info in self.nodes_service_infos:
for node_service_info_after_act in nodes_service_infos_after_act:
if node_service_info['node'].host \
== node_service_info_after_act['node'].host:
if len(node_service_info['service_info']['child_pids']) \
!= len(node_service_info_after_act['service_info']['child_pids']):
LOG.error(
"The killed pids were not automatically restarted!"
)
return False
LOG.info("The killed pids were automatically restarted!")
return True
def __init__(self, session, params=None, env=None):
self.params = params
self.env = env
roller_node = CloudNode(self.params.get('roller_ip'))
self.session = roller_node.get_ssh_session()
cloud_manager = CloudManager(params, env)
compute_node = cloud_manager.get_compute_node()
LOG.info("compute node is %s" % compute_node[0].host)
self.cmd = "ssh -q %s -t " % compute_node[0].host
self.fio_dir = ""
self.dstpath = '/root'
self.workload_path = ('%s' % data_dir.RELIABILITY_TEST_DIR) + \
"/workload"
self.fio_version = self.params.get('fio_version')
self.fio_working_path = \
self.fio_version[0:len(self.fio_version)-len('.tar.gz')]
self.mount_point = self.params.get('io_fault_mount_point')
self.partition = self.params.get('partition')
self.fio_filename = self.params.get('fio_filename')
self.warning_base = self.params.get('warning_base', '80')
self.disk_left = self.params.get('disk_left')
self.direct = self.params.get('direct', '0')
self.rw = self.params.get('rw', 'write')
self.bs = self.params.get('fio_bs')
self.numjobs = self.params.get('fio_numjobs')
self.report_name = self.params.get('report_name', 'fio_write')
def setup(self):
self.cloud_manager = CloudManager(self.params, self.env)
self.bridge_name = self.params.get('br_name')
self.fault_action = self.params.get("fault_action")
self.flash_count = self.params.get("flash_count")
self.flash_time = self.params.get("flash_time")
self.select_policy = self.params.get("select_policy", "random")
self.select_count = int(self.params.get("select_count", 1))
def __init__(self, session, params, env):
self.session = session
self.params = params
self.env = env
self.nodes_service_infos = []
self.service_name = self.params.get('fault_service_name')
self.select_policy = self.params.get('select_policy')
self.select_count = int(self.params.get('select_count'))
self.cloud_manager = CloudManager(self.params, self.env)
def __init__(self, params, env):
self.params = params
self.env = env
self.logger = logging.getLogger("avocado.test")
self.ret_flag = 0
self.ret_value_list = []
self.injection_list = []
self.session_list = []
self.injection_objects = []
self.cloud_manager = CloudManager(params, env)
self._init_module_and_environment()
self.threads = []
self.nodes_list = []
def setup(self):
if ((self.params["process_name"] is None)
and (self.params["service_name"] is None)
and (self.params["fault_action"] is None)
and (self.params["node_role"] is None)
and (self.params["node_status"] is None)):
raise Exception("Params fault.")
self.cloudmanager = CloudManager(self.params, self.env)
self.nodes = self.cloudmanager.get_nodes(
node_role=self.params["node_role"],
service_name=self.params["service_name"],
select_policy=self.params["select_policy"],
node_status=self.params["node_status"])
if self.nodes is None:
raise Exception("No valid node to inject.")
class Injection(object):
def __init__(self, params, env):
self.params = params
self.env = env
self.logger = logging.getLogger("avocado.test")
self.ret_flag = 0
self.ret_value_list = []
self.injection_list = []
self.session_list = []
self.injection_objects = []
self.cloud_manager = CloudManager(params, env)
self._init_module_and_environment()
self.threads = []
self.nodes_list = []
def _get_params_by_type(self, _type, sub_params):
_str = "%s_%s" % (_type, sub_params)
return self.params.get(_str)
def _generate_injection_object(self, injection_type):
mod_name = self._get_params_by_type(injection_type, "script")
cls_name = self._get_params_by_type(injection_type, "class_name")
injection_cls = self._get_class_from_module(mod_name, cls_name)
LOG.info("Try to initialize injection module: '%s:%s'" %
(mod_name, cls_name))
role = self._get_params_by_type(injection_type, "node_type")
select_policy = self._get_params_by_type(injection_type,
"injection_type")
service_name = self._get_params_by_type(injection_type, "service_name")
injection_nodes = self.cloud_manager.get_nodes(role, service_name,
select_policy)
injection_obj = injection_cls(injection_nodes, self.params, self.env)
return injection_obj
def _get_injection_object(self):
if self.params.get("workload_injection") in "true":
injection_obj = self._generate_injection_object("workload")
self.injection_objects.append(injection_obj)
if self.params.get("fault_injection") in "true":
injection_obj = self._generate_injection_object("fault")
self.injection_objects.append(injection_obj)
def _setup_injection_env(self):
for injection in self.injection_objects:
LOG.info("Calling setup for injection")
injection.setup()
def _init_module_and_environment(self):
try:
self._get_injection_object()
self._setup_injection_env()
except Exception, e:
if self.session_list:
self._close_all_sessions()
self.logger.error("Injection failed in init phase")
raise e
class NodeFault(object):
def __init__(self, session, params, env=None):
self.session = None
self.params = params
self.env = env or utils_env.Env(filename="/tmp/cloud_env")
self.nodes = None
def setup(self):
if ((self.params["process_name"] is None)
and (self.params["service_name"] is None)
and (self.params["fault_action"] is None)
and (self.params["node_role"] is None)
and (self.params["node_status"] is None)):
raise Exception("Params fault.")
self.cloudmanager = CloudManager(self.params, self.env)
self.nodes = self.cloudmanager.get_nodes(
node_role=self.params["node_role"],
service_name=self.params["service_name"],
select_policy=self.params["select_policy"],
node_status=self.params["node_status"])
if self.nodes is None:
raise Exception("No valid node to inject.")
def test(self):
def check_dead_with_ping():
cmd_check_boot = "ping -c 5 %s" % self.nodes[i].host
result = process.run(cmd_check_boot,
shell=True,
ignore_status=True,
verbose=False)
if result.exit_status is 0:
return False
else:
return True
results = []
try:
for i in range(len(self.nodes)):
if self.params["fault_action"] == "crash":
self.nodes[i].panic()
result = utils_misc.wait_for(
check_dead_with_ping,
self.params["wait_recovery_timeout"],
first=0)
if self.params["fault_action"] == "reboot":
self.nodes[i].soft_reboot()
result = utils_misc.wait_for(
check_dead_with_ping,
self.params["wait_recovery_timeout"],
first=0)
if self.params["fault_action"] == "shutdown":
self.nodes[i].poweroff()
results.append(result)
return results
except Exception, e:
LOG.error("%s" % e)
class NetworkPartition(object):
def __init__(self, session, params, env):
self.session = session
self.params = params
self.env = env
self.partition_ports = []
self.service_name = self.params.get('fault_service_name')
self.select_policy = self.params.get('select_policy')
self.select_count = int(self.params.get('select_count'))
self.cloud_manager = CloudManager(self.params, self.env)
def setup(self):
service_port = self.params.get("service_port")
if service_port:
self.partition_ports = service_port.split(',')
def test(self):
act_rule = {'action':'A', 'port': self.partition_ports}
# inject fault by inserting iptables rules
ret, self.nodes = self.cloud_manager.act_to_iptables(
act_rule,
self.service_name,
self.select_policy,
self.select_count)
def teardown(self):
"""
Delete the added iptables rules
:return:
"""
rets = []
act_rule = {'action': 'D', 'port': self.partition_ports}
for node in self.nodes:
rets.append(node.act_to_iptables(act_rule))
return all(rets)
def _runTest(self):
params = self.params
# If a dependency test prior to this test has failed, let's fail
# it right away as TestNA.
if params.get("dependency_failed") == 'yes':
raise exceptions.TestNotFoundError("Test dependency failed")
utils_misc.set_openstack_environment()
# Report cloud test version
# logging.info(version.get_pretty_version_info())
# Report the parameters we've received and write them as keyvals
self.log.info("Test parameters:")
keys = params.keys()
keys.sort()
for key in keys:
if key != 'test_cases':
self.log.info(" %s = %s", key, params[key])
self.ct_type = self.params.get('ct_type')
test_script = self.params.get('script')
class_name = self.params.get('class_name')
# Import the module
mod_name = 'cloudtest.tests.nfv.%s' % test_script
test_module = importlib.import_module(mod_name)
for _, obj in inspect.getmembers(test_module):
if (inspect.isclass(obj) and obj.__name__ == class_name
and inspect.getmodule(obj) == test_module):
test_class = obj
break
self.log.info("Initialize test class: %s" % class_name)
env_filename = os.path.join(data_dir.get_tmp_dir(),
params.get("env", "env"))
env = utils_env.Env(env_filename, self.env_version)
self.runner_queue.put({
"func_at_exit": utils_env.cleanup_env,
"args": (env_filename, self.env_version),
"once": True
})
comp = test_class(params, env)
test_passed = False
try:
try:
try:
# Preprocess
# try:
# params = env_process.preprocess(self, params, env)
# finally:
# self.__safe_env_save(env)
quotas_to_update = params.get('quotas_need_to_update', '')
if quotas_to_update:
self.log.info('Need to update quotas before test: %s' %
quotas_to_update)
compute_utils = Compute(self.params)
quotas_to_update = quotas_to_update.split(',')
expected_quota = dict()
for q in quotas_to_update:
k, v = q.split(':')
expected_quota[k] = v
compute_utils.update_quotas(expected_quota)
# Initialize injection tests including workload and fault
need_injection = (
self.params.get('workload_injection') in 'true'
or self.params.get('fault_injection') in 'true')
force_injection = (self.params.get(
'workload_injection_force',
'false') in 'true' or self.params.get(
'fault_injection_force', 'false') in 'true')
if need_injection:
injector = utils_injection.Injection(params, env)
if not injector:
self.log.error("Failed to initialize injection")
if force_injection:
raise Exception("Failed to inject"
"workload and/or fault")
if not injector.start() and force_injection:
msg = "Failed to inject workload/fault"
raise exceptions.InjectionFail(msg)
# Sleep specified time after injection
delay = int(params.get('sleep_after_injection', 3))
logging.info("Sleep %d seconds before running test" %
delay)
time.sleep(delay)
# Run the test function
self.log.info(
"Start to run NFV test: '%s:%s:%s'" %
(test_script, class_name, params.get('func_name')))
try:
comp.setup()
func = getattr(comp, params.get('func_name', 'test'))
func()
finally:
self.__safe_env_save(env)
except Exception, e:
# try:
# env_process.postprocess_on_error(self, params, env)
# finally:
# self.__safe_env_save(env)
stacktrace.log_exc_info(sys.exc_info(),
logger='avocado.test')
logging.debug("Exception happened during running test")
raise e
finally:
try:
comp.teardown()
except Exception, e:
self.log.error("Exception happened during teardown: %s" %
e)
# Postprocess
try:
try:
# Stop injection
if need_injection:
injector.stop()
params['test_passed'] = str(test_passed)
# env_process.postprocess(self, params, env)
error_message = funcatexit.run_exitfuncs(
env, self.ct_type)
if error_message:
logging.error(error_message)
except Exception, e:
if test_passed:
raise
self.log.error(
"Exception raised during "
"postprocessing: %s", e)
finally:
if self.__safe_env_save(env):
env.destroy() # Force-clean as it can't be stored
if params.get('sleep_after_test') is not None:
time.sleep(int(params.get('sleep_after_test')))
# Perform health check after test if needed
health_check = \
self.params.get("perform_health_check_after_case")
health_check = health_check.lower() == "true"
if health_check:
self._stop_logging()
self._start_logging_hc()
cloud_manager = CloudManager(params, env)
nodes = cloud_manager.get_controller_node(
select_policy="all")
nodes.extend(
cloud_manager.get_compute_node(
select_policy="all"))
self._run_health_check(nodes)
self._stop_logging_hc()
self._start_logging()
except Exception, e:
if params.get("abort_on_error") != "yes":
raise
# Abort on error
self.log.info("Aborting job (%s)", e)
class NetworkFault(object):
def __init__(self, session, params, env):
self.session = session
self.params = params
self.env = env
self.flag = False # Using for check fault action is tc cmd
self.nodes = []
def setup(self):
self.cloud_manager = CloudManager(self.params, self.env)
self.bridge_name = self.params.get('br_name')
self.fault_action = self.params.get("fault_action")
self.flash_count = self.params.get("flash_count")
self.flash_time = self.params.get("flash_time")
self.select_policy = self.params.get("select_policy", "random")
self.select_count = int(self.params.get("select_count", 1))
def test(self):
ret = False
for fault in FAULT_LIST:
if fault in self.fault_action:
self.flag = True
if self.flag:
ret, self.nodes = self.cloud_manager.act_to_net(
self.bridge_name, 'add', self.fault_action, self.select_policy,
self.select_count)
elif self.fault_action == DOWN:
ret = self.__network_down()
elif self.fault_action == INTERFACE_FLASH:
ret = self.__test_network_flash()
return ret
def teardown(self):
rets = []
for node in self.nodes:
if self.flag:
rets.append(
node.traffic_control(self.fault_action, self.bridge_name,
'del'))
elif self.fault_action == DOWN:
rets.append(node.act_to_network(self.bridge_name, 'up'))
return all(rets)
def __test_network_flash(self):
count = 0
rets = []
while count < self.flash_count:
ret1 = self.__network_down()
time.sleep(self.flash_time)
for node in self.nodes:
rets.append(node.act_to_network(self.bridge_name, 'up'))
ret2 = all(rets)
count = count + 1
if False in (ret1, ret2):
return False
return True
def __network_down(self):
# TODO: not run test
ret, self.nodes = self.cloud_manager.act_to_net(
self.bridge_name,
'down',
select=self.select_policy,
count=self.select_count)
return ret
def setup(self):
cloud_manager = CloudManager(self.params, self.env)
compute_node = cloud_manager.get_compute_node()
self.compute_ip = compute_node[0].ip
self.controller_ip = cloud_manager.get_controller_node()[0].ip