def get_ha_protect_service_info(host_ip, host_password, config_path, master=True):
session = RemoteRunner(client='ssh', host=host_ip, username="******",
port="22", password=host_password)
cmd = "ls %s|grep -E 'nova|neutron|rabbitmq|keystone' " % config_path
result = session.run(cmd)
config_files = result.stdout.split("\n")
LOG.info("Haproxy config file list: %s" % config_files)
if len(config_files) < 2:
config_file = config_files[0]
else:
config_file = config_files[random.randint(0, len(config_files)-2)]
cmd = "grep 'listen' %s%s" % (config_path, config_file)
result = session.run(cmd)
service_name = result.stdout.split(' ')[1]
tmp = re.findall(r'-\d+', service_name)
if tmp:
service_name = service_name.split(tmp[0])[0]
cmd = "grep 'server' %s%s" % (config_path, config_file)
result = session.run(cmd)
LOG.info("Service info : %s" % result.stdout)
info_list = re.findall(r"\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}"
r"(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b",
result.stdout)
if master:
info = info_list[0]
else:
info = info_list[1]
LOG.info("Service info: %s, %s" % (service_name, info))
return service_name.strip(), info
def __init__(self, nodes, params=None, env=None):
self.params = params
self.env = env
#self.session = session
storage_host = self.params.get('storage_host')
storage_host_password = self.params.get('storage_host_password')
if storage_host_password == "" or storage_host_password == None:
self.session = CloudNode(storage_host).get_ssh_session()
else:
self.session = RemoteRunner(client='ssh',
host=storage_host,
username="******",
port="22",
password=storage_host_password)
self.fio_dir = ""
self.dstpath = '/root'
self.workload_path = ('%s' % data_dir.RELIABILITY_TEST_DIR) + \
"/workload"
self.fio_version = self.params.get('fio_version')
self.fio_working_path = \
self.fio_version[0:len(self.fio_version) - len('.tar.gz')]
self.fio_filename = self.params.get('fio_parameter_filename')
self.direct = self.params.get('fio_parameter_direct')
self.bs = self.params.get('fio_parameter_bs')
self.size = self.params.get('fio_parameter_size')
self.numjobs = self.params.get('fio_parameter_numjobs')
self.report_name = self.params.get('fio_parameter_report_name')
self.runtime = self.params.get("fio_parameter_runtime")
self.cmd = ""
def kill_kvm_process(self, process_id, host, cont_ip, password):
session = RemoteRunner(client='ssh',
host=cont_ip,
username="******",
port="22",
password=password)
ssh_cmd = "ssh -q %s " % host
cmd = "'kill -9 %s'" % process_id
process_info = session.run(ssh_cmd + cmd)
self.log.info("kill process id exit num is %s" %
process_info.exit_status)
def get_kvm_process_id(self, instance_name, host, cont_ip, password):
session = RemoteRunner(client='ssh',
host=cont_ip,
username="******",
port="22",
password=password)
ssh_cmd = "ssh -q %s " % host
cmd = "'ps -aux|grep kvm| grep %s'" % instance_name
process_info = session.run(ssh_cmd + cmd)
process_id = process_info.stdout.split()[1]
self.log.info("KVM process id is %s" % process_id)
return process_id
def get_session_computenode(self, ip, usekey=False):
session_node = RemoteRunner(
client='ssh',
host=ip,
username=self.params["openstack_ssh_username"],
port="22",
password=self.params["openstack_ssh_password"],
use_key=usekey,
timeout=int(self.params["session_timeout"]))
return session_node
def get_session_computenode(self, ip, usekey=False):
if self.params.has_key("usekey_node"):
if self.params.get("usekey_node") in "False":
_usekey = False
if self.params.get("usekey_node") in "True":
_usekey = True
else:
_usekey = usekey
session = RemoteRunner(client='ssh',
host=ip,
username=self.computenode_username,
port="22",
password=self.computenode_password,
use_key=_usekey,
timeout=self.session_timeout,
internal_timeout=self.session_timeout)
return session
== node_service_info_after_act['node'].host:
if len(node_service_info['service_info']['child_pids']) \
!= len(node_service_info_after_act['service_info']['child_pids']):
LOG.error(
"The killed pids were not automatically restarted!"
)
return False
LOG.info("The killed pids were automatically restarted!")
return True
if __name__ == '__main__':
session = RemoteRunner(client='ssh',
host="10.100.4.105",
username="******",
port="22",
password="******")
env = utils_env.Env(filename='/tmp/cloud_env')
# params = {"process_name": "nova-api", "fault_action": "SIGKILL", "scale_type": "random"}
params = {
'roller_user': '******',
'roller_ip': '10.20.0.2',
"fault_service_name": "openstack-keystone",
"fault_action": "SIGSTOP",
"fault_scale": "random"
}
cf = ProcessFault(session, params, env)
cf.setup()
cf.test()
cf.teardown()
def _get_ssh_session(self):
return RemoteRunner(host=self.ip, use_key=True)
class FIO(object):
def __init__(self, nodes, params=None, env=None):
self.params = params
self.env = env
#self.session = session
storage_host = self.params.get('storage_host')
storage_host_password = self.params.get('storage_host_password')
if storage_host_password == "" or storage_host_password == None:
self.session = CloudNode(storage_host).get_ssh_session()
else:
self.session = RemoteRunner(client='ssh',
host=storage_host,
username="******",
port="22",
password=storage_host_password)
self.fio_dir = ""
self.dstpath = '/root'
self.workload_path = ('%s' % data_dir.RELIABILITY_TEST_DIR) + \
"/workload"
self.fio_version = self.params.get('fio_version')
self.fio_working_path = \
self.fio_version[0:len(self.fio_version) - len('.tar.gz')]
self.fio_filename = self.params.get('fio_parameter_filename')
self.direct = self.params.get('fio_parameter_direct')
self.bs = self.params.get('fio_parameter_bs')
self.size = self.params.get('fio_parameter_size')
self.numjobs = self.params.get('fio_parameter_numjobs')
self.report_name = self.params.get('fio_parameter_report_name')
self.runtime = self.params.get("fio_parameter_runtime")
self.cmd = ""
def setup(self):
self.install_fio(self.workload_path, self.dstpath, self.fio_version,
self.fio_working_path)
def teardown(self):
self.uninstall_fio(self.dstpath, self.fio_version,
self.fio_working_path)
def install_fio(self, srcpath, dstpath, fio_version, fio_working_path):
self.session.copy_file_to(os.path.join(srcpath, fio_version), dstpath)
cmd = "tar -zxvf %s" % fio_version
cmd = self.cmd + cmd
self.session.run(cmd)
cmd = "cd %s ; ./configure; make " % os.path.join(
dstpath, fio_working_path)
cmd = self.cmd + cmd
self.session.run(cmd)
def uninstall_fio(self, dstpath, fio_version, fio_working_path):
try:
cmd = 'pkill fio || true'
cmd = self.cmd + cmd
self.session.run(cmd)
cmd = "rm -rf %s %s" % (os.path.join(
dstpath, fio_version), os.path.join(dstpath, fio_working_path))
cmd = self.cmd + cmd
self.session.run(cmd)
except Exception, e:
return False
return True
# inject fault by inserting iptables rules
ret, self.nodes = self.cloud_manager.act_to_iptables(
act_rule,
self.service_name,
self.select_policy,
self.select_count)
def teardown(self):
"""
Delete the added iptables rules
:return:
"""
rets = []
act_rule = {'action': 'D', 'port': self.partition_ports}
for node in self.nodes:
rets.append(node.act_to_iptables(act_rule))
return all(rets)
if __name__ == '__main__':
session = RemoteRunner(client='ssh', host="10.100.64.36", username="******", port="22",
password="******")
env = utils_env.Env(filename='/tmp/cloud_env')
params = {'roller_user': '******',
'roller_ip': '10.20.0.2',
'fault_service_name': 'openstack-nova-api',
'service_port': '8774,8775'}
cf = NetworkPartition(session, params, env)
cf.setup()
cf.test()
cf.teardown()