def test(self):
def check_dead_with_ping():
cmd_check_boot = "ping -c 5 %s" % self.nodes[i].host
result = process.run(cmd_check_boot,
shell=True,
ignore_status=True,
verbose=False)
if result.exit_status is 0:
return False
else:
return True
results = []
try:
for i in range(len(self.nodes)):
if self.params["fault_action"] == "crash":
self.nodes[i].panic()
result = utils_misc.wait_for(
check_dead_with_ping,
self.params["wait_recovery_timeout"],
first=0)
if self.params["fault_action"] == "reboot":
self.nodes[i].soft_reboot()
result = utils_misc.wait_for(
check_dead_with_ping,
self.params["wait_recovery_timeout"],
first=0)
if self.params["fault_action"] == "shutdown":
self.nodes[i].poweroff()
results.append(result)
return results
except Exception, e:
LOG.error("%s" % e)
def __wait_for_ceph_in_status(self):
def is_in_status():
stdout_msg = remote.run_cmd_between_remotes(
mid_host_ip=self.mid_host_ip,
mid_host_user=self.mid_host_user,
mid_host_password=self.mid_host_password,
end_host_ip=self.end_host_ip,
end_host_user=self.end_host_user,
end_host_password=self.end_host_passwprd,
cmd='ceph -s',
timeout=1000)
pat = 'health (.*)'
result = re.findall(pat, stdout_msg)
if len(result) > 0:
if result[0] not in ('HEALTH_OK', 'HEALTH_WARN'):
return False
if 'HEALTH_WARN' in result[0]:
pat = 'Monitor clock skew detected'
war_msg = re.findall(pat, stdout_msg)
if not len(war_msg):
return False
else:
raise exceptions.TestFail('Msg data error, please check !')
return True
return utils_misc.wait_for(is_in_status,
timeout=1000,
first=0,
step=50,
text='Waiting for ceph in status')
def __wait_for_monitor_delete(self, cluster_id, server_id, timeout=30):
def is_monitor_delete():
resp_query = self.client.query(self.cluster_id, server_id)
if len(resp_query) == 0:
return True
return False
return utils_misc.wait_for(is_monitor_delete, timeout, first=0, step=5,
text='Waiting for server %s monitor delete' %
server_id)
def teardown(self):
if ((self.params["fault_action"] == "reboot")
or (self.params["fault_action"] == "crash")):
def check_alive_with_ping():
cmd_check_boot = "ping -c 5 %s" % self.nodes[i].host
result = process.run(cmd_check_boot,
shell=True,
ignore_status=True,
verbose=False)
if result.exit_status is 0:
return True
else:
return False
for i in range(len(self.nodes)):
utils_misc.wait_for(check_alive_with_ping,
self.params["wait_recovery_timeout"],
first=5)
def __check_rbd_capacity(self, rbd_name, timeout=100):
def is_capacity_right():
resp = self.rbd_client.query(self.pool_id)
for i in range(len(resp)):
if resp[i].get('name') == rbd_name:
if resp[i].get('capacity') == CAPACITY_MODIFY:
return True
return False
return utils_misc.wait_for(is_capacity_right,
timeout=timeout, first=0, step=5,
text='Waiting for rbd update!')
def wait_for_available_vgroup(group_client, group_id, timeout=600):
def exist_available_vgroup():
vgroups = group_client.query_logic_group(group_id)
if len(vgroups) > 0:
return True
return False
return utils_misc.wait_for(exist_available_vgroup,
timeout,
first=0,
step=5,
text='Waiting for available logic group.')
def wait_for_pool_created(client, pool_name, timeout=600):
def is_pool_created():
resp = client.query()
for i in range(len(resp)):
if resp[i]['name'] == pool_name \
and resp[i]['state'] == 1:
return True
return False
return utils_misc.wait_for(is_pool_created,
timeout,
first=0,
step=5,
text='Waiting for pool %s create.' % pool_name)
def wait_for_rbd_create(client, pool_id, rbd_name, timeout=100):
def is_rbd_create():
resp = client.query(pool_id)
for i in range(len(resp)):
if resp[i]['name'] == rbd_name \
and resp[i]['status'] in 'ready':
return True
return False
return utils_misc.wait_for(is_rbd_create,
timeout,
first=0,
step=5,
text='Waiting for rbd %s create.' % rbd_name)
def __wait_for_live_migrate(self, target_host):
def is_migration():
host = self.compute_utils.get_server_host(self.vm_dpdk.name)
if host == target_host:
return True
return False
return utils_misc.wait_for(
is_migration,
timeout=100,
first=0,
step=5,
text="Waiting for %s live migrate from %s to %s" %
(self.vm_dpdk.name, self.host_name, target_host))
def __wait_for_write_rbd(self, pool_id, rbd_name, timeout=60):
def is_rbd_create():
resp = self.rbd_client.query(pool_id)
for i in range(len(resp)):
if resp[i]['name'] == rbd_name \
and resp[i]['usedsize'] >= 0:
return True
return False
return utils_misc.wait_for(is_rbd_create,
timeout,
first=0,
step=5,
text='Waiting for rbd %s write.' % rbd_name)
def wait_for_gateway_created(client, gateway_name, timeout=600):
def is_gateway_created():
resp = client.query("gateway")
for gateway in resp['items']:
if gateway['name'] == gateway_name and gateway['recovery_mode'] == 'NORMAL' and gateway['node_num'] == \
gateway['node_OK_num']:
return True
return False
return utils_misc.wait_for(is_gateway_created,
timeout,
first=0,
step=5,
text='Waiting for gateway %s create.' %
gateway_name)
def __wait_for_osd_in_status(self, status):
def is_in_status():
resp = self.osd_client.get_osd_capacity(self.server_id)
self.osd_list = resp['osds']
for i in range(len(self.osd_list)):
osd = self.osd_list[i]
if osd['osdStatus'] not in status:
return False
return True
return utils_misc.wait_for(is_in_status,
timeout=360,
first=0,
step=30,
text='Waiting for osd in status!')
def __waite_for_backup(self, pool_id, rbd_id):
def is_backup_ready():
resp = self.client.get_backup_list(pool_id=pool_id, rbd_id=rbd_id)
for backup in resp:
if str(backup.get("rbd_id")) == rbd_id:
self.env["backup_pool_id"] = str(backup.get("pool_id"))
self.env["backup_rbd_id"] = str(backup.get("rbd_id"))
self.env["backup_snapshot_name"] = backup.get("snap_name")
self.env["backup_site"] = backup.get("site_uuid")
return True
return False
return utils_misc.wait_for(func=is_backup_ready, timeout=1200,
first=0.0, step=10.0,
text="Waiting for rbd remote backup")
def _wait_for_recovery_to_begin(self, timeout=60):
def is_recovery_start():
resp = self.zabbix_client.get_item_history(self.recover_item_id, 1)
if not len(resp):
return False
LOG.info("Item ceph.cluster.recovering_bytes latest data: %s" %
resp)
if resp[0].get('value') != '0':
return True
return False
return utils_misc.wait_for(is_recovery_start,
timeout,
first=0,
step=5,
text='Waiting for recovery to begin.')
def _wait_for_rbd_in_status(self, pool_id, rbd_name, status, timeout=300):
status_map = {'copying': 6, 'ready': 0}
def is_rbd_create():
resp = self.rbd_client.query(pool_id)
for i in range(len(resp)):
if resp[i]['name'] == rbd_name:
if resp[i]['status'] == status_map[status]:
return True
return False
return utils_misc.wait_for(is_rbd_create,
timeout,
first=0,
step=5,
text='Waiting for rbd %s create.' %
rbd_name)
def _wait_for_pool_create(self, pool_name, timeout=1000):
def is_pool_create():
resp = self.pool_client.query()
for i in range(len(resp)):
if resp[i]['name'] == pool_name \
and resp[i]['state'] == 1 \
and resp[i]['size'] == 3 \
and resp[i]['pg_num'] == 128:
return True
return False
return utils_misc.wait_for(is_pool_create,
timeout,
first=0,
step=5,
text='Waiting for pool %s create.' %
pool_name)
def wait_for_remote_backup_or_restore_complete(id, client, state, timeout=60):
"""
Wait until the remote backup or restore complete
"""
status_map = {'restored': 2, 'backed_up': 2, 'backing_up': 1}
def _is_in_state():
extra_url = '/list_rbtasks?count=1024&begin_index=0'
rbtasks = client.list_rbtasks(extra_url)
for rbtask in rbtasks:
if rbtask.get('id') == id:
if rbtask.get('state') == status_map.get(state):
return True
return False
return utils_misc.wait_for(
_is_in_state,
timeout,
first=0,
step=5,
text='Waiting for remote backup or restore to complete!')
def get_osd_id_stateless(server_id, params, timeout=300):
"""
After cluster is deployed,maybe osd_capacity of server is null,
then need to wait until os_capacity is filled!
so I used this utils_misc.wait_for.
"""
osd_client = OsdClient(params)
def _is_osd_in_server():
resp = osd_client.get_osd_capacity(server_id)
if resp.get('osds'):
return resp
return False
resp = utils_misc.wait_for(_is_osd_in_server,
timeout,
first=0,
step=5,
text='Waiting for osd in server %s' % server_id)
for i in range(len(resp['osds'])):
return resp['osds'][i]['osdId']
def wait_for_server_in_status(key, value, client, state, status, timeout=600):
"""
Wait until the server is in expected status
:param server_name: the name of the server
:param client: the instance of ServersClient
:param state: expected status to wait for
"""
state_map = {
'stopped': 0,
'added': 1,
'deploying': 2,
'active': 3,
'maintenance': 5
}
def _is_server_in_status():
query_body = {'marker': 0, 'pagesize': 1024}
result = client.query(**query_body)
if isinstance(result, list):
servers = result
else:
servers = result['items']
for server in servers:
if server.get(key) == value:
if (server.get('state') == state_map.get(state)
and server.get('status') == status):
return True
else:
return False
return False
return utils_misc.wait_for(_is_server_in_status,
timeout,
first=0,
step=5,
text='Waiting for server %s is %s, '
'and status is %d' % (value, state, status))
def wait_for_cluster_in_status(cluster_id, client, status, timeout=900):
"""
Wait until the cluster is deployed
:param cluster_id: the ID of cluster
:param client: the instance of ClustersClient
:param status: expected status to wait for
"""
status_map = {'stopped': 2, 'deployed': 1}
def _is_cluster_in_status():
resp = client.query(cluster_id=cluster_id, validate=False)
if (resp.body.get('status') == status_map.get(status)
and resp.body.get('state') == 1):
return True
return False
return utils_misc.wait_for(_is_cluster_in_status,
timeout,
first=0,
step=5,
text='Waiting for cluster %d is %s' %
(cluster_id, status))
def wait_for_pool_in_state(pool_id, client, state, timeout=900):
"""
Wait until the pool is ready for using
:param client: the instance of PoolClient
:param state: expected state to wait for
"""
status_map = {'adding': 0, 'ready': 1}
def _is_pool_in_state():
pools = client.query()
for pool in pools:
if pool.get('id') == int(pool_id):
if pool.get('state') == status_map.get(state):
return True
return False
return utils_misc.wait_for(_is_pool_in_state,
timeout,
first=0,
step=5,
text='Waiting for pool %d is %s' %
(pool_id, state))
def wait_for_disk_info_in_osd(osd_client, server_id, osd_id, timeout=120):
"""
Wait until disk info in osd
:param client: the instance of OsdClient
:param state: expected state to wait for
"""
resp = ""
def _is_disk_info_in_osd():
try:
resp = osd_client.get_osd_disk(server_id, osd_id)
return True
except:
pass
return False
return utils_misc.wait_for(
_is_disk_info_in_osd,
timeout,
first=0,
step=5,
text='Waiting for disk info in osd ,server id is %s,osd id %s' %
(server_id, osd_id)), resp
def _check_vm_responsive():
try:
cmd = params.get('test_vm_responsive_cmd', 'hostname')
ssh_cmd = "ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null"
ssh_cmd += " -o StrictHostKeyChecking=no -p 22 "
ssh_cmd += "%s@%s '%s'" % (image_login_user, vm_ip, cmd)
result = process.run(ssh_cmd, timeout=2, ignore_status=True)
if image_login_user in result.stdout:
return True
return False
except Exception, e:
msg = "Exception happened during execute cmd within vm: %s" % e
LOG.error(msg)
return False
if not utils_misc.wait_for(_check_vm_responsive, login_benchmark,
text='Check execute command within VM'):
raise exceptions.TestFail('Failed to run command within VM')
return True
def get_node_vm_count_dict(params):
compute_utils = Compute(params)
vm_count_dict = compute_utils.get_host_vm_count()
LOG.info("VM count for each host: %s" % vm_count_dict)
return vm_count_dict
def get_test_network(params):
"""
Check if the network in config file is ready for use. If not, create one.
