def test(self): def check_dead_with_ping(): cmd_check_boot = "ping -c 5 %s" % self.nodes[i].host result = process.run(cmd_check_boot, shell=True, ignore_status=True, verbose=False) if result.exit_status is 0: return False else: return True results = [] try: for i in range(len(self.nodes)): if self.params["fault_action"] == "crash": self.nodes[i].panic() result = utils_misc.wait_for( check_dead_with_ping, self.params["wait_recovery_timeout"], first=0) if self.params["fault_action"] == "reboot": self.nodes[i].soft_reboot() result = utils_misc.wait_for( check_dead_with_ping, self.params["wait_recovery_timeout"], first=0) if self.params["fault_action"] == "shutdown": self.nodes[i].poweroff() results.append(result) return results except Exception, e: LOG.error("%s" % e)
def __wait_for_ceph_in_status(self): def is_in_status(): stdout_msg = remote.run_cmd_between_remotes( mid_host_ip=self.mid_host_ip, mid_host_user=self.mid_host_user, mid_host_password=self.mid_host_password, end_host_ip=self.end_host_ip, end_host_user=self.end_host_user, end_host_password=self.end_host_passwprd, cmd='ceph -s', timeout=1000) pat = 'health (.*)' result = re.findall(pat, stdout_msg) if len(result) > 0: if result[0] not in ('HEALTH_OK', 'HEALTH_WARN'): return False if 'HEALTH_WARN' in result[0]: pat = 'Monitor clock skew detected' war_msg = re.findall(pat, stdout_msg) if not len(war_msg): return False else: raise exceptions.TestFail('Msg data error, please check !') return True return utils_misc.wait_for(is_in_status, timeout=1000, first=0, step=50, text='Waiting for ceph in status')
def __wait_for_monitor_delete(self, cluster_id, server_id, timeout=30): def is_monitor_delete(): resp_query = self.client.query(self.cluster_id, server_id) if len(resp_query) == 0: return True return False return utils_misc.wait_for(is_monitor_delete, timeout, first=0, step=5, text='Waiting for server %s monitor delete' % server_id)
def teardown(self): if ((self.params["fault_action"] == "reboot") or (self.params["fault_action"] == "crash")): def check_alive_with_ping(): cmd_check_boot = "ping -c 5 %s" % self.nodes[i].host result = process.run(cmd_check_boot, shell=True, ignore_status=True, verbose=False) if result.exit_status is 0: return True else: return False for i in range(len(self.nodes)): utils_misc.wait_for(check_alive_with_ping, self.params["wait_recovery_timeout"], first=5)
def __check_rbd_capacity(self, rbd_name, timeout=100): def is_capacity_right(): resp = self.rbd_client.query(self.pool_id) for i in range(len(resp)): if resp[i].get('name') == rbd_name: if resp[i].get('capacity') == CAPACITY_MODIFY: return True return False return utils_misc.wait_for(is_capacity_right, timeout=timeout, first=0, step=5, text='Waiting for rbd update!')
def wait_for_available_vgroup(group_client, group_id, timeout=600): def exist_available_vgroup(): vgroups = group_client.query_logic_group(group_id) if len(vgroups) > 0: return True return False return utils_misc.wait_for(exist_available_vgroup, timeout, first=0, step=5, text='Waiting for available logic group.')
def wait_for_pool_created(client, pool_name, timeout=600): def is_pool_created(): resp = client.query() for i in range(len(resp)): if resp[i]['name'] == pool_name \ and resp[i]['state'] == 1: return True return False return utils_misc.wait_for(is_pool_created, timeout, first=0, step=5, text='Waiting for pool %s create.' % pool_name)
def wait_for_rbd_create(client, pool_id, rbd_name, timeout=100): def is_rbd_create(): resp = client.query(pool_id) for i in range(len(resp)): if resp[i]['name'] == rbd_name \ and resp[i]['status'] in 'ready': return True return False return utils_misc.wait_for(is_rbd_create, timeout, first=0, step=5, text='Waiting for rbd %s create.' % rbd_name)
def __wait_for_live_migrate(self, target_host): def is_migration(): host = self.compute_utils.get_server_host(self.vm_dpdk.name) if host == target_host: return True return False return utils_misc.wait_for( is_migration, timeout=100, first=0, step=5, text="Waiting for %s live migrate from %s to %s" % (self.vm_dpdk.name, self.host_name, target_host))
def __wait_for_write_rbd(self, pool_id, rbd_name, timeout=60): def is_rbd_create(): resp = self.rbd_client.query(pool_id) for i in range(len(resp)): if resp[i]['name'] == rbd_name \ and resp[i]['usedsize'] >= 0: return True return False return utils_misc.wait_for(is_rbd_create, timeout, first=0, step=5, text='Waiting for rbd %s write.' % rbd_name)
def wait_for_gateway_created(client, gateway_name, timeout=600): def is_gateway_created(): resp = client.query("gateway") for gateway in resp['items']: if gateway['name'] == gateway_name and gateway['recovery_mode'] == 'NORMAL' and gateway['node_num'] == \ gateway['node_OK_num']: return True return False return utils_misc.wait_for(is_gateway_created, timeout, first=0, step=5, text='Waiting for gateway %s create.' % gateway_name)
def __wait_for_osd_in_status(self, status): def is_in_status(): resp = self.osd_client.get_osd_capacity(self.server_id) self.osd_list = resp['osds'] for i in range(len(self.osd_list)): osd = self.osd_list[i] if osd['osdStatus'] not in status: return False return True return utils_misc.wait_for(is_in_status, timeout=360, first=0, step=30, text='Waiting for osd in status!')
def __waite_for_backup(self, pool_id, rbd_id): def is_backup_ready(): resp = self.client.get_backup_list(pool_id=pool_id, rbd_id=rbd_id) for backup in resp: if str(backup.get("rbd_id")) == rbd_id: self.env["backup_pool_id"] = str(backup.get("pool_id")) self.env["backup_rbd_id"] = str(backup.get("rbd_id")) self.env["backup_snapshot_name"] = backup.get("snap_name") self.env["backup_site"] = backup.get("site_uuid") return True return False return utils_misc.wait_for(func=is_backup_ready, timeout=1200, first=0.0, step=10.0, text="Waiting for rbd remote backup")
def _wait_for_recovery_to_begin(self, timeout=60): def is_recovery_start(): resp = self.zabbix_client.get_item_history(self.recover_item_id, 1) if not len(resp): return False LOG.info("Item ceph.cluster.recovering_bytes latest data: %s" % resp) if resp[0].get('value') != '0': return True return False return utils_misc.wait_for(is_recovery_start, timeout, first=0, step=5, text='Waiting for recovery to begin.')
def _wait_for_rbd_in_status(self, pool_id, rbd_name, status, timeout=300): status_map = {'copying': 6, 'ready': 0} def is_rbd_create(): resp = self.rbd_client.query(pool_id) for i in range(len(resp)): if resp[i]['name'] == rbd_name: if resp[i]['status'] == status_map[status]: return True return False return utils_misc.wait_for(is_rbd_create, timeout, first=0, step=5, text='Waiting for rbd %s create.' % rbd_name)
def _wait_for_pool_create(self, pool_name, timeout=1000): def is_pool_create(): resp = self.pool_client.query() for i in range(len(resp)): if resp[i]['name'] == pool_name \ and resp[i]['state'] == 1 \ and resp[i]['size'] == 3 \ and resp[i]['pg_num'] == 128: return True return False return utils_misc.wait_for(is_pool_create, timeout, first=0, step=5, text='Waiting for pool %s create.' % pool_name)
def wait_for_remote_backup_or_restore_complete(id, client, state, timeout=60): """ Wait until the remote backup or restore complete """ status_map = {'restored': 2, 'backed_up': 2, 'backing_up': 1} def _is_in_state(): extra_url = '/list_rbtasks?count=1024&begin_index=0' rbtasks = client.list_rbtasks(extra_url) for rbtask in rbtasks: if rbtask.get('id') == id: if rbtask.get('state') == status_map.get(state): return True return False return utils_misc.wait_for( _is_in_state, timeout, first=0, step=5, text='Waiting for remote backup or restore to complete!')
def get_osd_id_stateless(server_id, params, timeout=300): """ After cluster is deployed,maybe osd_capacity of server is null, then need to wait until os_capacity is filled! so I used this utils_misc.wait_for. """ osd_client = OsdClient(params) def _is_osd_in_server(): resp = osd_client.get_osd_capacity(server_id) if resp.get('osds'): return resp return False resp = utils_misc.wait_for(_is_osd_in_server, timeout, first=0, step=5, text='Waiting for osd in server %s' % server_id) for i in range(len(resp['osds'])): return resp['osds'][i]['osdId']
def wait_for_server_in_status(key, value, client, state, status, timeout=600): """ Wait until the server is in expected status :param server_name: the name of the server :param client: the instance of ServersClient :param state: expected status to wait for """ state_map = { 'stopped': 0, 'added': 1, 'deploying': 2, 'active': 3, 'maintenance': 5 } def _is_server_in_status(): query_body = {'marker': 0, 'pagesize': 1024} result = client.query(**query_body) if isinstance(result, list): servers = result else: servers = result['items'] for server in servers: if server.get(key) == value: if (server.get('state') == state_map.get(state) and server.get('status') == status): return True else: return False return False return utils_misc.wait_for(_is_server_in_status, timeout, first=0, step=5, text='Waiting for server %s is %s, ' 'and status is %d' % (value, state, status))
def wait_for_cluster_in_status(cluster_id, client, status, timeout=900): """ Wait until the cluster is deployed :param cluster_id: the ID of cluster :param client: the instance of ClustersClient :param status: expected status to wait for """ status_map = {'stopped': 2, 'deployed': 1} def _is_cluster_in_status(): resp = client.query(cluster_id=cluster_id, validate=False) if (resp.body.get('status') == status_map.get(status) and resp.body.get('state') == 1): return True return False return utils_misc.wait_for(_is_cluster_in_status, timeout, first=0, step=5, text='Waiting for cluster %d is %s' % (cluster_id, status))
def wait_for_pool_in_state(pool_id, client, state, timeout=900): """ Wait until the pool is ready for using :param client: the instance of PoolClient :param state: expected state to wait for """ status_map = {'adding': 0, 'ready': 1} def _is_pool_in_state(): pools = client.query() for pool in pools: if pool.get('id') == int(pool_id): if pool.get('state') == status_map.get(state): return True return False return utils_misc.wait_for(_is_pool_in_state, timeout, first=0, step=5, text='Waiting for pool %d is %s' % (pool_id, state))
def wait_for_disk_info_in_osd(osd_client, server_id, osd_id, timeout=120): """ Wait until disk info in osd :param client: the instance of OsdClient :param state: expected state to wait for """ resp = "" def _is_disk_info_in_osd(): try: resp = osd_client.get_osd_disk(server_id, osd_id) return True except: pass return False return utils_misc.wait_for( _is_disk_info_in_osd, timeout, first=0, step=5, text='Waiting for disk info in osd ,server id is %s,osd id %s' % (server_id, osd_id)), resp
def _check_vm_responsive(): try: cmd = params.get('test_vm_responsive_cmd', 'hostname') ssh_cmd = "ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null" ssh_cmd += " -o StrictHostKeyChecking=no -p 22 " ssh_cmd += "%s@%s '%s'" % (image_login_user, vm_ip, cmd) result = process.run(ssh_cmd, timeout=2, ignore_status=True) if image_login_user in result.stdout: return True return False except Exception, e: msg = "Exception happened during execute cmd within vm: %s" % e LOG.error(msg) return False if not utils_misc.wait_for(_check_vm_responsive, login_benchmark, text='Check execute command within VM'): raise exceptions.TestFail('Failed to run command within VM') return True def get_node_vm_count_dict(params): compute_utils = Compute(params) vm_count_dict = compute_utils.get_host_vm_count() LOG.info("VM count for each host: %s" % vm_count_dict) return vm_count_dict def get_test_network(params): """ Check if the network in config file is ready for use. If not, create one.