def test_volume_inconsistencies(self): # Examine Gluster cluster and Heketi that there is no inconsistencies out = heketi_ops.heketi_examine_gluster( self.heketi_client_node, self.heketi_server_url) if ("heketi volume list matches with volume list of all nodes" not in out['report']): self.skipTest( "heketi and Gluster are inconsistent to each other") # create volume vol = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 1, json=True) self.addCleanup( heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol['id']) # delete volume from gluster cluster directly openshift_ops.cmd_run_on_gluster_pod_or_node( self.node, "gluster vol stop %s force --mode=script" % vol['name']) openshift_ops.cmd_run_on_gluster_pod_or_node( self.node, "gluster vol delete %s --mode=script" % vol['name']) # verify that heketi is reporting inconsistencies out = heketi_ops.heketi_examine_gluster( self.heketi_client_node, self.heketi_server_url) self.assertNotIn( "heketi volume list matches with volume list of all nodes", out['report'])
def test_verify_arbiter_brick_able_to_contain_expected_amount_of_files( self, pvc_size_gb, avg_file_size): """Validate arbiter brick creation with different avg file size""" # Create sc with gluster arbiter info self.create_storage_class( is_arbiter_vol=True, arbiter_avg_file_size=avg_file_size) # Create PVC and wait for it to be in 'Bound' state self.create_and_wait_for_pvc(pvc_size_gb) # Get volume info vol_info = get_gluster_vol_info_by_pvc_name(self.node, self.pvc_name) # Verify proportion of data and arbiter bricks bricks_info = ( self.verify_amount_and_proportion_of_arbiter_and_data_bricks( vol_info)) expected_file_amount = pvc_size_gb * 1024**2 / (avg_file_size or 64) expected_file_amount = (expected_file_amount / bricks_info['arbiter_amount']) # Try to create expected amount of files on arbiter brick mount passed_arbiter_bricks = [] not_found = "Mount Not Found" for brick in bricks_info['arbiter_list']: # "brick path" looks like following: # ip_addr:/path/to/vg/brick_unique_name/brick gluster_ip, brick_path = brick["name"].split(":") brick_path = brick_path[0:-6] cmd = "mount | grep %s || echo '%s'" % (brick_path, not_found) out = cmd_run_on_gluster_pod_or_node(self.node, cmd, gluster_ip) if out != not_found: cmd = ( "python -c \"[" " open('%s/foo_file{0}'.format(i), 'a').close()" " for i in range(%s)" "]\"" % (brick_path, expected_file_amount) ) cmd_run_on_gluster_pod_or_node(self.node, cmd, gluster_ip) passed_arbiter_bricks.append(brick["name"]) # Make sure all the arbiter bricks were checked for brick in bricks_info['arbiter_list']: self.assertIn( brick["name"], passed_arbiter_bricks, "Arbiter brick '%s' was not verified. Looks like it was " "not found on any of gluster PODs/nodes." % brick["name"])
def dynamic_provisioning_glusterfile(self, create_vol_name_prefix): # Create secret and storage class self.create_storage_class( create_vol_name_prefix=create_vol_name_prefix) # Create PVC pvc_name = self.create_and_wait_for_pvc() # Create DC with POD and attached PVC to it. dc_name = oc_create_app_dc_with_io(self.node, pvc_name) self.addCleanup(oc_delete, self.node, 'dc', dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) # Verify Heketi volume name for prefix presence if provided if create_vol_name_prefix: ret = verify_volume_name_prefix(self.node, self.sc['volumenameprefix'], self.sc['secretnamespace'], pvc_name, self.sc['resturl']) self.assertTrue(ret, "verify volnameprefix failed") else: # Get the volume name and volume id from PV pv_name = get_pv_name_from_pvc(self.ocp_client[0], pvc_name) custom = [ r':spec.glusterfs.path', r':metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi-volume-id"' ] pv_vol_name, vol_id = oc_get_custom_resource( self.ocp_client[0], 'pv', custom, pv_name) # check if the pv_volume_name is present in heketi # Check if volume name is "vol_"+volumeid or not heketi_vol_name = heketi_volume_info( self.ocp_client[0], self.heketi_server_url, vol_id, json=True)['name'] self.assertEqual(pv_vol_name, heketi_vol_name, 'Volume with vol_id = %s not found' 'in heketidb' % vol_id) self.assertEqual(heketi_vol_name, 'vol_' + vol_id, 'Volume with vol_id = %s have a' 'custom perfix' % vol_id) out = cmd_run_on_gluster_pod_or_node(self.ocp_master_node[0], "gluster volume list") self.assertIn(pv_vol_name, out, "Volume with id %s does not exist" % vol_id) # Make sure we are able to work with files on the mounted volume filepath = "/mnt/file_for_testing_io.log" for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100", "ls -lrt %s", "rm -rf %s"): cmd = cmd % filepath ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to execute '%s' command on %s" % (cmd, self.node))
def _find_bricks(self, brick_paths, present): """Make sure that vol brick paths either exist or not in fstab file.""" oc_node = self.ocp_master_node[0] cmd = ( 'bash -c "' 'if [ -d "%s" ]; then echo present; else echo absent; fi"') g_hosts = list(g.config.get("gluster_servers", {}).keys()) results = [] assertion_method = self.assertIn if present else self.assertNotIn for brick_path in brick_paths: for g_host in g_hosts: out = openshift_ops.cmd_run_on_gluster_pod_or_node( oc_node, cmd % brick_path, gluster_node=g_host) results.append(out) assertion_method('present', results)
def test_check_max_brick_per_process(self): """Check if the max-brick process is set to 250""" cmd = ("gluster v get all all | grep cluster.max-bricks-per-process |" "awk '{print $2}'") # Get brick per process value bprocess_status = cmd_run_on_gluster_pod_or_node( self.ocp_master_node[0], cmd) # Validate the result err_msg = ("Got unexepeted max-brick process - '%s' " "Expected max brick process is : 250") % ( bprocess_status) self.assertIn(bprocess_status, '250', err_msg)
def test_glusterblock_logs_presence_verification(self): """Validate presence of glusterblock provisioner POD and it's status""" # Get glusterblock provisioner dc name cmd = ("oc get dc | awk '{ print $1 }' | " "grep -e glusterblock -e provisioner") dc_name = cmd_run(cmd, self.ocp_master_node[0], True) # Get glusterblock provisioner pod name and it's status gb_prov_name, gb_prov_status = oc_get_custom_resource( self.node, 'pod', custom=':.metadata.name,:.status.phase', selector='deploymentconfig=%s' % dc_name)[0] self.assertEqual(gb_prov_status, 'Running') # Create Secret, SC and PVC self.create_storage_class() self.create_and_wait_for_pvc() # Get list of Gluster nodes g_hosts = list(g.config.get("gluster_servers", {}).keys()) self.assertGreater( len(g_hosts), 0, "We expect, at least, one Gluster Node/POD:\n %s" % g_hosts) # Perform checks on Gluster nodes/PODs logs = ("gluster-block-configshell", "gluster-blockd") gluster_pods = oc_get_pods( self.ocp_client[0], selector="glusterfs-node=pod") if gluster_pods: cmd = "tail -n 5 /var/log/glusterfs/gluster-block/%s.log" else: cmd = "tail -n 5 /var/log/gluster-block/%s.log" for g_host in g_hosts: for log in logs: out = cmd_run_on_gluster_pod_or_node( self.ocp_client[0], cmd % log, gluster_node=g_host) self.assertTrue(out, "Command '%s' output is empty." % cmd)
def test_glusterblock_logs_presence_verification(self): """Validate presence of glusterblock provisioner POD and it's status""" gb_prov_cmd = ("oc get pods --all-namespaces " "-l glusterfs=block-%s-provisioner-pod " "-o=custom-columns=:.metadata.name,:.status.phase" % ( self.storage_project_name)) ret, out, err = g.run(self.ocp_client[0], gb_prov_cmd, "root") self.assertEqual(ret, 0, "Failed to get Glusterblock provisioner POD.") gb_prov_name, gb_prov_status = out.split() self.assertEqual(gb_prov_status, 'Running') # Create Secret, SC and PVC self.create_storage_class() self.create_and_wait_for_pvc() # Get list of Gluster nodes g_hosts = list(g.config.get("gluster_servers", {}).keys()) self.assertGreater( len(g_hosts), 0, "We expect, at least, one Gluster Node/POD:\n %s" % g_hosts) # Perform checks on Gluster nodes/PODs logs = ("gluster-block-configshell", "gluster-blockd") gluster_pods = oc_get_pods( self.ocp_client[0], selector="glusterfs-node=pod") if gluster_pods: cmd = "tail -n 5 /var/log/glusterfs/gluster-block/%s.log" else: cmd = "tail -n 5 /var/log/gluster-block/%s.log" for g_host in g_hosts: for log in logs: out = cmd_run_on_gluster_pod_or_node( self.ocp_client[0], cmd % log, gluster_node=g_host) self.assertTrue(out, "Command '%s' output is empty." % cmd)
def restart_gluster_vol_brick_processes(ocp_client_node, file_vol, gluster_nodes): """Restarts brick process of a file volume. Args: ocp_client_node (str): Node to execute OCP commands on. file_vol (str): file volume name. gluster_nodes (str/list): One or several IPv4 addresses of Gluster nodes, where 'file_vol' brick processes must be recreated. """ if not isinstance(gluster_nodes, (list, set, tuple)): gluster_nodes = [gluster_nodes] # Get Gluster vol brick PIDs gluster_volume_status = get_gluster_vol_status(file_vol) pids = () for gluster_node in gluster_nodes: pid = None for g_node, g_node_data in gluster_volume_status.items(): if g_node != gluster_node: continue for process_name, process_data in g_node_data.items(): if not process_name.startswith("/var"): continue pid = process_data["pid"] # When birck is down, pid of the brick is returned as -1. # Which is unexepeted situation. So, add appropriate assertion. assert pid != "-1", ( "Got unexpected PID (-1) for '%s' gluster vol on '%s' " "node." % file_vol, gluster_node) assert pid, ("Could not find 'pid' in Gluster vol data for '%s' " "Gluster node. Data: %s" % ( gluster_node, gluster_volume_status)) pids.append((gluster_node, pid)) # Restart Gluster vol brick processes using found PIDs for gluster_node, pid in pids: cmd = "kill -9 %s" % pid cmd_run_on_gluster_pod_or_node(ocp_client_node, cmd, gluster_node) # Wait for Gluster vol brick processes to be recreated for gluster_node, pid in pids: killed_pid_cmd = "ps -eaf | grep %s | grep -v grep | awk '{print $2}'" _waiter = waiter.Waiter(timeout=60, interval=2) for w in _waiter: result = cmd_run_on_gluster_pod_or_node( ocp_client_node, killed_pid_cmd, gluster_node) if result.strip() == pid: continue g.log.info("Brick process '%s' was killed successfully on '%s'" % ( pid, gluster_node)) break if w.expired: error_msg = ("Process ID '%s' still exists on '%s' after waiting " "for it 60 seconds to get killed." % ( pid, gluster_node)) g.log.error(error_msg) raise exceptions.ExecutionError(error_msg) # Start volume after gluster vol brick processes recreation ret, out, err = volume_start( "auto_get_gluster_endpoint", file_vol, force=True) if ret != 0: err_msg = "Failed to start gluster volume %s on %s. error: %s" % ( file_vol, gluster_node, err) g.log.error(err_msg) raise AssertionError(err_msg)
def test_prometheous_kill_bhv_brick_process(self): """Validate kill brick process of block hosting volume with prometheus workload running""" # Add check for CRS version openshift_ops.switch_oc_project(self._master, self._registry_project_name) if not self.is_containerized_gluster(): self.skipTest("Skipping this test case as CRS" " version check can not be implemented") # Get one of the prometheus pod name and respective pvc name openshift_ops.switch_oc_project(self._master, self._prometheus_project_name) prometheus_pods = openshift_ops.oc_get_pods( self._master, selector=self._prometheus_resources_selector) if not prometheus_pods: self.skipTest(prometheus_pods, "Skipping test as prometheus" " pod is not present") # Validate iscsi and multipath prometheus_pod = list(prometheus_pods.keys())[0] pvc_name = openshift_ops.oc_get_custom_resource( self._master, "pod", ":.spec.volumes[*].persistentVolumeClaim.claimName", prometheus_pod) self.assertTrue(pvc_name, "Failed to get PVC name") pvc_name = pvc_name[0] self.verify_iscsi_sessions_and_multipath( pvc_name, prometheus_pod, rtype='pod', heketi_server_url=self._registry_heketi_server_url, is_registry_gluster=True) # Try to fetch metric from prometheus pod self._fetch_metric_from_promtheus_pod( metric='heketi_device_brick_count') # Kill the brick process of a BHV gluster_node = list(self._registry_servers_info.keys())[0] openshift_ops.switch_oc_project(self._master, self._registry_project_name) bhv_name = self.get_block_hosting_volume_by_pvc_name( pvc_name, heketi_server_url=self._registry_heketi_server_url, gluster_node=gluster_node, ocp_client_node=self._master) vol_status = gluster_ops.get_gluster_vol_status(bhv_name) gluster_node_ip, brick_pid = None, None for g_node, g_node_data in vol_status.items(): for process_name, process_data in g_node_data.items(): if process_name.startswith("/var"): gluster_node_ip = g_node brick_pid = process_data["pid"] break if gluster_node_ip and brick_pid: break self.assertIsNotNone(brick_pid, "Could not find pid for brick") cmd = "kill -9 {}".format(brick_pid) openshift_ops.cmd_run_on_gluster_pod_or_node(self._master, cmd, gluster_node_ip) self.addCleanup(self._guster_volume_cleanup, bhv_name) # Check if the brick-process has been killed killed_pid_cmd = ("ps -p {} -o pid --no-headers".format(brick_pid)) try: openshift_ops.cmd_run_on_gluster_pod_or_node( self._master, killed_pid_cmd, gluster_node_ip) except exceptions.ExecutionError: g.log.info("Brick process {} was killed" "successfully".format(brick_pid)) # Try to fetch metric from prometheus pod openshift_ops.switch_oc_project(self._master, self._prometheus_project_name) self._fetch_metric_from_promtheus_pod( metric='heketi_device_brick_count') # Start the bhv using force openshift_ops.switch_oc_project(self._master, self._registry_project_name) start_vol, _, _ = volume_ops.volume_start(gluster_node_ip, bhv_name, force=True) self.assertFalse( start_vol, "Failed to start volume {}" " using force".format(bhv_name)) # Validate iscsi and multipath openshift_ops.switch_oc_project(self._master, self._prometheus_project_name) self.verify_iscsi_sessions_and_multipath( pvc_name, prometheus_pod, rtype='pod', heketi_server_url=self._registry_heketi_server_url, is_registry_gluster=True) # Try to fetch metric from prometheus pod self._fetch_metric_from_promtheus_pod( metric='heketi_device_brick_count')
def restart_gluster_vol_brick_processes(ocp_client_node, file_vol, gluster_nodes): """Restarts brick process of a file volume. Args: ocp_client_node (str): Node to execute OCP commands on. file_vol (str): file volume name. gluster_nodes (str/list): One or several IPv4 addresses of Gluster nodes, where 'file_vol' brick processes must be recreated. """ if not isinstance(gluster_nodes, (list, set, tuple)): gluster_nodes = [gluster_nodes] # Get Gluster vol brick PIDs gluster_volume_status = get_gluster_vol_status(file_vol) pids = [] for gluster_node in gluster_nodes: pid = None for g_node, g_node_data in gluster_volume_status.items(): if g_node != gluster_node: continue for process_name, process_data in g_node_data.items(): if not process_name.startswith("/var"): continue pid = process_data["pid"] # When birck is down, pid of the brick is returned as -1. # Which is unexepeted situation. So, add appropriate assertion. assert pid != "-1", ( "Got unexpected PID (-1) for '%s' gluster vol on '%s' " "node." % file_vol, gluster_node) assert pid, ("Could not find 'pid' in Gluster vol data for '%s' " "Gluster node. Data: %s" % (gluster_node, gluster_volume_status)) pids.append((gluster_node, pid)) # Restart Gluster vol brick processes using found PIDs for gluster_node, pid in pids: cmd = "kill -9 %s" % pid cmd_run_on_gluster_pod_or_node(ocp_client_node, cmd, gluster_node) # Wait for Gluster vol brick processes to be recreated for gluster_node, pid in pids: killed_pid_cmd = "ps -eaf | grep %s | grep -v grep | awk '{print $2}'" _waiter = waiter.Waiter(timeout=60, interval=2) for w in _waiter: result = cmd_run_on_gluster_pod_or_node(ocp_client_node, killed_pid_cmd, gluster_node) if result.strip() == pid: continue g.log.info("Brick process '%s' was killed successfully on '%s'" % (pid, gluster_node)) break if w.expired: error_msg = ("Process ID '%s' still exists on '%s' after waiting " "for it 60 seconds to get killed." % (pid, gluster_node)) g.log.error(error_msg) raise exceptions.ExecutionError(error_msg) # Start volume after gluster vol brick processes recreation ret, out, err = volume_start("auto_get_gluster_endpoint", file_vol, force=True) if ret != 0: err_msg = "Failed to start gluster volume %s on %s. error: %s" % ( file_vol, gluster_node, err) g.log.error(err_msg) raise AssertionError(err_msg)
def dynamic_provisioning_glusterfile(self, create_vol_name_prefix): # Create secret and storage class self.create_storage_class( create_vol_name_prefix=create_vol_name_prefix) # Create PVC pvc_name = self.create_and_wait_for_pvc() # Create DC with POD and attached PVC to it. dc_name = oc_create_app_dc_with_io( self.node, pvc_name, image=self.io_container_image_cirros) self.addCleanup(oc_delete, self.node, 'dc', dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) # Verify Heketi volume name for prefix presence if provided if create_vol_name_prefix: ret = verify_volume_name_prefix(self.node, self.sc['volumenameprefix'], self.sc['secretnamespace'], pvc_name, self.sc['resturl']) self.assertTrue(ret, "verify volnameprefix failed") else: # Get the volume name and volume id from PV pv_name = get_pv_name_from_pvc(self.ocp_client[0], pvc_name) custom = [ r':spec.glusterfs.path', r':metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi-volume-id"' ] pv_vol_name, vol_id = oc_get_custom_resource( self.ocp_client[0], 'pv', custom, pv_name) # check if the pv_volume_name is present in heketi # Check if volume name is "vol_"+volumeid or not heketi_vol_name = heketi_volume_info(self.ocp_client[0], self.heketi_server_url, vol_id, json=True)['name'] self.assertEqual( pv_vol_name, heketi_vol_name, 'Volume with vol_id = %s not found' 'in heketidb' % vol_id) self.assertEqual( heketi_vol_name, 'vol_' + vol_id, 'Volume with vol_id = %s have a' 'custom perfix' % vol_id) out = cmd_run_on_gluster_pod_or_node(self.ocp_master_node[0], "gluster volume list") self.assertIn(pv_vol_name, out, "Volume with id %s does not exist" % vol_id) # Make sure we are able to work with files on the mounted volume filepath = "/mnt/file_for_testing_io.log" for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100", "ls -lrt %s", "rm -rf %s"): cmd = cmd % filepath ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to execute '%s' command on %s" % (cmd, self.node))
def test_expand_arbiter_volume_according_to_avg_file_size( self, avg_file_size, expected_brick_size, vol_expand=True): """Validate expansion of arbiter volume with diff avg file size""" data_hosts = [] arbiter_hosts = [] # set tags arbiter:disabled, arbiter:required for i, node_id in enumerate(self.node_id_list): self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'disabled' if i < 2 else 'required') node_info = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) (data_hosts.append(node_info['hostnames']['storage'][0]) if i < 2 else arbiter_hosts.append(node_info['hostnames']['storage'][0])) self.assertEqual( node_info['tags']['arbiter'], 'disabled' if i < 2 else 'required') # Create sc with gluster arbiter info self.create_storage_class( is_arbiter_vol=True, allow_volume_expansion=True, arbiter_avg_file_size=avg_file_size) # Create PVC and wait for it to be in 'Bound' state self.create_and_wait_for_pvc() vol_expanded = False for i in range(2): vol_info = get_gluster_vol_info_by_pvc_name( self.node, self.pvc_name) bricks = ( self.verify_amount_and_proportion_of_arbiter_and_data_bricks( vol_info, arbiter_bricks=(2 if vol_expanded else 1), data_bricks=(4 if vol_expanded else 2) ) ) # verify arbiter bricks lies on arbiter hosts for brick in bricks['arbiter_list']: ip, brick_name = brick['name'].split(':') self.assertIn(ip, arbiter_hosts) # verify the size of arbiter brick cmd = "df -h %s --output=size | tail -1" % brick_name out = cmd_run_on_gluster_pod_or_node(self.node, cmd, ip) self.assertEqual(out, expected_brick_size) # verify that data bricks lies on data hosts for brick in bricks['data_list']: self.assertIn(brick['name'].split(':')[0], data_hosts) if vol_expanded or not vol_expand: break # Expand PVC and verify the size pvc_size = 2 resize_pvc(self.node, self.pvc_name, pvc_size) verify_pvc_size(self.node, self.pvc_name, pvc_size) vol_expanded = True
def test_targetcli_when_block_hosting_volume_down(self): """Validate no inconsistencies occur in targetcli when block volumes are created with one block hosting volume down.""" h_node, h_server = self.heketi_client_node, self.heketi_server_url cmd = ("targetcli ls | egrep '%s' || echo unavailable") error_msg = ("targetcli has inconsistencies when block devices are " "created with one block hosting volume %s is down") # Delete BHV which has no BV or fill it completely bhv_list = get_block_hosting_volume_list(h_node, h_server).keys() for bhv in bhv_list: bhv_info = heketi_volume_info(h_node, h_server, bhv, json=True) if not bhv_info["blockinfo"].get("blockvolume", []): heketi_volume_delete(h_node, h_server, bhv) continue free_size = bhv_info["blockinfo"].get("freesize", 0) if free_size: bv = heketi_volume_create(h_node, h_server, free_size, json=True) self.addCleanup(heketi_volume_delete, h_node, h_server, bv["id"]) # Create BV bv = heketi_blockvolume_create(h_node, h_server, 2, json=True) self.addCleanup(heketi_blockvolume_delete, h_node, h_server, bv["id"]) # Bring down BHV bhv_name = get_block_hosting_volume_name(h_node, h_server, bv["id"]) ret, out, err = volume_stop("auto_get_gluster_endpoint", bhv_name) if ret != 0: err_msg = "Failed to stop gluster volume %s. error: %s" % ( bhv_name, err) g.log.error(err_msg) raise AssertionError(err_msg) self.addCleanup(podcmd.GlustoPod()(volume_start), "auto_get_gluster_endpoint", bhv_name) ocp_node = self.ocp_master_node[0] gluster_block_svc = "gluster-block-target" self.addCleanup(wait_for_service_status_on_gluster_pod_or_node, ocp_node, gluster_block_svc, "active", "exited", gluster_node=self.gluster_servers[0]) self.addCleanup(restart_service_on_gluster_pod_or_node, ocp_node, gluster_block_svc, self.gluster_servers[0]) for condition in ("continue", "break"): restart_service_on_gluster_pod_or_node( ocp_node, gluster_block_svc, gluster_node=self.gluster_servers[0]) wait_for_service_status_on_gluster_pod_or_node( ocp_node, gluster_block_svc, "active", "exited", gluster_node=self.gluster_servers[0]) targetcli = cmd_run_on_gluster_pod_or_node(ocp_node, cmd % bv["id"], self.gluster_servers[0]) if condition == "continue": self.assertEqual(targetcli, "unavailable", error_msg % bhv_name) else: self.assertNotEqual(targetcli, "unavailable", error_msg % bhv_name) break # Bring up the same BHV ret, out, err = volume_start("auto_get_gluster_endpoint", bhv_name) if ret != 0: err = "Failed to start gluster volume %s on %s. error: %s" % ( bhv_name, h_node, err) raise exceptions.ExecutionError(err)