def test_targetcli_weak_permissions_config_files(self): """Validate permissions on config files""" ocp_node = self.ocp_master_node[0] gluster_node = self.gluster_servers[0] dir_perm_before, dir_perm_after = "drwxrwxrwx.", "drw-------." file_perm_before, file_perm_after = "-rwxrwxrwx.", "-rw-------." services = ("tcmu-runner", "gluster-block-target", "gluster-blockd") cmd = "chmod -R 777 /etc/target/" # Check the permissions on '/etc/target' and '/etc/target/backup' cmd_run_on_gluster_pod_or_node(ocp_node, cmd, gluster_node) for service in services: state = ('exited' if service == 'gluster-block-target' else 'running') self.addCleanup(wait_for_service_status_on_gluster_pod_or_node, ocp_node, service, 'active', state, gluster_node) self.addCleanup(restart_service_on_gluster_pod_or_node, ocp_node, service, gluster_node) self._validate_permission(ocp_node, gluster_node, dir_perm_before, file_perm_before) # Restart the services for service in services: state = ('exited' if service == 'gluster-block-target' else 'running') restart_service_on_gluster_pod_or_node(ocp_node, service, gluster_node) wait_for_service_status_on_gluster_pod_or_node( ocp_node, service, 'active', state, gluster_node) # Permission on '/etc/target' should be changed to default self._validate_permission(ocp_node, gluster_node, dir_perm_after, file_perm_after)
def test_restart_services_provision_volume_and_run_io(self, service): """Restart gluster service then validate volumes""" block_hosting_vol = self.get_block_hosting_volume_by_pvc_name( self.pvc_name) g_nodes = get_gluster_vol_hosting_nodes(block_hosting_vol) self.assertGreater(len(g_nodes), 2) # restarts glusterfs service restart_service_on_gluster_pod_or_node( self.oc_node, service, g_nodes[0]) # wait for deployed user pod to be in Running state after restarting # service wait_for_pod_be_ready( self.oc_node, self.pod_name, timeout=60, wait_step=5) # checks if all glusterfs services are in running state for g_node in g_nodes: for service in (SERVICE_BLOCKD, SERVICE_TCMU, SERVICE_TARGET): status = "exited" if service == SERVICE_TARGET else "running" self.assertTrue(wait_for_service_status_on_gluster_pod_or_node( self.oc_node, service, status, g_node)) # validates pvc, pv, heketi block and gluster block count after # service restarts self.validate_volumes_and_blocks()
def test_restart_services_provision_volume_and_run_io(self, service): """Restart gluster service then validate volumes""" block_hosting_vol = self.get_block_hosting_volume_by_pvc_name( self.pvc_name) g_nodes = get_gluster_vol_hosting_nodes(block_hosting_vol) self.assertGreater(len(g_nodes), 2) # restarts glusterfs service restart_service_on_gluster_pod_or_node(self.oc_node, service, g_nodes[0]) # wait for deployed user pod to be in Running state after restarting # service wait_for_pod_be_ready(self.oc_node, self.pod_name, timeout=60, wait_step=5) # checks if all glusterfs services are in running state for g_node in g_nodes: for service in (SERVICE_BLOCKD, SERVICE_TCMU, SERVICE_TARGET): status = "exited" if service == SERVICE_TARGET else "running" self.assertTrue( wait_for_service_status_on_gluster_pod_or_node( self.oc_node, service, status, g_node)) # validates pvc, pv, heketi block and gluster block count after # service restarts self.validate_volumes_and_blocks()
def test_restart_services_provision_volume_and_run_io(self, service): """Restart gluster service then validate volumes""" skip_msg = ("Skipping this test case due to bugs " "BZ-1634745, BZ-1635736, BZ-1636477, BZ-1641668") # TODO(vamahaja): Add check for CRS version if not self.is_containerized_gluster(): self.skipTest(skip_msg + " and not implemented CRS version check") if get_openshift_storage_version() < "3.11.2": self.skipTest(skip_msg) self.deploy_and_verify_resouces() block_hosting_vol = self.get_block_hosting_volume_by_pvc_name( self.pvc_name) g_nodes = get_gluster_vol_hosting_nodes(block_hosting_vol) self.assertGreater(len(g_nodes), 2) # restarts glusterfs service restart_service_on_gluster_pod_or_node(self.oc_node, service, g_nodes[0]) # wait for deployed user pod to be in Running state after restarting # service wait_for_pod_be_ready(self.oc_node, self.pod_name, timeout=60, wait_step=5) # checks if all glusterfs services are in running state for g_node in g_nodes: for service in (SERVICE_BLOCKD, SERVICE_TCMU, SERVICE_TARGET): state = "exited" if service == SERVICE_TARGET else "running" self.assertTrue( wait_for_service_status_on_gluster_pod_or_node( self.oc_node, service, 'active', state, g_node)) # validates pvc, pv, heketi block and gluster block count after # service restarts self.validate_volumes_and_blocks()
def test_heketi_manual_cleanup_operation_in_bhv(self): """Validate heketi db cleanup will resolve the mismatch in the free size of the block hosting volume with failed block device create operations. """ bhv_size_before, bhv_size_after, vol_count = [], [], 5 ocp_node, g_node = self.ocp_master_node[0], self.gluster_servers[0] h_node, h_url = self.heketi_client_node, self.heketi_server_url # Get existing heketi volume list existing_volumes = heketi_volume_list(h_node, h_url, json=True) # Add function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Get nodes id list node_id_list = heketi_node_list(h_node, h_url) # Disable 4th and other nodes for node_id in node_id_list[3:]: heketi_node_disable(h_node, h_url, node_id) self.addCleanup(heketi_node_enable, h_node, h_url, node_id) # Calculate heketi volume size free_space, nodenum = get_total_free_space(h_node, h_url) free_space_available = int(free_space / nodenum) if free_space_available > vol_count: h_volume_size = int(free_space_available / vol_count) if h_volume_size > 50: h_volume_size = 50 else: h_volume_size, vol_count = 1, free_space_available # Create BHV in case blockvolume size is greater than default BHV size default_bhv_size = get_default_block_hosting_volume_size( h_node, self.heketi_dc_name) if default_bhv_size < h_volume_size: h_volume_name = "autotest-{}".format(utils.get_random_str()) bhv_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, free_space_available, raise_on_cleanup_error=False, block=True, json=True) free_space_available -= ( int(bhv_info.get("blockinfo").get("reservedsize")) + 1) h_volume_size = int(free_space_available / vol_count) # Get BHV list h_bhv_list = get_block_hosting_volume_list(h_node, h_url).keys() self.assertTrue(h_bhv_list, "Failed to get the BHV list") # Get BHV size for bhv in h_bhv_list: vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bhv_vol_size_before = vol_info.get("freesize") bhv_size_before.append(bhv_vol_size_before) # Kill Tcmu-runner service services = ("tcmu-runner", "gluster-block-target", "gluster-blockd") kill_service_on_gluster_pod_or_node(ocp_node, "tcmu-runner", g_node) # Restart the services for service in services: state = ('exited' if service == 'gluster-block-target' else 'running') self.addCleanup(wait_for_service_status_on_gluster_pod_or_node, ocp_node, service, 'active', state, g_node) self.addCleanup(restart_service_on_gluster_pod_or_node, ocp_node, service, g_node) def run_async(cmd, hostname, raise_on_error=True): return g.run_async(host=hostname, command=cmd) # Create stale block volumes in async for count in range(vol_count): with mock.patch.object(json, 'loads', side_effect=(lambda j: j)): with mock.patch.object(command, 'cmd_run', side_effect=run_async): heketi_blockvolume_create(h_node, h_url, h_volume_size, json=True) # Wait for pending operation to get generated self._check_for_pending_operations(h_node, h_url) # Restart the services for service in services: state = ('exited' if service == 'gluster-block-target' else 'running') restart_service_on_gluster_pod_or_node(ocp_node, service, g_node) wait_for_service_status_on_gluster_pod_or_node( ocp_node, service, 'active', state, g_node) # Cleanup pending operation heketi_server_operation_cleanup(h_node, h_url) # wait for pending operation to get cleaned up for w in waiter.Waiter(timeout=120, interval=10): # Get BHV size for bhv in h_bhv_list: vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bhv_vol_size_after = vol_info.get("freesize") bhv_size_after.append(bhv_vol_size_after) if (set(bhv_size_before) == set(bhv_size_after)): break if w.expired: raise exceptions.ExecutionError( "Failed to Validate volume size Actual:{}," " Expected:{}".format(set(bhv_size_before), set(bhv_size_after)))
def test_targetcli_when_block_hosting_volume_down(self): """Validate no inconsistencies occur in targetcli when block volumes are created with one block hosting volume down.""" h_node, h_server = self.heketi_client_node, self.heketi_server_url cmd = ("targetcli ls | egrep '%s' || echo unavailable") error_msg = ("targetcli has inconsistencies when block devices are " "created with one block hosting volume %s is down") # Delete BHV which has no BV or fill it completely bhv_list = get_block_hosting_volume_list(h_node, h_server).keys() for bhv in bhv_list: bhv_info = heketi_volume_info(h_node, h_server, bhv, json=True) if not bhv_info["blockinfo"].get("blockvolume", []): heketi_volume_delete(h_node, h_server, bhv) continue free_size = bhv_info["blockinfo"].get("freesize", 0) if free_size: bv = heketi_volume_create(h_node, h_server, free_size, json=True) self.addCleanup(heketi_volume_delete, h_node, h_server, bv["id"]) # Create BV bv = heketi_blockvolume_create(h_node, h_server, 2, json=True) self.addCleanup(heketi_blockvolume_delete, h_node, h_server, bv["id"]) # Bring down BHV bhv_name = get_block_hosting_volume_name(h_node, h_server, bv["id"]) ret, out, err = volume_stop("auto_get_gluster_endpoint", bhv_name) if ret != 0: err_msg = "Failed to stop gluster volume %s. error: %s" % ( bhv_name, err) g.log.error(err_msg) raise AssertionError(err_msg) self.addCleanup(podcmd.GlustoPod()(volume_start), "auto_get_gluster_endpoint", bhv_name) ocp_node = self.ocp_master_node[0] gluster_block_svc = "gluster-block-target" self.addCleanup(wait_for_service_status_on_gluster_pod_or_node, ocp_node, gluster_block_svc, "active", "exited", gluster_node=self.gluster_servers[0]) self.addCleanup(restart_service_on_gluster_pod_or_node, ocp_node, gluster_block_svc, self.gluster_servers[0]) for condition in ("continue", "break"): restart_service_on_gluster_pod_or_node( ocp_node, gluster_block_svc, gluster_node=self.gluster_servers[0]) wait_for_service_status_on_gluster_pod_or_node( ocp_node, gluster_block_svc, "active", "exited", gluster_node=self.gluster_servers[0]) targetcli = cmd_run_on_gluster_pod_or_node(ocp_node, cmd % bv["id"], self.gluster_servers[0]) if condition == "continue": self.assertEqual(targetcli, "unavailable", error_msg % bhv_name) else: self.assertNotEqual(targetcli, "unavailable", error_msg % bhv_name) break # Bring up the same BHV ret, out, err = volume_start("auto_get_gluster_endpoint", bhv_name) if ret != 0: err = "Failed to start gluster volume %s on %s. error: %s" % ( bhv_name, h_node, err) raise exceptions.ExecutionError(err)