def test_write_io_mount_point_resumed_quorum_restored_x3(self): """ - set cluster.quorum-type to auto - start I/O from the mount point - Do IO and check on subvols with two nodes to reboot (do for each subvol) - get files to delete/create for nodes to be offline - delete files from mountpoint - reboot nodes - creating files on nodes while rebooting - validate for rofs - wait for volume processes to be online - creating files on nodes after rebooting - validate IO - Do IO and check on subvols without nodes to reboot (do for each subvol) - get files to delete/create for nodes to be online - delete files from mountpoint - reboot nodes - creating files on online nodes while rebooting other nodes - validate IO - Do IO and check and reboot two nodes on all subvols - get files to delete/create for nodes to be offline - delete files from mountpoint - reboot nodes - creating files on nodes while rebooting - validate for rofs - wait for volume processes to be online - creating files on nodes after rebooting - validate IO """ # pylint: disable=too-many-locals,too-many-statements,too-many-branches # set cluster.quorum-type to auto options = {"cluster.quorum-type": "auto"} g.log.info("setting cluster.quorum-type to auto on volume %s", self.volname) ret = set_volume_options(self.mnode, self.volname, options) self.assertTrue(ret, ("Unable to set volume option %s for" "volume %s" % (options, self.volname))) g.log.info("Successfully set %s for volume %s", options, self.volname) # Creating files on client side for mount_obj in self.mounts: g.log.info("Generating data for %s:%s", mount_obj.client_system, mount_obj.mountpoint) # Creating files cmd = "/usr/bin/env python %s create_files -f 30 %s" % ( self.script_upload_path, mount_obj.mountpoint) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) self.all_mounts_procs.append(proc) # Validate IO self.io_validation_complete = False self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts), "IO failed on some of the clients") self.io_validation_complete = True # Do IO and check on subvols with nodes to reboot subvols_dict = get_subvols(self.mnode, self.volname) for subvol in subvols_dict['volume_subvols']: # define nodes to reboot brick_list = subvol[0:2] nodes_to_reboot = [] for brick in brick_list: node, brick_path = brick.split(':') nodes_to_reboot.append(node) # get files to delete/create for nodes to be offline node, brick_path = brick_list[0].split(':') ret, brick_file_list, _ = g.run(node, 'ls %s' % brick_path) self.assertFalse(ret, 'Failed to ls files on %s' % node) file_list = brick_file_list.splitlines() # delete files from mountpoint for mount_obj in self.mounts: g.log.info("Deleting data for %s:%s", mount_obj.client_system, mount_obj.mountpoint) cmd = ('cd %s/ ; rm -rf %s' % (mount_obj.mountpoint, ' '.join(file_list))) ret, _, _ = g.run(mount_obj.client_system, cmd) self.assertFalse( ret, 'Failed to rm file on %s' % mount_obj.client_system) g.log.info('Files %s are deleted', file_list) # reboot nodes on subvol and wait while rebooting g.log.info("Rebooting the nodes %s", nodes_to_reboot) ret = reboot_nodes(nodes_to_reboot) self.assertTrue(ret, 'Failed to reboot nodes %s ' % nodes_to_reboot) # Creating files on nodes while rebooting self.all_mounts_procs = [] for mount_obj in self.mounts: g.log.info("Creating data for %s:%s", mount_obj.client_system, mount_obj.mountpoint) # Creating files cmd = ("cd %s/ ;" "touch %s" % (mount_obj.mountpoint, ' '.join(file_list))) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) self.all_mounts_procs.append(proc) # Validate IO self.io_validation_complete = False g.log.info("Validating if IO failed with read-only filesystem") ret = is_io_procs_fail_with_rofs(self, self.all_mounts_procs, self.mounts) self.assertTrue(ret, ("Unexpected error and IO successful" " on read-only filesystem")) self.io_validation_complete = True g.log.info("EXPECTED: " "Read-only file system in IO while creating file") # check if nodes are online counter = 0 timeout = 300 _rc = False while counter < timeout: ret, reboot_results = are_nodes_online(nodes_to_reboot) if not ret: g.log.info("Nodes are offline, Retry after 5 seconds ... ") time.sleep(5) counter = counter + 5 else: _rc = True break if not _rc: for node in reboot_results: if reboot_results[node]: g.log.info("Node %s is online", node) else: g.log.error( "Node %s is offline even after " "%d minutes", node, timeout / 60.0) else: g.log.info("All nodes %s are up and running", nodes_to_reboot) # Wait for volume processes to be online g.log.info("Wait for volume processes to be online") ret = wait_for_volume_process_to_be_online(self.mnode, self.volname) self.assertTrue(ret, ("Failed to wait for volume %s processes to " "be online", self.volname)) g.log.info( "Successful in waiting for volume %s processes to be " "online", self.volname) # Verify volume's all process are online g.log.info("Verifying volume's all process are online") ret = verify_all_process_of_volume_are_online( self.mnode, self.volname) self.assertTrue( ret, ("Volume %s : All process are not online" % self.volname)) g.log.info("Volume %s : All process are online", self.volname) # Creating files on nodes after rebooting self.all_mounts_procs = [] for mount_obj in self.mounts: g.log.info("Creating data for %s:%s", mount_obj.client_system, mount_obj.mountpoint) # Creating files cmd = ("cd %s/ ;" "touch %s" % (mount_obj.mountpoint, ' '.join(file_list))) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) self.all_mounts_procs.append(proc) # Validate IO self.io_validation_complete = False self.assertTrue( validate_io_procs(self.all_mounts_procs, self.mounts), "IO failed on some of the clients") self.io_validation_complete = True # Do IO and check on subvols without nodes to reboot subvols_dict = get_subvols(self.mnode, self.volname) for subvol in subvols_dict['volume_subvols']: # define nodes to reboot brick_list = subvol[0:2] nodes_to_reboot = [] for brick in brick_list: node, brick_path = brick.split(':') nodes_to_reboot.append(node) # get files to delete/create for nodes to be online new_subvols_dict = get_subvols(self.mnode, self.volname) subvol_to_operate = new_subvols_dict['volume_subvols'] subvol_to_operate.remove(subvol) brick_list_subvol_online = subvol_to_operate[0] node, brick_path_vol_online = \ brick_list_subvol_online[0].split(':') ret, brick_file_list, _ = g.run(node, 'ls %s' % brick_path_vol_online) self.assertFalse(ret, 'Failed to ls files on %s' % node) file_list = brick_file_list.splitlines() # delete files from mountpoint for mount_obj in self.mounts: g.log.info("Deleting data for %s:%s", mount_obj.client_system, mount_obj.mountpoint) cmd = ('cd %s/ ; rm -rf %s' % (mount_obj.mountpoint, ' '.join(file_list))) ret, _, _ = g.run(mount_obj.client_system, cmd) self.assertFalse( ret, 'Failed to rm file on %s' % mount_obj.client_system) g.log.info('Files %s are deleted', file_list) # reboot nodes on subvol and wait while rebooting g.log.info("Rebooting the nodes %s", nodes_to_reboot) ret = reboot_nodes(nodes_to_reboot) self.assertTrue(ret, 'Failed to reboot nodes %s ' % nodes_to_reboot) # Creating files on nodes while rebooting self.all_mounts_procs = [] for mount_obj in self.mounts: g.log.info("Creating data for %s:%s", mount_obj.client_system, mount_obj.mountpoint) # Creating files cmd = ("cd %s/ ;" "touch %s" % (mount_obj.mountpoint, ' '.join(file_list))) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) self.all_mounts_procs.append(proc) # Validate IO self.io_validation_complete = False self.assertTrue( validate_io_procs(self.all_mounts_procs, self.mounts), "IO failed on some of the clients") self.io_validation_complete = True # check if nodes are online counter = 0 timeout = 300 _rc = False while counter < timeout: ret, reboot_results = are_nodes_online(nodes_to_reboot) if not ret: g.log.info("Nodes are offline, Retry after 5 seconds ... ") time.sleep(5) counter = counter + 5 else: _rc = True break if not _rc: for node in reboot_results: if reboot_results[node]: g.log.info("Node %s is online", node) else: g.log.error( "Node %s is offline even after " "%d minutes", node, timeout / 60.0) else: g.log.info("All nodes %s are up and running", nodes_to_reboot) # Wait for volume processes to be online g.log.info("Wait for volume processes to be online") ret = wait_for_volume_process_to_be_online(self.mnode, self.volname) self.assertTrue(ret, ("Failed to wait for volume %s processes to " "be online", self.volname)) g.log.info( "Successful in waiting for volume %s processes to be " "online", self.volname) # Verify volume's all process are online g.log.info("Verifying volume's all process are online") ret = verify_all_process_of_volume_are_online( self.mnode, self.volname) self.assertTrue( ret, ("Volume %s : All process are not online" % self.volname)) g.log.info("Volume %s : All process are online", self.volname) # Do IO and check and reboot nodes on all subvols subvols_dict = get_subvols(self.mnode, self.volname) nodes_to_reboot = [] file_list_for_all_subvols = [] for subvol in subvols_dict['volume_subvols']: # define nodes to reboot brick_list = subvol[0:2] for brick in brick_list: node, brick_path = brick.split(':') nodes_to_reboot.append(node) # get files to delete/create for nodes to be offline node, brick_path = brick_list[0].split(':') ret, brick_file_list, _ = g.run(node, 'ls %s' % brick_path) self.assertFalse(ret, 'Failed to ls files on %s' % node) file_list = brick_file_list.splitlines() file_list_for_all_subvols.append(file_list) # delete files from mountpoint for mount_obj in self.mounts: g.log.info("Deleting data for %s:%s", mount_obj.client_system, mount_obj.mountpoint) cmd = ('cd %s/ ; rm -rf %s' % (mount_obj.mountpoint, ' '.join(file_list))) ret, _, _ = g.run(mount_obj.client_system, cmd) self.assertFalse(ret, 'Failed to rm file on %s' % node) g.log.info('Files %s are deleted', file_list) # reboot nodes on subvol and wait while rebooting g.log.info("Rebooting the nodes %s", nodes_to_reboot) ret = reboot_nodes(nodes_to_reboot) self.assertTrue(ret, 'Failed to reboot nodes %s ' % nodes_to_reboot) # Creating files on nodes while rebooting all_mounts_procs, all_mounts_procs_1, all_mounts_procs_2 = [], [], [] # Create files for 1-st subvol and get all_mounts_procs_1 for mount_obj in self.mounts: g.log.info("Creating data for %s:%s", mount_obj.client_system, mount_obj.mountpoint) # Creating files cmd = ( "cd %s/ ;" "touch %s" % (mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[0]))) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs_1.append(proc) all_mounts_procs.append(all_mounts_procs_1) # Create files for 2-st subvol and get all_mounts_procs_2 for mount_obj in self.mounts: g.log.info("Creating data for %s:%s", mount_obj.client_system, mount_obj.mountpoint) # Creating files cmd = ( "cd %s/ ;" "touch %s" % (mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[1]))) proc2 = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs_2.append(proc2) all_mounts_procs.append(all_mounts_procs_2) for mounts_procs in all_mounts_procs: # Validate IO self.io_validation_complete = False g.log.info("Validating if IO failed with read-only filesystem") ret = is_io_procs_fail_with_rofs(self, mounts_procs, self.mounts) self.assertTrue(ret, ("Unexpected error and IO successful" " on read-only filesystem")) self.io_validation_complete = True g.log.info("EXPECTED: " "Read-only file system in IO while creating file") # check if nodes are online counter = 0 timeout = 300 _rc = False while counter < timeout: ret, reboot_results = are_nodes_online(nodes_to_reboot) if not ret: g.log.info("Nodes are offline, Retry after 5 seconds ... ") time.sleep(5) counter = counter + 5 else: _rc = True break if not _rc: for node in reboot_results: if reboot_results[node]: g.log.info("Node %s is online", node) else: g.log.error("Node %s is offline even after " "%d minutes", node, timeout / 60.0) else: g.log.info("All nodes %s are up and running", nodes_to_reboot) # Wait for volume processes to be online g.log.info("Wait for volume processes to be online") ret = wait_for_volume_process_to_be_online(self.mnode, self.volname) self.assertTrue(ret, ("Failed to wait for volume %s processes to " "be online", self.volname)) g.log.info( "Successful in waiting for volume %s processes to be " "online", self.volname) # Verify volume's all process are online g.log.info("Verifying volume's all process are online") ret = verify_all_process_of_volume_are_online(self.mnode, self.volname) self.assertTrue( ret, ("Volume %s : All process are not online" % self.volname)) g.log.info("Volume %s : All process are online", self.volname) # Creating files on nodes after rebooting all_mounts_procs, all_mounts_procs_1, all_mounts_procs_2 = [], [], [] # Create files for 1-st subvol and get all_mounts_procs_1 for mount_obj in self.mounts: g.log.info("Creating data for %s:%s", mount_obj.client_system, mount_obj.mountpoint) # Creating files cmd = ( "cd %s/ ;" "touch %s" % (mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[0]))) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs_1.append(proc) all_mounts_procs.append(all_mounts_procs_1) # Create files for 2-st subvol and get all_mounts_procs_2 for mount_obj in self.mounts: g.log.info("Creating data for %s:%s", mount_obj.client_system, mount_obj.mountpoint) # Creating files cmd = ( "cd %s/ ;" "touch %s" % (mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[1]))) proc2 = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs_2.append(proc2) all_mounts_procs.append(all_mounts_procs_2) for mounts_procs in all_mounts_procs: # Validate IO self.io_validation_complete = False self.assertTrue( validate_io_procs(self.all_mounts_procs, self.mounts), "IO failed on some of the clients") self.io_validation_complete = True
def test_client_side_quorum_with_fixed_for_cross2(self): """ Test Script to verify the Client Side Quorum with fixed for cross 2 volume * Disable self heal daemom * set cluster.quorum-type to fixed. * start I/O( write and read )from the mount point - must succeed * Bring down brick1 * start I/0 ( write and read ) - must succeed * set the cluster.quorum-count to 1 * start I/0 ( write and read ) - must succeed * set the cluster.quorum-count to 2 * start I/0 ( write and read ) - read must pass, write will fail * bring back the brick1 online * start I/0 ( write and read ) - must succeed * Bring down brick2 * start I/0 ( write and read ) - read must pass, write will fail * set the cluster.quorum-count to 1 * start I/0 ( write and read ) - must succeed * cluster.quorum-count back to 2 and cluster.quorum-type to auto * start I/0 ( write and read ) - must succeed * Bring back brick2 online * Bring down brick1 * start I/0 ( write and read ) - read must pass, write will fail * set the quorum-type to none * start I/0 ( write and read ) - must succeed """ # pylint: disable=too-many-branches,too-many-statements # Disable self heal daemon options = {"cluster.self-heal-daemon": "off"} g.log.info("setting %s for the volume %s", options, self.volname) ret = set_volume_options(self.mnode, self.volname, options) self.assertTrue(ret, ("Unable to set %s for volume %s" % (options, self.volname))) g.log.info("Successfully set %s for volume %s", options, self.volname) # set cluster.quorum-type to fixed options = {"cluster.quorum-type": "fixed"} g.log.info("setting %s for the volume %s", options, self.volname) ret = set_volume_options(self.mnode, self.volname, options) self.assertTrue(ret, ("Unable to set %s for volume %s" % (options, self.volname))) g.log.info("Successfully set %s for volume %s", options, self.volname) # start I/O( write ) - must succeed g.log.info("Starting IO on all mounts...") g.log.info("mounts: %s", self.mounts) all_mounts_procs = [] cmd = ("python %s create_files " "-f 10 --base-file-name file %s" % (self.script_upload_path, self.mounts[0].mountpoint)) proc = g.run_async(self.mounts[0].client_system, cmd, user=self.mounts[0].user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "IO failed on some of the clients") # read the file g.log.info("Start reading files on all mounts") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s read " "%s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "Reads failed on some of the clients") # get the subvolumes g.log.info("Starting to get sub-volumes for volume %s", self.volname) subvols_dict = get_subvols(self.mnode, self.volname) num_subvols = len(subvols_dict['volume_subvols']) g.log.info("Number of subvolumes in volume %s:", num_subvols) # Bring down brick1 for all the subvolumes subvolumes_first_brick_list = [] subvolumes_second_brick_list = [] for i in range(0, num_subvols): subvol_brick_list = subvols_dict['volume_subvols'][i] g.log.info("sub-volume %s brick list : %s", i, subvol_brick_list) subvolumes_first_brick_list.append(subvol_brick_list[0]) subvolumes_second_brick_list.append(subvol_brick_list[1]) g.log.info("Going to bring down the brick process " "for %s", subvolumes_first_brick_list) ret = bring_bricks_offline(self.volname, subvolumes_first_brick_list) self.assertTrue(ret, ("Failed to bring down the bricks. Please " "check the log file for more details.")) g.log.info("Brought down the brick process " "for %s successfully", subvolumes_first_brick_list) # start I/0 ( write and read ) - must succeed g.log.info("Starting IO on all mounts...") g.log.info("mounts: %s", self.mounts) all_mounts_procs = [] cmd = ("python %s create_files " "-f 10 --base-file-name second_file %s" % (self.script_upload_path, self.mounts[0].mountpoint)) proc = g.run_async(self.mounts[0].client_system, cmd, user=self.mounts[0].user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "IO failed on some of the clients") # read the file g.log.info("Start reading files on all mounts") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s read " "%s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "Reads failed on some of the clients") # set the cluster.quorum-count to 1 options = {"cluster.quorum-count": "1"} g.log.info("setting %s for the volume %s", options, self.volname) ret = set_volume_options(self.mnode, self.volname, options) self.assertTrue(ret, ("Unable to set %s for volume %s" % (options, self.volname))) g.log.info("Successfully set %s for volume %s", options, self.volname) # start I/0 ( write and read ) - must succeed g.log.info("Starting IO on mount.....") all_mounts_procs = [] cmd = ("python %s create_files " "-f 10 --base-file-name third_file %s" % (self.script_upload_path, self.mounts[0].mountpoint)) proc = g.run_async(self.mounts[0].client_system, cmd, user=self.mounts[0].user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "IO failed on some of the clients") # read the file g.log.info("Start reading files on all mounts") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s read " "%s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "Reads failed on some of the clients") # set the cluster.quorum-count to 2 options = {"cluster.quorum-count": "2"} g.log.info("setting %s for the volume %s", options, self.volname) ret = set_volume_options(self.mnode, self.volname, options) self.assertTrue(ret, ("Unable to set %s for volume %s" % (options, self.volname))) g.log.info("Successfully set %s for volume %s", options, self.volname) # start I/0 ( write and read ) - read must pass, write will fail g.log.info("Starting IO on mount......") all_mounts_procs = [] cmd = ("python %s create_files " "-f 10 --base-file-name fourth_file %s" % (self.script_upload_path, self.mounts[0].mountpoint)) proc = g.run_async(self.mounts[0].client_system, cmd, user=self.mounts[0].user) all_mounts_procs.append(proc) # Validate IO g.log.info("Validating whether IO failed with Read Only File System") ret, _ = is_io_procs_fail_with_rofs(self, all_mounts_procs, self.mounts) self.assertTrue(ret, ("Unexpected Error and IO successful" " on Read-Only File System")) g.log.info("EXPECTED Read-only file system in IO while creating file") # read the file g.log.info("Start reading files on all mounts") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s read " "%s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "Reads failed on some of the clients") # bring back the brick1 online for all subvolumes g.log.info("bringing up the bricks : %s online", subvolumes_first_brick_list) ret = bring_bricks_online(self.mnode, self.volname, subvolumes_first_brick_list) self.assertTrue(ret, ("Failed to brought the bricks %s online" % subvolumes_first_brick_list)) g.log.info("Successfully brought the bricks %s online", subvolumes_first_brick_list) # start I/0 ( write and read ) - must succeed g.log.info("Starting IO on mount.....") all_mounts_procs = [] cmd = ("python %s create_files " "-f 10 --base-file-name fifth_file %s" % (self.script_upload_path, self.mounts[0].mountpoint)) proc = g.run_async(self.mounts[0].client_system, cmd, user=self.mounts[0].user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "IO failed on some of the clients") # read the file g.log.info("Start reading files on all mounts") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s read " "%s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "Reads failed on some of the clients") # Bring down brick2 for all the subvolumes g.log.info("Going to bring down the brick process " "for %s", subvolumes_second_brick_list) ret = bring_bricks_offline(self.volname, subvolumes_second_brick_list) self.assertTrue(ret, ("Failed to bring down the bricks. Please " "check the log file for more details.")) g.log.info("Brought down the brick process " "for %s successfully", subvolumes_second_brick_list) # start I/0 ( write and read ) - read must pass, write will fail g.log.info("Start creating files on mounts.....") all_mounts_procs = [] cmd = ("python %s create_files " "-f 10 --base-file-name sixth_file %s" % (self.script_upload_path, self.mounts[0].mountpoint)) proc = g.run_async(self.mounts[0].client_system, cmd, user=self.mounts[0].user) all_mounts_procs.append(proc) # Validate IO g.log.info("Validating whether IO failed with Read Only File System") ret, _ = is_io_procs_fail_with_rofs(self, all_mounts_procs, self.mounts) self.assertTrue(ret, ("Unexpected Error and IO successful" " on Read-Only File System")) g.log.info("EXPECTED Read-only file system in IO while creating file") # read the file g.log.info("Start reading files on all mounts") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s read " "%s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "Reads failed on some of the clients") # set the cluster.quorum-count to 1 options = {"cluster.quorum-count": "1"} g.log.info("setting %s for the volume %s", options, self.volname) ret = set_volume_options(self.mnode, self.volname, options) self.assertTrue(ret, ("Unable to set %s for volume %s" % (options, self.volname))) g.log.info("Successfully set %s for volume %s", options, self.volname) # start I/0 ( write and read ) - must succeed g.log.info("Starting IO on mount.....") all_mounts_procs = [] cmd = ("python %s create_files " "-f 10 --base-file-name seventh_file %s" % (self.script_upload_path, self.mounts[0].mountpoint)) proc = g.run_async(self.mounts[0].client_system, cmd, user=self.mounts[0].user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "IO failed on some of the clients") # read the file g.log.info("Start reading files on all mounts") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s read " "%s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "Reads failed on some of the clients") # set cluster.quorum-type to auto and cluster.quorum-count back to 2 options = {"cluster.quorum-type": "auto", "cluster.quorum-count": "2"} g.log.info("setting %s for the volume %s", options, self.volname) ret = set_volume_options(self.mnode, self.volname, options) self.assertTrue(ret, ("Unable to set %s for volume %s" % (options, self.volname))) g.log.info("Successfully set %s for volume %s", options, self.volname) # start I/0 ( write and read ) - must succeed g.log.info("Starting IO on mount.....") all_mounts_procs = [] cmd = ("python %s create_files " "-f 10 --base-file-name eigth_file %s" % (self.script_upload_path, self.mounts[0].mountpoint)) proc = g.run_async(self.mounts[0].client_system, cmd, user=self.mounts[0].user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "IO failed on some of the clients") # read the file g.log.info("Start reading files on all mounts") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s read " "%s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "Reads failed on some of the clients") # Bring back brick2 online for all the subvolumes g.log.info("bringing up the bricks : %s online", subvolumes_second_brick_list) ret = bring_bricks_online(self.mnode, self.volname, subvolumes_second_brick_list) self.assertTrue(ret, ("Failed to brought the brick %s online" % subvolumes_second_brick_list)) g.log.info("Successfully brought the brick %s online", subvolumes_second_brick_list) # Bring down brick1 again for all the subvolumes g.log.info("Going to bring down the brick process " "for %s", subvolumes_first_brick_list) ret = bring_bricks_offline(self.volname, subvolumes_first_brick_list) self.assertTrue(ret, ("Failed to bring down the bricks. Please " "check the log file for more details.")) g.log.info("Brought down the brick process " "for %s successfully", subvolumes_first_brick_list) # start I/0 ( write and read ) - read must pass, write will fail g.log.info("Start creating files on mounts.....") all_mounts_procs = [] cmd = ("python %s create_files " "-f 10 --base-file-name ninth_file %s" % (self.script_upload_path, self.mounts[0].mountpoint)) proc = g.run_async(self.mounts[0].client_system, cmd, user=self.mounts[0].user) all_mounts_procs.append(proc) # Validate IO g.log.info("Validating whether IO failed with Read Only File System") ret, _ = is_io_procs_fail_with_rofs(self, all_mounts_procs, self.mounts) self.assertTrue(ret, ("Unexpected Error and IO successful" " on Read-Only File System")) g.log.info("EXPECTED Read-only file system in IO while creating file") # read the file g.log.info("Start reading files on all mounts") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s read " "%s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "Reads failed on some of the clients") # set the quorum-type to none options = {"cluster.quorum-type": "none"} g.log.info("setting %s for the volume %s", options, self.volname) ret = set_volume_options(self.mnode, self.volname, options) self.assertTrue(ret, ("Unable to set %s for volume %s" % (options, self.volname))) g.log.info("Successfully set %s for volume %s", options, self.volname) # start I/0 ( write and read ) - must succeed g.log.info("Starting IO on mount.....") all_mounts_procs = [] cmd = ("python %s create_files " "-f 10 --base-file-name tenth_file %s" % (self.script_upload_path, self.mounts[0].mountpoint)) proc = g.run_async(self.mounts[0].client_system, cmd, user=self.mounts[0].user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "IO failed on some of the clients") # read the file g.log.info("Start reading files on all mounts") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s read " "%s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts), "Reads failed on some of the clients") # bring back the bricks online for all subvolumes g.log.info("bringing up the brick : %s online", subvolumes_first_brick_list) ret = bring_bricks_online(self.mnode, self.volname, subvolumes_first_brick_list) self.assertTrue(ret, ("Failed to brought the brick %s online" % subvolumes_first_brick_list)) g.log.info("Successfully brought the bricks")
def test_client_side_quorum_with_auto_option(self): """ Test Script to verify the Client Side Quorum with auto option * set cluster.quorum-type to auto. * start I/O from the mount point. * kill 2 of the brick process from the each and every replica set * perform ops """ # set cluster.quorum-type to auto options = {"cluster.quorum-type": "auto"} g.log.info("setting cluster.quorum-type to auto on " "volume %s" % self.volname) ret = set_volume_options(self.mnode, self.volname, options) self.assertTrue(ret, ("Unable to set volume option %s for" "volume %s" % (options, self.volname))) g.log.info("Sucessfully set %s for volume %s" % (options, self.volname)) # write files on all mounts g.log.info("Starting IO on all mounts...") g.log.info("mounts: %s" % self.mounts) all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s create_files " "-f 10 --base-file-name file %s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO g.log.info("Validating IO on mounts") ret = validate_io_procs(all_mounts_procs, self.mounts) self.assertTrue(ret, "IO failed on some of the clients") g.log.info("IO is successful on all mounts") # get the subvolumes g.log.info("Starting to get sub-volumes for volume %s" % self.volname) subvols_dict = get_subvols(self.mnode, self.volname) num_subvols = len(subvols_dict['volume_subvols']) g.log.info("Number of subvolumes in volume %s:" % num_subvols) # bring bricks offline( 2 bricks ) for all the subvolumes for i in range(0, num_subvols): subvol_brick_list = subvols_dict['volume_subvols'][i] g.log.info("sub-volume %s brick list : %s" % (i, subvol_brick_list)) # For volume type: 1 * 2, bring 1 brick offline if len(subvol_brick_list) == 2: bricks_to_bring_offline = subvol_brick_list[0:1] else: bricks_to_bring_offline = subvol_brick_list[0:2] g.log.info("Going to bring down the brick process " "for %s" % bricks_to_bring_offline) ret = bring_bricks_offline(self.volname, bricks_to_bring_offline) self.assertTrue(ret, ("Failed to bring down the bricks. Please " "check the log file for more details.")) g.log.info("Brought down the brick process " "for %s succesfully" % bricks_to_bring_offline) # create 2 files named newfile0.txt and newfile1.txt g.log.info("Start creating 2 files on all mounts...") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s create_files " "-f 2 --base-file-name newfile %s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO g.log.info("Validating whether IO failed with read-only filesystem") ret = is_io_procs_fail_with_rofs(self, all_mounts_procs, self.mounts) self.assertTrue(ret, ("Unexpected error and IO successfull" " on read-only filesystem")) g.log.info("EXPECTED: Read-only file system in IO while creating file") # create directory user1 g.log.info("Start creating directory on all mounts...") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s create_deep_dir " "%s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO g.log.info("Validating whether IO failed with read-only filesystem") ret = is_io_procs_fail_with_rofs(self, all_mounts_procs, self.mounts) self.assertTrue(ret, ("Unexpected error and IO successfull" " on read-only filesystem")) g.log.info("EXPECTED: Read-only file system in IO while" " creating directory") # create h/w link to file g.log.info("Start creating hard link for file0.txt on all mounts") for mount_obj in self.mounts: cmd = "ln %s/file0.txt %s/file0.txt_hwlink" \ % (mount_obj.mountpoint, mount_obj.mountpoint) ret, out, err = g.run(mount_obj.client_system, cmd) self.assertTrue(ret, ("Unexpected error and creating hard link" " successful on read-only filesystem")) self.assertIn( "Read-only file system", err, "Read-only filesystem not found in " "IO while truncating file") g.log.info("EXPECTED: Read-only file system in IO") # create s/w link g.log.info("Start creating soft link for file1.txt on all mounts") for mount_obj in self.mounts: cmd = "ln -s %s/file1.txt %s/file1.txt_swlink" %\ (mount_obj.mountpoint, mount_obj.mountpoint) ret, out, err = g.run(mount_obj.client_system, cmd) self.assertTrue(ret, ("Unexpected error and creating soft link" " successful on read-only filesystem")) self.assertIn( "Read-only file system", err, "Read-only filesystem not found in " "IO while truncating file") g.log.info("EXPECTED: Read-only file system in IO") # append to file g.log.info("Appending to file1.txt on all mounts") for mount_obj in self.mounts: cmd = "cat %s/file0.txt >> %s/file1.txt" %\ (mount_obj.mountpoint, mount_obj.mountpoint) ret, out, err = g.run(mount_obj.client_system, cmd) self.assertTrue(ret, ("Unexpected error and append successful" " on read-only filesystem")) self.assertIn( "Read-only file system", err, "Read-only filesystem not found in " "IO while truncating file") g.log.info("EXPECTED: Read-only file system in IO") # modify the file g.log.info("Modifying file1.txt on all mounts") for mount_obj in self.mounts: cmd = "echo 'Modify Contents' > %s/file1.txt"\ % (mount_obj.mountpoint) ret, out, err = g.run(mount_obj.client_system, cmd) self.assertTrue(ret, ("Unexpected error and modifying successful" " on read-only filesystem")) self.assertIn( "Read-only file system", err, "Read-only filesystem not found in " "IO while truncating file") g.log.info("EXPECTED: Read-only file system in IO") # truncate the file g.log.info("Truncating file1.txt on all mounts") for mount_obj in self.mounts: cmd = "truncate -s 0 %s/file1.txt" % (mount_obj.mountpoint) ret, out, err = g.run(mount_obj.client_system, cmd) self.assertTrue(ret, ("Unexpected error and truncating file" " successful on read-only filesystem")) self.assertIn( "Read-only file system", err, "Read-only filesystem not found in " "IO while truncating file") g.log.info("EXPECTED: Read-only file system in IO") # read the file g.log.info("Starting reading files on all mounts") all_mounts_procs = [] for mount_obj in self.mounts: cmd = ("python %s read " "%s" % (self.script_upload_path, mount_obj.mountpoint)) proc = g.run_async(mount_obj.client_system, cmd, user=mount_obj.user) all_mounts_procs.append(proc) # Validate IO g.log.info("validating IO on all mounts") ret = validate_io_procs(all_mounts_procs, self.mounts) self.assertTrue(ret, "Reads failed on some of the clients") g.log.info("Reads successful on all mounts") # stat on file g.log.info("stat on file1.txt on all mounts") for mount_obj in self.mounts: cmd = "stat %s/file1.txt" % (mount_obj.mountpoint) ret, out, err = g.run(mount_obj.client_system, cmd) self.assertFalse(ret, ("Unexpected error and stat on file fails" " on read-only filesystem")) g.log.info("stat on file is successfull on read-only filesystem") # stat on dir g.log.info("stat on directory on all mounts") for mount_obj in self.mounts: cmd = ("python %s stat %s" % (self.script_upload_path, mount_obj.mountpoint)) ret, out, err = g.run(mount_obj.client_system, cmd) self.assertFalse(ret, ("Unexpected error and stat on directory" " fails on read-only filesystem")) g.log.info("stat on dir is successfull on read-only filesystem") # ls on mount point g.log.info("ls on mount point on all mounts") for mount_obj in self.mounts: cmd = ("python %s ls %s" % (self.script_upload_path, mount_obj.mountpoint)) ret, out, err = g.run(mount_obj.client_system, cmd) self.assertFalse(ret, ("Unexpected error and listing file fails" " on read-only filesystem")) g.log.info("listing files is successfull on read-only filesystem")
def test_client_side_quorum_auto_local_to_volume_not_cluster(self): """ - create four volume as below vol1->2x2 vol2->2x2 vol3->2x3 vol4->2x3 vol5->a pure distribute volume - do IO to all vols - set client side quorum to auto for vol1 and vol3 - get the client side quorum value for all vols and check for result - bring down b0 on vol1 and b0 and b1 on vol3 - try to create files on all vols and check for result """ # pylint: disable=too-many-locals,too-many-statements # Creating files for all volumes for mount_point in self.mount_points: self.all_mounts_procs = [] g.log.info('Creating files...') command = ("python %s create_files -f 50 " "--fixed-file-size 1k %s" % (self.script_upload_path, mount_point)) proc = g.run_async(self.mounts[0].client_system, command) self.all_mounts_procs.append(proc) self.io_validation_complete = False # Validate IO self.assertTrue( validate_io_procs(self.all_mounts_procs, self.mounts), "IO failed on some of the clients") self.io_validation_complete = True volumes_to_change_options = ['1', '3'] # set cluster.quorum-type to auto for vol_number in volumes_to_change_options: vol_name = ('testvol_distributed-replicated_%s' % vol_number) options = {"cluster.quorum-type": "auto"} g.log.info( "setting cluster.quorum-type to auto on " "volume testvol_distributed-replicated_%s", vol_number) ret = set_volume_options(self.mnode, vol_name, options) self.assertTrue(ret, ("Unable to set volume option %s for " "volume %s" % (options, vol_name))) g.log.info("Successfully set %s for volume %s", options, vol_name) # check is options are set correctly volume_list = get_volume_list(self.mnode) for volume in volume_list: g.log.info('Checking for cluster.quorum-type option for %s', volume) volume_options_dict = get_volume_options(self.mnode, volume, 'cluster.quorum-type') if (volume == 'testvol_distributed-replicated_1' or volume == 'testvol_distributed-replicated_3' or volume == 'testvol_distributed-replicated_4'): self.assertEqual( volume_options_dict['cluster.quorum-type'], 'auto', 'Option cluster.quorum-type ' 'is not AUTO for %s' % volume) g.log.info('Option cluster.quorum-type is AUTO for %s', volume) else: self.assertEqual( volume_options_dict['cluster.quorum-type'], 'none', 'Option cluster.quorum-type ' 'is not NONE for %s' % volume) g.log.info('Option cluster.quorum-type is NONE for %s', volume) # Get first brick server and brick path # and get first file from filelist then delete it from volume vols_file_list = {} for volume in volume_list: brick_list = get_all_bricks(self.mnode, volume) brick_server, brick_path = brick_list[0].split(':') ret, file_list, _ = g.run(brick_server, 'ls %s' % brick_path) self.assertFalse(ret, 'Failed to ls files on %s' % brick_server) file_from_vol = file_list.splitlines()[0] ret, _, _ = g.run(brick_server, 'rm -rf %s/%s' % (brick_path, file_from_vol)) self.assertFalse(ret, 'Failed to rm file on %s' % brick_server) vols_file_list[volume] = file_from_vol # bring bricks offline # bring first brick for testvol_distributed-replicated_1 volname = 'testvol_distributed-replicated_1' brick_list = get_all_bricks(self.mnode, volname) bricks_to_bring_offline = brick_list[0:1] g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline) ret = bring_bricks_offline(volname, bricks_to_bring_offline) self.assertTrue( ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline) ret = are_bricks_offline(self.mnode, volname, bricks_to_bring_offline) self.assertTrue(ret, 'Bricks %s are not offline' % bricks_to_bring_offline) g.log.info('Bringing bricks %s offline is successful', bricks_to_bring_offline) # bring first two bricks for testvol_distributed-replicated_3 volname = 'testvol_distributed-replicated_3' brick_list = get_all_bricks(self.mnode, volname) bricks_to_bring_offline = brick_list[0:2] g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline) ret = bring_bricks_offline(volname, bricks_to_bring_offline) self.assertTrue( ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline) ret = are_bricks_offline(self.mnode, volname, bricks_to_bring_offline) self.assertTrue(ret, 'Bricks %s are not offline' % bricks_to_bring_offline) g.log.info('Bringing bricks %s offline is successful', bricks_to_bring_offline) # merge two dicts (volname: file_to_delete) and (volname: mountpoint) temp_dict = [vols_file_list, self.mount_points_and_volnames] file_to_delete_to_mountpoint_dict = {} for k in vols_file_list.iterkeys(): file_to_delete_to_mountpoint_dict[k] = (tuple( file_to_delete_to_mountpoint_dict[k] for file_to_delete_to_mountpoint_dict in temp_dict)) # create files on all volumes and check for result for volname, file_and_mountpoint in \ file_to_delete_to_mountpoint_dict.iteritems(): filename, mountpoint = file_and_mountpoint # check for ROFS error for read-only file system for # testvol_distributed-replicated_1 and # testvol_distributed-replicated_3 if (volname == 'testvol_distributed-replicated_1' or volname == 'testvol_distributed-replicated_3'): # create new file taken from vols_file_list g.log.info("Start creating new file on all mounts...") all_mounts_procs = [] cmd = ("touch %s/%s" % (mountpoint, filename)) proc = g.run_async(self.client, cmd) all_mounts_procs.append(proc) # Validate IO g.log.info("Validating if IO failed with read-only filesystem") ret, _ = is_io_procs_fail_with_rofs(self, all_mounts_procs, self.mounts) self.assertTrue(ret, ("Unexpected error and IO successful" " on read-only filesystem")) g.log.info("EXPECTED: " "Read-only file system in IO while creating file") # check for no errors for all the rest volumes else: # create new file taken from vols_file_list g.log.info("Start creating new file on all mounts...") all_mounts_procs = [] cmd = ("touch %s/%s" % (mountpoint, filename)) proc = g.run_async(self.client, cmd) all_mounts_procs.append(proc) # Validate IO self.assertTrue( validate_io_procs(all_mounts_procs, self.mounts), "IO failed on some of the clients")