def test_write_io_mount_point_resumed_quorum_restored_x3(self):
"""
- set cluster.quorum-type to auto
- start I/O from the mount point
- Do IO and check on subvols with two nodes to reboot
(do for each subvol)
- get files to delete/create for nodes to be offline
- delete files from mountpoint
- reboot nodes
- creating files on nodes while rebooting
- validate for rofs
- wait for volume processes to be online
- creating files on nodes after rebooting
- validate IO
- Do IO and check on subvols without nodes to reboot
(do for each subvol)
- get files to delete/create for nodes to be online
- delete files from mountpoint
- reboot nodes
- creating files on online nodes while rebooting other nodes
- validate IO
- Do IO and check and reboot two nodes on all subvols
- get files to delete/create for nodes to be offline
- delete files from mountpoint
- reboot nodes
- creating files on nodes while rebooting
- validate for rofs
- wait for volume processes to be online
- creating files on nodes after rebooting
- validate IO
"""
# pylint: disable=too-many-locals,too-many-statements,too-many-branches
# set cluster.quorum-type to auto
options = {"cluster.quorum-type": "auto"}
g.log.info("setting cluster.quorum-type to auto on volume %s",
self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for"
"volume %s" % (options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = "/usr/bin/env python %s create_files -f 30 %s" % (
self.script_upload_path, mount_obj.mountpoint)
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Do IO and check on subvols with nodes to reboot
subvols_dict = get_subvols(self.mnode, self.volname)
for subvol in subvols_dict['volume_subvols']:
# define nodes to reboot
brick_list = subvol[0:2]
nodes_to_reboot = []
for brick in brick_list:
node, brick_path = brick.split(':')
nodes_to_reboot.append(node)
# get files to delete/create for nodes to be offline
node, brick_path = brick_list[0].split(':')
ret, brick_file_list, _ = g.run(node, 'ls %s' % brick_path)
self.assertFalse(ret, 'Failed to ls files on %s' % node)
file_list = brick_file_list.splitlines()
# delete files from mountpoint
for mount_obj in self.mounts:
g.log.info("Deleting data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ('cd %s/ ; rm -rf %s' %
(mount_obj.mountpoint, ' '.join(file_list)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret, 'Failed to rm file on %s' % mount_obj.client_system)
g.log.info('Files %s are deleted', file_list)
# reboot nodes on subvol and wait while rebooting
g.log.info("Rebooting the nodes %s", nodes_to_reboot)
ret = reboot_nodes(nodes_to_reboot)
self.assertTrue(ret,
'Failed to reboot nodes %s ' % nodes_to_reboot)
# Creating files on nodes while rebooting
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = ("cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list)))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
g.log.info("Validating if IO failed with read-only filesystem")
ret = is_io_procs_fail_with_rofs(self, self.all_mounts_procs,
self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on read-only filesystem"))
self.io_validation_complete = True
g.log.info("EXPECTED: "
"Read-only file system in IO while creating file")
# check if nodes are online
counter = 0
timeout = 300
_rc = False
while counter < timeout:
ret, reboot_results = are_nodes_online(nodes_to_reboot)
if not ret:
g.log.info("Nodes are offline, Retry after 5 seconds ... ")
time.sleep(5)
counter = counter + 5
else:
_rc = True
break
if not _rc:
for node in reboot_results:
if reboot_results[node]:
g.log.info("Node %s is online", node)
else:
g.log.error(
"Node %s is offline even after "
"%d minutes", node, timeout / 60.0)
else:
g.log.info("All nodes %s are up and running", nodes_to_reboot)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(
self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Creating files on nodes after rebooting
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = ("cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list)))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Do IO and check on subvols without nodes to reboot
subvols_dict = get_subvols(self.mnode, self.volname)
for subvol in subvols_dict['volume_subvols']:
# define nodes to reboot
brick_list = subvol[0:2]
nodes_to_reboot = []
for brick in brick_list:
node, brick_path = brick.split(':')
nodes_to_reboot.append(node)
# get files to delete/create for nodes to be online
new_subvols_dict = get_subvols(self.mnode, self.volname)
subvol_to_operate = new_subvols_dict['volume_subvols']
subvol_to_operate.remove(subvol)
brick_list_subvol_online = subvol_to_operate[0]
node, brick_path_vol_online = \
brick_list_subvol_online[0].split(':')
ret, brick_file_list, _ = g.run(node,
'ls %s' % brick_path_vol_online)
self.assertFalse(ret, 'Failed to ls files on %s' % node)
file_list = brick_file_list.splitlines()
# delete files from mountpoint
for mount_obj in self.mounts:
g.log.info("Deleting data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ('cd %s/ ; rm -rf %s' %
(mount_obj.mountpoint, ' '.join(file_list)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret, 'Failed to rm file on %s' % mount_obj.client_system)
g.log.info('Files %s are deleted', file_list)
# reboot nodes on subvol and wait while rebooting
g.log.info("Rebooting the nodes %s", nodes_to_reboot)
ret = reboot_nodes(nodes_to_reboot)
self.assertTrue(ret,
'Failed to reboot nodes %s ' % nodes_to_reboot)
# Creating files on nodes while rebooting
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = ("cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list)))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# check if nodes are online
counter = 0
timeout = 300
_rc = False
while counter < timeout:
ret, reboot_results = are_nodes_online(nodes_to_reboot)
if not ret:
g.log.info("Nodes are offline, Retry after 5 seconds ... ")
time.sleep(5)
counter = counter + 5
else:
_rc = True
break
if not _rc:
for node in reboot_results:
if reboot_results[node]:
g.log.info("Node %s is online", node)
else:
g.log.error(
"Node %s is offline even after "
"%d minutes", node, timeout / 60.0)
else:
g.log.info("All nodes %s are up and running", nodes_to_reboot)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(
self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Do IO and check and reboot nodes on all subvols
subvols_dict = get_subvols(self.mnode, self.volname)
nodes_to_reboot = []
file_list_for_all_subvols = []
for subvol in subvols_dict['volume_subvols']:
# define nodes to reboot
brick_list = subvol[0:2]
for brick in brick_list:
node, brick_path = brick.split(':')
nodes_to_reboot.append(node)
# get files to delete/create for nodes to be offline
node, brick_path = brick_list[0].split(':')
ret, brick_file_list, _ = g.run(node, 'ls %s' % brick_path)
self.assertFalse(ret, 'Failed to ls files on %s' % node)
file_list = brick_file_list.splitlines()
file_list_for_all_subvols.append(file_list)
# delete files from mountpoint
for mount_obj in self.mounts:
g.log.info("Deleting data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ('cd %s/ ; rm -rf %s' %
(mount_obj.mountpoint, ' '.join(file_list)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, 'Failed to rm file on %s' % node)
g.log.info('Files %s are deleted', file_list)
# reboot nodes on subvol and wait while rebooting
g.log.info("Rebooting the nodes %s", nodes_to_reboot)
ret = reboot_nodes(nodes_to_reboot)
self.assertTrue(ret, 'Failed to reboot nodes %s ' % nodes_to_reboot)
# Creating files on nodes while rebooting
all_mounts_procs, all_mounts_procs_1, all_mounts_procs_2 = [], [], []
# Create files for 1-st subvol and get all_mounts_procs_1
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[0])))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_1.append(proc)
all_mounts_procs.append(all_mounts_procs_1)
# Create files for 2-st subvol and get all_mounts_procs_2
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[1])))
proc2 = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_2.append(proc2)
all_mounts_procs.append(all_mounts_procs_2)
for mounts_procs in all_mounts_procs:
# Validate IO
self.io_validation_complete = False
g.log.info("Validating if IO failed with read-only filesystem")
ret = is_io_procs_fail_with_rofs(self, mounts_procs, self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on read-only filesystem"))
self.io_validation_complete = True
g.log.info("EXPECTED: "
"Read-only file system in IO while creating file")
# check if nodes are online
counter = 0
timeout = 300
_rc = False
while counter < timeout:
ret, reboot_results = are_nodes_online(nodes_to_reboot)
if not ret:
g.log.info("Nodes are offline, Retry after 5 seconds ... ")
time.sleep(5)
counter = counter + 5
else:
_rc = True
break
if not _rc:
for node in reboot_results:
if reboot_results[node]:
g.log.info("Node %s is online", node)
else:
g.log.error("Node %s is offline even after "
"%d minutes", node, timeout / 60.0)
else:
g.log.info("All nodes %s are up and running", nodes_to_reboot)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Creating files on nodes after rebooting
all_mounts_procs, all_mounts_procs_1, all_mounts_procs_2 = [], [], []
# Create files for 1-st subvol and get all_mounts_procs_1
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[0])))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_1.append(proc)
all_mounts_procs.append(all_mounts_procs_1)
# Create files for 2-st subvol and get all_mounts_procs_2
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[1])))
proc2 = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_2.append(proc2)
all_mounts_procs.append(all_mounts_procs_2)
for mounts_procs in all_mounts_procs:
# Validate IO
self.io_validation_complete = False
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
def test_client_side_quorum_with_fixed_for_cross2(self):
"""
Test Script to verify the Client Side Quorum with fixed
for cross 2 volume
* Disable self heal daemom
* set cluster.quorum-type to fixed.
* start I/O( write and read )from the mount point - must succeed
* Bring down brick1
* start I/0 ( write and read ) - must succeed
* set the cluster.quorum-count to 1
* start I/0 ( write and read ) - must succeed
* set the cluster.quorum-count to 2
* start I/0 ( write and read ) - read must pass, write will fail
* bring back the brick1 online
* start I/0 ( write and read ) - must succeed
* Bring down brick2
* start I/0 ( write and read ) - read must pass, write will fail
* set the cluster.quorum-count to 1
* start I/0 ( write and read ) - must succeed
* cluster.quorum-count back to 2 and cluster.quorum-type to auto
* start I/0 ( write and read ) - must succeed
* Bring back brick2 online
* Bring down brick1
* start I/0 ( write and read ) - read must pass, write will fail
* set the quorum-type to none
* start I/0 ( write and read ) - must succeed
"""
# pylint: disable=too-many-branches,too-many-statements
# Disable self heal daemon
options = {"cluster.self-heal-daemon": "off"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# set cluster.quorum-type to fixed
options = {"cluster.quorum-type": "fixed"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/O( write ) - must succeed
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s", self.mounts)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# get the subvolumes
g.log.info("Starting to get sub-volumes for volume %s", self.volname)
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:", num_subvols)
# Bring down brick1 for all the subvolumes
subvolumes_first_brick_list = []
subvolumes_second_brick_list = []
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
g.log.info("sub-volume %s brick list : %s", i, subvol_brick_list)
subvolumes_first_brick_list.append(subvol_brick_list[0])
subvolumes_second_brick_list.append(subvol_brick_list[1])
g.log.info("Going to bring down the brick process "
"for %s", subvolumes_first_brick_list)
ret = bring_bricks_offline(self.volname, subvolumes_first_brick_list)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s successfully", subvolumes_first_brick_list)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s", self.mounts)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name second_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# set the cluster.quorum-count to 1
options = {"cluster.quorum-count": "1"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mount.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name third_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# set the cluster.quorum-count to 2
options = {"cluster.quorum-count": "2"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - read must pass, write will fail
g.log.info("Starting IO on mount......")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name fourth_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with Read Only File System")
ret, _ = is_io_procs_fail_with_rofs(self, all_mounts_procs,
self.mounts)
self.assertTrue(ret, ("Unexpected Error and IO successful"
" on Read-Only File System"))
g.log.info("EXPECTED Read-only file system in IO while creating file")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# bring back the brick1 online for all subvolumes
g.log.info("bringing up the bricks : %s online",
subvolumes_first_brick_list)
ret = bring_bricks_online(self.mnode, self.volname,
subvolumes_first_brick_list)
self.assertTrue(ret, ("Failed to brought the bricks %s online" %
subvolumes_first_brick_list))
g.log.info("Successfully brought the bricks %s online",
subvolumes_first_brick_list)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mount.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name fifth_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# Bring down brick2 for all the subvolumes
g.log.info("Going to bring down the brick process "
"for %s", subvolumes_second_brick_list)
ret = bring_bricks_offline(self.volname, subvolumes_second_brick_list)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s successfully", subvolumes_second_brick_list)
# start I/0 ( write and read ) - read must pass, write will fail
g.log.info("Start creating files on mounts.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name sixth_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with Read Only File System")
ret, _ = is_io_procs_fail_with_rofs(self, all_mounts_procs,
self.mounts)
self.assertTrue(ret, ("Unexpected Error and IO successful"
" on Read-Only File System"))
g.log.info("EXPECTED Read-only file system in IO while creating file")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# set the cluster.quorum-count to 1
options = {"cluster.quorum-count": "1"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mount.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name seventh_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# set cluster.quorum-type to auto and cluster.quorum-count back to 2
options = {"cluster.quorum-type": "auto", "cluster.quorum-count": "2"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mount.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name eigth_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# Bring back brick2 online for all the subvolumes
g.log.info("bringing up the bricks : %s online",
subvolumes_second_brick_list)
ret = bring_bricks_online(self.mnode, self.volname,
subvolumes_second_brick_list)
self.assertTrue(ret, ("Failed to brought the brick %s online" %
subvolumes_second_brick_list))
g.log.info("Successfully brought the brick %s online",
subvolumes_second_brick_list)
# Bring down brick1 again for all the subvolumes
g.log.info("Going to bring down the brick process "
"for %s", subvolumes_first_brick_list)
ret = bring_bricks_offline(self.volname, subvolumes_first_brick_list)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s successfully", subvolumes_first_brick_list)
# start I/0 ( write and read ) - read must pass, write will fail
g.log.info("Start creating files on mounts.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name ninth_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with Read Only File System")
ret, _ = is_io_procs_fail_with_rofs(self, all_mounts_procs,
self.mounts)
self.assertTrue(ret, ("Unexpected Error and IO successful"
" on Read-Only File System"))
g.log.info("EXPECTED Read-only file system in IO while creating file")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# set the quorum-type to none
options = {"cluster.quorum-type": "none"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mount.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name tenth_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# bring back the bricks online for all subvolumes
g.log.info("bringing up the brick : %s online",
subvolumes_first_brick_list)
ret = bring_bricks_online(self.mnode, self.volname,
subvolumes_first_brick_list)
self.assertTrue(ret, ("Failed to brought the brick %s online" %
subvolumes_first_brick_list))
g.log.info("Successfully brought the bricks")
def test_client_side_quorum_with_auto_option(self):
"""
Test Script to verify the Client Side Quorum with auto option
* set cluster.quorum-type to auto.
* start I/O from the mount point.
* kill 2 of the brick process from the each and every replica set
* perform ops
"""
# set cluster.quorum-type to auto
options = {"cluster.quorum-type": "auto"}
g.log.info("setting cluster.quorum-type to auto on "
"volume %s" % self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for"
"volume %s" % (options, self.volname)))
g.log.info("Sucessfully set %s for volume %s" %
(options, self.volname))
# write files on all mounts
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s" % self.mounts)
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating IO on mounts")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
# get the subvolumes
g.log.info("Starting to get sub-volumes for volume %s" % self.volname)
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:" % num_subvols)
# bring bricks offline( 2 bricks ) for all the subvolumes
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
g.log.info("sub-volume %s brick list : %s" %
(i, subvol_brick_list))
# For volume type: 1 * 2, bring 1 brick offline
if len(subvol_brick_list) == 2:
bricks_to_bring_offline = subvol_brick_list[0:1]
else:
bricks_to_bring_offline = subvol_brick_list[0:2]
g.log.info("Going to bring down the brick process "
"for %s" % bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s succesfully" % bricks_to_bring_offline)
# create 2 files named newfile0.txt and newfile1.txt
g.log.info("Start creating 2 files on all mounts...")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s create_files "
"-f 2 --base-file-name newfile %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with read-only filesystem")
ret = is_io_procs_fail_with_rofs(self, all_mounts_procs, self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successfull"
" on read-only filesystem"))
g.log.info("EXPECTED: Read-only file system in IO while creating file")
# create directory user1
g.log.info("Start creating directory on all mounts...")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s create_deep_dir "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with read-only filesystem")
ret = is_io_procs_fail_with_rofs(self, all_mounts_procs, self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successfull"
" on read-only filesystem"))
g.log.info("EXPECTED: Read-only file system in IO while"
" creating directory")
# create h/w link to file
g.log.info("Start creating hard link for file0.txt on all mounts")
for mount_obj in self.mounts:
cmd = "ln %s/file0.txt %s/file0.txt_hwlink" \
% (mount_obj.mountpoint, mount_obj.mountpoint)
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertTrue(ret, ("Unexpected error and creating hard link"
" successful on read-only filesystem"))
self.assertIn(
"Read-only file system", err,
"Read-only filesystem not found in "
"IO while truncating file")
g.log.info("EXPECTED: Read-only file system in IO")
# create s/w link
g.log.info("Start creating soft link for file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = "ln -s %s/file1.txt %s/file1.txt_swlink" %\
(mount_obj.mountpoint, mount_obj.mountpoint)
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertTrue(ret, ("Unexpected error and creating soft link"
" successful on read-only filesystem"))
self.assertIn(
"Read-only file system", err,
"Read-only filesystem not found in "
"IO while truncating file")
g.log.info("EXPECTED: Read-only file system in IO")
# append to file
g.log.info("Appending to file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = "cat %s/file0.txt >> %s/file1.txt" %\
(mount_obj.mountpoint, mount_obj.mountpoint)
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertTrue(ret, ("Unexpected error and append successful"
" on read-only filesystem"))
self.assertIn(
"Read-only file system", err,
"Read-only filesystem not found in "
"IO while truncating file")
g.log.info("EXPECTED: Read-only file system in IO")
# modify the file
g.log.info("Modifying file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = "echo 'Modify Contents' > %s/file1.txt"\
% (mount_obj.mountpoint)
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertTrue(ret, ("Unexpected error and modifying successful"
" on read-only filesystem"))
self.assertIn(
"Read-only file system", err,
"Read-only filesystem not found in "
"IO while truncating file")
g.log.info("EXPECTED: Read-only file system in IO")
# truncate the file
g.log.info("Truncating file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = "truncate -s 0 %s/file1.txt" % (mount_obj.mountpoint)
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertTrue(ret, ("Unexpected error and truncating file"
" successful on read-only filesystem"))
self.assertIn(
"Read-only file system", err,
"Read-only filesystem not found in "
"IO while truncating file")
g.log.info("EXPECTED: Read-only file system in IO")
# read the file
g.log.info("Starting reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("validating IO on all mounts")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "Reads failed on some of the clients")
g.log.info("Reads successful on all mounts")
# stat on file
g.log.info("stat on file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = "stat %s/file1.txt" % (mount_obj.mountpoint)
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, ("Unexpected error and stat on file fails"
" on read-only filesystem"))
g.log.info("stat on file is successfull on read-only filesystem")
# stat on dir
g.log.info("stat on directory on all mounts")
for mount_obj in self.mounts:
cmd = ("python %s stat %s" %
(self.script_upload_path, mount_obj.mountpoint))
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, ("Unexpected error and stat on directory"
" fails on read-only filesystem"))
g.log.info("stat on dir is successfull on read-only filesystem")
# ls on mount point
g.log.info("ls on mount point on all mounts")
for mount_obj in self.mounts:
cmd = ("python %s ls %s" %
(self.script_upload_path, mount_obj.mountpoint))
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, ("Unexpected error and listing file fails"
" on read-only filesystem"))
g.log.info("listing files is successfull on read-only filesystem")
def test_client_side_quorum_auto_local_to_volume_not_cluster(self):
"""
- create four volume as below
vol1->2x2
vol2->2x2
vol3->2x3
vol4->2x3
vol5->a pure distribute volume
- do IO to all vols
- set client side quorum to auto for vol1 and vol3
- get the client side quorum value for all vols and check for result
- bring down b0 on vol1 and b0 and b1 on vol3
- try to create files on all vols and check for result
"""
# pylint: disable=too-many-locals,too-many-statements
# Creating files for all volumes
for mount_point in self.mount_points:
self.all_mounts_procs = []
g.log.info('Creating files...')
command = ("python %s create_files -f 50 "
"--fixed-file-size 1k %s" %
(self.script_upload_path, mount_point))
proc = g.run_async(self.mounts[0].client_system, command)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
volumes_to_change_options = ['1', '3']
# set cluster.quorum-type to auto
for vol_number in volumes_to_change_options:
vol_name = ('testvol_distributed-replicated_%s' % vol_number)
options = {"cluster.quorum-type": "auto"}
g.log.info(
"setting cluster.quorum-type to auto on "
"volume testvol_distributed-replicated_%s", vol_number)
ret = set_volume_options(self.mnode, vol_name, options)
self.assertTrue(ret, ("Unable to set volume option %s for "
"volume %s" % (options, vol_name)))
g.log.info("Successfully set %s for volume %s", options, vol_name)
# check is options are set correctly
volume_list = get_volume_list(self.mnode)
for volume in volume_list:
g.log.info('Checking for cluster.quorum-type option for %s',
volume)
volume_options_dict = get_volume_options(self.mnode, volume,
'cluster.quorum-type')
if (volume == 'testvol_distributed-replicated_1'
or volume == 'testvol_distributed-replicated_3'
or volume == 'testvol_distributed-replicated_4'):
self.assertEqual(
volume_options_dict['cluster.quorum-type'], 'auto',
'Option cluster.quorum-type '
'is not AUTO for %s' % volume)
g.log.info('Option cluster.quorum-type is AUTO for %s', volume)
else:
self.assertEqual(
volume_options_dict['cluster.quorum-type'], 'none',
'Option cluster.quorum-type '
'is not NONE for %s' % volume)
g.log.info('Option cluster.quorum-type is NONE for %s', volume)
# Get first brick server and brick path
# and get first file from filelist then delete it from volume
vols_file_list = {}
for volume in volume_list:
brick_list = get_all_bricks(self.mnode, volume)
brick_server, brick_path = brick_list[0].split(':')
ret, file_list, _ = g.run(brick_server, 'ls %s' % brick_path)
self.assertFalse(ret, 'Failed to ls files on %s' % brick_server)
file_from_vol = file_list.splitlines()[0]
ret, _, _ = g.run(brick_server,
'rm -rf %s/%s' % (brick_path, file_from_vol))
self.assertFalse(ret, 'Failed to rm file on %s' % brick_server)
vols_file_list[volume] = file_from_vol
# bring bricks offline
# bring first brick for testvol_distributed-replicated_1
volname = 'testvol_distributed-replicated_1'
brick_list = get_all_bricks(self.mnode, volname)
bricks_to_bring_offline = brick_list[0:1]
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, volname, bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# bring first two bricks for testvol_distributed-replicated_3
volname = 'testvol_distributed-replicated_3'
brick_list = get_all_bricks(self.mnode, volname)
bricks_to_bring_offline = brick_list[0:2]
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, volname, bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# merge two dicts (volname: file_to_delete) and (volname: mountpoint)
temp_dict = [vols_file_list, self.mount_points_and_volnames]
file_to_delete_to_mountpoint_dict = {}
for k in vols_file_list.iterkeys():
file_to_delete_to_mountpoint_dict[k] = (tuple(
file_to_delete_to_mountpoint_dict[k]
for file_to_delete_to_mountpoint_dict in temp_dict))
# create files on all volumes and check for result
for volname, file_and_mountpoint in \
file_to_delete_to_mountpoint_dict.iteritems():
filename, mountpoint = file_and_mountpoint
# check for ROFS error for read-only file system for
# testvol_distributed-replicated_1 and
# testvol_distributed-replicated_3
if (volname == 'testvol_distributed-replicated_1'
or volname == 'testvol_distributed-replicated_3'):
# create new file taken from vols_file_list
g.log.info("Start creating new file on all mounts...")
all_mounts_procs = []
cmd = ("touch %s/%s" % (mountpoint, filename))
proc = g.run_async(self.client, cmd)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating if IO failed with read-only filesystem")
ret, _ = is_io_procs_fail_with_rofs(self, all_mounts_procs,
self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on read-only filesystem"))
g.log.info("EXPECTED: "
"Read-only file system in IO while creating file")
# check for no errors for all the rest volumes
else:
# create new file taken from vols_file_list
g.log.info("Start creating new file on all mounts...")
all_mounts_procs = []
cmd = ("touch %s/%s" % (mountpoint, filename))
proc = g.run_async(self.client, cmd)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")