def test_add_brick_already_part_of_another_volume(self):
""" Test adding bricks to the volume which are already part of another
volume.
"""
# create and start a volume
self.volume['name'] = "existing_volume"
self.volname = "existing_volume"
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
g.log.info("Volume created and started successfully")
sub_vols = get_subvols(self.mnode, self.volname)['volume_subvols']
# create and start a new volume
self.volume['name'] = "new_volume"
self.volname = "new_volume"
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
g.log.info("Volume created and started successfully")
cmd = ("gluster volume add-brick %s %s " % (self.volname,
' '.join(sub_vols[0])))
g.log.info("Adding bricks to volume %s which are already part of an"
"another volume", self.volname)
_, _, err = g.run(self.mnode, cmd)
self.assertIn("Brick may be containing or be contained by an existing"
" brick", err, "add-brick is successful")
g.log.info("Volume add-brick failed with error %s ", err)
def setup_samba_ctdb_cluster(cls):
"""
Create ctdb-samba cluster if doesn't exists
Returns:
bool: True if successfully setup samba else false
"""
# Check if ctdb setup is up and running
if is_ctdb_status_healthy(cls.primary_node):
g.log.info("ctdb setup already up skipping " "ctdb setup creation")
return True
g.log.info("Proceeding with ctdb setup creation")
for mnode in cls.servers:
ret = edit_hook_script(mnode, cls.ctdb_volname)
if not ret:
return False
ret = enable_ctdb_cluster(mnode)
if not ret:
return False
ret = create_nodes_file(mnode, cls.ctdb_nodes)
if not ret:
return False
ret = create_public_address_file(mnode, cls.ctdb_vips)
if not ret:
return False
server_info = cls.all_servers_info
ctdb_config = cls.ctdb_volume_config
g.log.info("Setting up ctdb volume %s", cls.ctdb_volname)
ret = setup_volume(mnode=cls.primary_node,
all_servers_info=server_info,
volume_config=ctdb_config)
if not ret:
g.log.error("Failed to setup ctdb volume %s", cls.ctdb_volname)
return False
g.log.info("Successful in setting up volume %s", cls.ctdb_volname)
# Wait for volume processes to be online
g.log.info("Wait for volume %s processes to be online",
cls.ctdb_volname)
ret = wait_for_volume_process_to_be_online(cls.mnode, cls.ctdb_volname)
if not ret:
g.log.error(
"Failed to wait for volume %s processes to "
"be online", cls.ctdb_volname)
return False
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", cls.ctdb_volname)
# start ctdb services
ret = start_ctdb_service(cls.servers)
if not ret:
return False
ret = is_ctdb_status_healthy(cls.primary_node)
if not ret:
g.log.error("CTDB setup creation failed - exiting")
return False
g.log.info("CTDB setup creation successfull")
return True
def setUp(self):
self.get_super_method(self, 'setUp')()
# Overriding the volume type to specifically test the volume type
if self.volume_type == "distributed-replicated":
self.volume['voltype'] = {
'type': 'distributed-replicated',
'replica_count': 2,
'dist_count': 4,
'transport': 'tcp'
}
# Create a distributed-replicated volume with replica count 2
# using first four nodes
servers_info_from_four_nodes = {}
for server in self.servers[0:4]:
servers_info_from_four_nodes[server] = self.all_servers_info[
server]
self.volume['servers'] = self.servers[0:4]
ret = setup_volume(self.mnode,
servers_info_from_four_nodes,
self.volume,
force=False)
if not ret:
raise ExecutionError("Volume create failed on four nodes")
g.log.info("Distributed replicated volume created successfully")
def setUp(self):
# Calling GlusterBaseClass setUp
GlusterBaseClass.setUp.im_func(self)
self.extra_servers = self.servers[-2:]
self.servers = self.servers[:-2]
# Performing peer detach
for server in self.extra_servers:
# Peer detach
ret, _, _ = peer_detach(self.mnode, server)
if ret:
raise ExecutionError("Peer detach failed")
g.log.info("Peer detach successful.")
# Create volume using first four nodes
servers_info_from_four_nodes = {}
for server in self.servers:
servers_info_from_four_nodes[server] = self.all_servers_info[
server]
self.volume['servers'] = self.servers
ret = setup_volume(self.mnode,
servers_info_from_four_nodes,
self.volume,
force=False)
if not ret:
raise ExecutionError("Volume create failed on four nodes")
g.log.info("Distributed replicated volume created successfully")
# Verfiy glustershd process releases its parent process
ret = is_shd_daemonized(self.servers)
if not ret:
raise ExecutionError("Self Heal Daemon process was still"
" holding parent process.")
g.log.info("Self Heal Daemon processes are online")
def setUp(self):
GlusterBaseClass.setUp.im_func(self)
# Create and start a volume.
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
if ret:
ExecutionError("Failed to create and start volume")
def setUp(self):
GlusterBaseClass.setUp.im_func(self)
self.test_method_complete = False
# Creating a volume and starting it
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
if not ret:
raise ExecutionError("Failed to create volume")
g.log.info("Volume created successfully")
def setUp(self):
self.get_super_method(self, 'setUp')()
# Create and start a volume.
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
if ret:
ExecutionError("Failed to create and start volume")
def setUp(self):
self.get_super_method(self, 'setUp')()
self.test_method_complete = False
# Creating a volume and starting it
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
if not ret:
raise ExecutionError("Failed to create volume")
g.log.info("Volume created successfully")
def test_volume_reduce_replica_count(self):
"""
Test case:
1) Create a 2x3 replica volume.
2) Remove bricks in the volume to make it a 2x2 replica volume.
3) Remove bricks in the volume to make it a distribute volume.
"""
# Create and start a volume
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
# Getting a list of all the bricks.
g.log.info("Get all the bricks of the volume")
self.brick_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(self.brick_list, "Failed to get the brick list")
g.log.info("Successfully got the list of bricks of volume")
# Converting 2x3 to 2x2 volume.
remove_brick_list = [self.brick_list[0], self.brick_list[3]]
ret, _, _ = remove_brick(self.mnode,
self.volname,
remove_brick_list,
'force',
replica_count=2)
self.assertEqual(ret, 0, "Failed to start remove brick operation")
g.log.info("Remove brick operation successfully")
# Checking if volume is 2x2 or not.
volume_info = get_volume_info(self.mnode, self.volname)
brick_count = int(volume_info[self.volname]['brickCount'])
self.assertEqual(brick_count, 4, "Failed to remove 2 bricks.")
g.log.info("Successfully removed 2 bricks.")
type_string = volume_info[self.volname]['typeStr']
self.assertEqual(type_string, 'Distributed-Replicate',
"Convertion to 2x2 failed.")
g.log.info("Convertion to 2x2 successful.")
# Converting 2x2 to distribute volume.
remove_brick_list = [self.brick_list[1], self.brick_list[4]]
ret, _, _ = remove_brick(self.mnode,
self.volname,
remove_brick_list,
'force',
replica_count=1)
self.assertEqual(ret, 0, "Failed to start remove brick operation")
g.log.info("Remove brick operation successfully")
# Checking if volume is pure distribute or not.
volume_info = get_volume_info(self.mnode, self.volname)
brick_count = int(volume_info[self.volname]['brickCount'])
self.assertEqual(brick_count, 2, "Failed to remove 2 bricks.")
g.log.info("Successfully removed 2 bricks.")
type_string = volume_info[self.volname]['typeStr']
self.assertEqual(type_string, 'Distribute',
"Convertion to distributed failed.")
g.log.info("Convertion to distributed successful.")
def setUp(self):
"""
setUp method for every test
"""
# calling GlusterBaseClass setUp
GlusterBaseClass.setUp.im_func(self)
self.volume_list = []
# create a volume
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.volume_list.append(self.volname)
if not ret:
raise ExecutionError("Volume creation failed: %s" % self.volname)
# Creating another volume
second_volume = "second_volume"
self.volume['name'] = second_volume
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.volume_list.append(second_volume)
if not ret:
raise ExecutionError("Volume creation failed: %s" % second_volume)
def setUpClass(cls):
"""
setup volume and initialize necessary variables
which is used in tests
"""
# calling GlusterBaseClass setUpClass
cls.get_super_method(cls, 'setUpClass')()
list_of_vol = [
'distributed-dispersed', 'replicated', 'dispersed', 'distributed',
'distributed-replicated'
]
cls.volume_configs = []
if cls.default_volume_type_config['distributed']['dist_count'] > 3:
cls.default_volume_type_config['distributed']['dist_count'] = 3
for volume_type in list_of_vol:
cls.volume_configs.append({
'name':
'testvol_%s' % (volume_type),
'servers':
cls.servers,
'voltype':
cls.default_volume_type_config[volume_type]
})
for volume_config in cls.volume_configs:
ret = setup_volume(mnode=cls.mnode,
all_servers_info=cls.all_servers_info,
volume_config=volume_config,
multi_vol=True)
volname = volume_config['name']
if not ret:
raise ExecutionError("Failed to setup Volume" " %s" % volname)
g.log.info("Successful in setting volume %s", volname)
# Verify volume's all process are online for 60 sec
g.log.info("Verifying volume's all process are online")
ret = wait_for_volume_process_to_be_online(cls.mnode, volname, 60)
if not ret:
raise ExecutionError("Volume %s : All process are not online" %
volname)
g.log.info("Successfully Verified volume %s processes are online",
volname)
# Verfiy glustershd process releases its parent process
g.log.info("Verifying Self Heal Daemon process is daemonized")
ret = is_shd_daemonized(cls.servers)
if not ret:
raise ExecutionError("Self Heal Daemon process was still"
" holding parent process.")
g.log.info("Self Heal Daemon processes are online")
def setup_volume(cls, volume_create_force=False):
"""Setup the volume:
- Create the volume, Start volume, Set volume
options, enable snapshot/quota/tier if specified in the config
file.
- Wait for volume processes to be online
- Export volume as NFS/SMB share if mount_type is NFS or SMB
- Log volume info and status
Args:
volume_create_force(bool): True if create_volume should be
executed with 'force' option.
Returns (bool): True if all the steps mentioned in the descriptions
passes. False otherwise.
"""
force_volume_create = False
if cls.volume_create_force:
force_volume_create = True
# Validate peers before setting up volume
g.log.info("Validate peers before setting up volume ")
ret = cls.validate_peers_are_connected()
if not ret:
g.log.error("Failed to validate peers are in connected state "
"before setting up volume")
return False
g.log.info("Successfully validated peers are in connected state "
"before setting up volume")
# Setup Volume
g.log.info("Setting up volume %s", cls.volname)
ret = setup_volume(mnode=cls.mnode,
all_servers_info=cls.all_servers_info,
volume_config=cls.volume, force=force_volume_create)
if not ret:
g.log.error("Failed to Setup volume %s", cls.volname)
return False
g.log.info("Successful in setting up volume %s", cls.volname)
# ToDo : Wait for volume processes to be online
# Log Volume Info and Status
g.log.info("Log Volume %s Info and Status", cls.volname)
ret = log_volume_info_and_status(cls.mnode, cls.volname)
if not ret:
g.log.error("Logging volume %s info and status failed",
cls.volname)
return False
g.log.info("Successful in logging volume %s info and status",
cls.volname)
return True
def test_enable_brickmux_create_and_stop_three_volumes(self):
"""
Test Case:
1.Set cluster.brick-multiplex to enabled.
2.Create three 1x3 replica volumes.
3.Start all the three volumes.
4.Stop three volumes one by one.
"""
# Timestamp of current test case of start time
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
# Setting cluster.brick-multiplex to enable
ret = set_volume_options(self.mnode, 'all',
{'cluster.brick-multiplex': 'enable'})
self.assertTrue(ret, "Failed to set brick-multiplex to enable.")
g.log.info("Successfully set brick-multiplex to enable.")
# Create and start 3 volume
for number in range(1, 4):
self.volume['name'] = ("test_volume_%s" % number)
self.volname = ("test_volume_%s" % number)
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret,
"Failed to create and start %s" % self.volname)
g.log.info("Successfully created and started volume %s.",
self.volname)
# Checking brick process count.
for brick in get_all_bricks(self.mnode, self.volname):
server = brick.split(":")[0]
count = get_brick_processes_count(server)
self.assertEqual(
count, 1, "ERROR: More than one brick process on %s." % server)
g.log.info("Only one brick process present on %s", server)
# Stop three volumes one by one.
for number in range(1, 4):
self.volume['name'] = ("test_volume_%s" % number)
self.volname = ("test_volume_%s" % number)
ret, _, _ = volume_stop(self.mnode, self.volname)
self.assertEqual(ret, 0,
"Failed to stop the volume %s" % self.volname)
g.log.info("Volume %s stopped successfully", self.volname)
# Checking for core files.
ret = is_core_file_created(self.servers, test_timestamp)
self.assertTrue(ret, "Core file found.")
g.log.info("No core files found, glusterd service running "
"successfully")
def use_config_setup_volumes(self):
"""
A function to setup volumes based on volume_configs.
"""
for volume_config in self.volume_configs:
ret = setup_volume(mnode=self.mnode,
all_servers_info=self.all_servers_info,
volume_config=volume_config,
force=False)
if not ret:
raise ExecutionError("Failed to setup Volume %s"
% volume_config['name'])
g.log.info("Successful in setting volume %s",
volume_config['name'])
def test_glusterd_start_stop_cycle(self):
"""
Test Glusterd stop-start cycle of gluster processes.
1. Create a gluster volume.
2. Kill all gluster related processes.
3. Start glusterd service.
4. Verify that all gluster processes are up.
5. Repeat the above steps 5 times.
"""
# Create and start a volume
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
for _ in range(5):
killed_gluster_process_count = []
# Kill gluster processes in all servers
for server in self.servers:
cmd = ('pkill --signal 9 -c -e "(glusterd|glusterfsd|glusterfs'
')"|tail -1')
ret, out, err = g.run(server, cmd)
self.assertEqual(ret, 0, err)
killed_gluster_process_count.append(int(out))
# Start glusterd on all servers.
ret = start_glusterd(self.servers)
self.assertTrue(ret, ("Failed to restart glusterd on desired"
" nodes."))
g.log.info("Glusterd started on desired nodes.")
# Wait for gluster processes to come up.
self._wait_for_gluster_process_online_state()
spawned_gluster_process_count = []
# Get number of gluster processes spawned in all server
for server in self.servers:
cmd = ('pgrep -c "(glusterd|glusterfsd|glusterfs)"')
ret, out, err = g.run(server, cmd)
self.assertEqual(ret, 0, err)
spawned_gluster_process_count.append(int(out))
# Compare process count in each server.
for index, server in enumerate(self.servers):
self.assertEqual(
killed_gluster_process_count[index],
spawned_gluster_process_count[index],
("All processes not up and running on %s", server))
def setUp(self):
"""
setup volume and initialize necessary variables
which is used in tests
"""
# Calling GlusterBaseClass setUp
self.get_super_method(self, 'setUp')()
# Setup Volume for all the volume types
self.volume_configs = []
for volume_type in self.default_volume_type_config:
self.volume_configs.append({
'name':
'testvol_%s' % volume_type,
'servers':
self.servers,
'voltype':
self.default_volume_type_config[volume_type]
})
for volume_config in self.volume_configs[1:]:
ret = setup_volume(mnode=self.mnode,
all_servers_info=self.all_servers_info,
volume_config=volume_config,
multi_vol=True)
volname = volume_config['name']
if not ret:
raise ExecutionError("Failed to setup Volume" " %s" % volname)
g.log.info("Successful in setting volume %s", volname)
# Verify volume's all process are online for 60 sec
ret = wait_for_volume_process_to_be_online(self.mnode, volname, 60)
if not ret:
raise ExecutionError("Volume %s : All process are not online" %
volname)
g.log.info("Successfully Verified volume %s processes are online",
volname)
# Verfiy glustershd process releases its parent process
ret = is_shd_daemonized(self.servers)
if not ret:
raise ExecutionError("Self Heal Daemon process was still"
" holding parent process.")
g.log.info("Self Heal Daemon processes are online")
self.glustershd = "/var/lib/glusterd/glustershd/glustershd-server.vol"
def test_uuid_in_volume_info_xml(self):
# create a two node cluster
ret = peer_probe_servers(self.servers[0], self.servers[1])
self.assertTrue(
ret,
"Peer probe failed to %s from %s" % (self.mnode, self.servers[1]))
# create a 2x2 volume
servers_info_from_two_node_cluster = {}
for server in self.servers[0:2]:
servers_info_from_two_node_cluster[server] = self.all_servers_info[
server]
self.volume['servers'] = self.servers[0:2]
self.volume['voltype']['replica_count'] = 2
self.volume['voltype']['dist_count'] = 2
ret = setup_volume(self.mnode, servers_info_from_two_node_cluster,
self.volume)
self.assertTrue(ret, ("Failed to create"
"and start volume %s" % self.volname))
# probe a new node from cluster
ret = peer_probe_servers(self.mnode, self.servers[2])
self.assertTrue(
ret,
"Peer probe failed to %s from %s" % (self.mnode, self.servers[2]))
# check gluster vol info --xml from newly probed node
xml_output = get_volume_info(self.servers[2], self.volname)
self.assertIsNotNone(xml_output,
("Failed to get volume info --xml for"
"volume %s from newly probed node %s" %
(self.volname, self.servers[2])))
# volume info --xml should have non zero UUID for host and brick
uuid_with_zeros = '00000000-0000-0000-0000-000000000000'
len_of_uuid = len(uuid_with_zeros)
number_of_bricks = int(xml_output[self.volname]['brickCount'])
for i in range(number_of_bricks):
uuid = xml_output[self.volname]['bricks']['brick'][i]['hostUuid']
self.assertEqual(len(uuid), len_of_uuid, "Invalid uuid length")
self.assertNotEqual(uuid, uuid_with_zeros,
("Invalid uuid %s" % uuid))
def test_setting_vol_option_with_max_characters(self):
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, ("Failed to create "
"and start volume %s" % self.volname))
auth_list = []
for ip_addr in range(256):
auth_list.append('192.168.122.%d' % ip_addr)
for ip_addr in range(7):
auth_list.append('192.168.123.%d' % ip_addr)
ip_list = ','.join(auth_list)
# set auth.allow with <4096 characters and restart the glusterd
g.log.info("Setting auth.allow with string of length %d for %s",
len(ip_list), self.volname)
self.options = {"auth.allow": ip_list}
ret = set_volume_options(self.mnode, self.volname, self.options)
self.assertTrue(ret, ("Failed to set auth.allow with string of length"
" %d for %s" % (len(ip_list), self.volname)))
ret = restart_glusterd(self.mnode)
self.assertTrue(ret,
"Failed to restart the glusterd on %s" % self.mnode)
# set auth.allow with >4096 characters and restart the glusterd
ip_list = ip_list + ",192.168.123.7"
self.options = {"auth.allow": ip_list}
g.log.info("Setting auth.allow with string of length %d for %s",
len(ip_list), self.volname)
ret = set_volume_options(self.mnode, self.volname, self.options)
self.assertTrue(ret, ("Failed to set auth.allow with string of length"
" %d for %s" % (len(ip_list), self.volname)))
ret = restart_glusterd(self.mnode)
self.assertTrue(ret,
"Failed to restart the glusterd on %s" % self.mnode)
count = 0
while count < 60:
ret = is_glusterd_running(self.mnode)
if not ret:
break
sleep(2)
count += 1
self.assertEqual(ret, 0, "glusterd is not running on %s" % self.mnode)
def test_setting_vol_option_with_max_characters(self):
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, ("Failed to create "
"and start volume %s" % self.volname))
auth_list = []
for ip_addr in range(256):
auth_list.append('192.168.122.%d' % ip_addr)
for ip_addr in range(7):
auth_list.append('192.168.123.%d' % ip_addr)
ip_list = ','.join(auth_list)
# set auth.allow with <4096 characters and restart the glusterd
g.log.info("Setting auth.allow with string of length %d for %s",
len(ip_list), self.volname)
self.options = {"auth.allow": ip_list}
ret = set_volume_options(self.mnode, self.volname, self.options)
self.assertTrue(ret, ("Failed to set auth.allow with string of length"
" %d for %s" % (len(ip_list), self.volname)))
ret = restart_glusterd(self.mnode)
self.assertTrue(ret, "Failed to restart the glusterd on %s"
% self.mnode)
# set auth.allow with >4096 characters and restart the glusterd
ip_list = ip_list + ",192.168.123.7"
self.options = {"auth.allow": ip_list}
g.log.info("Setting auth.allow with string of length %d for %s",
len(ip_list), self.volname)
ret = set_volume_options(self.mnode, self.volname, self.options)
self.assertTrue(ret, ("Failed to set auth.allow with string of length"
" %d for %s" % (len(ip_list), self.volname)))
ret = restart_glusterd(self.mnode)
self.assertTrue(ret, "Failed to restart the glusterd on %s"
% self.mnode)
ret = wait_for_glusterd_to_start(self.servers)
self.assertTrue(ret, "glusterd is not running on %s"
% self.servers)
g.log.info("Glusterd start on the nodes : %s "
"succeeded", self.servers)
def test_vol_delete_when_one_of_nodes_is_down(self):
# create a volume and start it
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start the volume")
g.log.info("Successfully created and started the volume")
# get the bricks list
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, "Failed to get the bricks list")
# get a random node other than self.mnode
if len(bricks_list) >= len(self.servers):
random_index = random.randint(1, len(self.servers) - 1)
else:
random_index = random.randint(1, len(bricks_list) - 1)
# stop glusterd on the random node
node_to_stop_glusterd = self.servers[random_index]
ret = stop_glusterd(node_to_stop_glusterd)
self.assertTrue(ret, "Failed to stop glusterd")
# stop the volume, it should succeed
ret, _, _ = volume_stop(self.mnode, self.volname)
self.assertEqual(ret, 0, "Volume stop failed")
# try to delete the volume, it should fail
ret, out, err = g.run(
self.mnode, "gluster volume delete %s "
"--mode=script" % self.volname)
self.assertNotEqual(
ret, 0, "Volume delete succeeded when one of the"
" brick node is down")
if re.search(r'Some of the peers are down', err):
g.log.info("Volume delete failed with expected error message")
else:
g.log.info("Volume delete failed with unexpected error message")
def setUpClass(cls):
"""Setup volume exports volume with nfs-ganesha,
mounts the volume.
"""
NfsGaneshaClusterSetupClass.setUpClass.im_func(cls)
# Peer probe servers
ret = peer_probe_servers(cls.mnode, cls.servers)
if not ret:
raise ExecutionError("Failed to peer probe servers")
g.log.info("All peers are in connected state")
# Peer Status from mnode
peer_status(cls.mnode)
for server in cls.servers:
mount_info = [{
'protocol': 'glusterfs',
'mountpoint': '/run/gluster/shared_storage',
'server': server,
'client': {
'host': server
},
'volname': 'gluster_shared_storage',
'options': ''
}]
mount_obj = create_mount_objs(mount_info)
if not mount_obj[0].is_mounted():
ret = mount_obj[0].mount()
if not ret:
raise ExecutionError(
"Unable to mount volume '%s:%s' "
"on '%s:%s'" %
(mount_obj.server_system, mount_obj.volname,
mount_obj.client_system, mount_obj.mountpoint))
# Setup Volume
ret = setup_volume(mnode=cls.mnode,
all_servers_info=cls.all_servers_info,
volume_config=cls.volume,
force=True)
if not ret:
raise ExecutionError("Setup volume %s failed", cls.volume)
time.sleep(10)
# Export volume with nfs ganesha, if it is not exported already
vol_option = get_volume_options(cls.mnode,
cls.volname,
option='ganesha.enable')
if vol_option is None:
raise ExecutionError("Failed to get ganesha.enable volume option "
"for %s " % cls.volume)
if vol_option['ganesha.enable'] != 'on':
ret, out, err = export_nfs_ganesha_volume(mnode=cls.mnode,
volname=cls.volname)
if ret != 0:
raise ExecutionError(
"Failed to export volume %s "
"as NFS export", cls.volname)
time.sleep(5)
ret = wait_for_nfs_ganesha_volume_to_get_exported(
cls.mnode, cls.volname)
if not ret:
raise ExecutionError("Failed to export volume %s. volume is "
"not listed in showmount" % cls.volname)
else:
g.log.info("Volume %s is exported successfully" % cls.volname)
# Log Volume Info and Status
ret = log_volume_info_and_status(cls.mnode, cls.volname)
if not ret:
raise ExecutionError("Logging volume %s info and status failed",
cls.volname)
# Create Mounts
_rc = True
for mount_obj in cls.mounts:
ret = mount_obj.mount()
if not ret:
g.log.error("Unable to mount volume '%s:%s' on '%s:%s'",
mount_obj.server_system, mount_obj.volname,
mount_obj.client_system, mount_obj.mountpoint)
_rc = False
if not _rc:
raise ExecutionError("Mounting volume %s on few clients failed",
cls.volname)
# Get info of mount before the IO
log_mounts_info(cls.mounts)
def setup_volume(cls, volume_create_force=False):
"""Setup the volume:
- Create the volume, Start volume, Set volume
options, enable snapshot/quota/tier if specified in the config
file.
- Wait for volume processes to be online
- Export volume as NFS/SMB share if mount_type is NFS or SMB
- Log volume info and status
Args:
volume_create_force(bool): True if create_volume should be
executed with 'force' option.
Returns (bool): True if all the steps mentioned in the descriptions
passes. False otherwise.
"""
force_volume_create = False
if volume_create_force or cls.volume_create_force:
force_volume_create = True
# Validate peers before setting up volume
g.log.info("Validate peers before setting up volume ")
ret = cls.validate_peers_are_connected()
if not ret:
g.log.error("Failed to validate peers are in connected state "
"before setting up volume")
return False
g.log.info("Successfully validated peers are in connected state "
"before setting up volume")
# Setup Volume
g.log.info("Setting up volume %s", cls.volname)
ret = setup_volume(mnode=cls.mnode,
all_servers_info=cls.all_servers_info,
volume_config=cls.volume,
force=force_volume_create)
if not ret:
g.log.error("Failed to Setup volume %s", cls.volname)
return False
g.log.info("Successful in setting up volume %s", cls.volname)
# Wait for volume processes to be online
g.log.info("Wait for volume %s processes to be online", cls.volname)
ret = wait_for_volume_process_to_be_online(cls.mnode, cls.volname)
if not ret:
g.log.error(
"Failed to wait for volume %s processes to "
"be online", cls.volname)
return False
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", cls.volname)
# Export/Share the volume based on mount_type
if cls.mount_type != "glusterfs":
g.log.info("Export/Sharing the volume %s", cls.volname)
if "nfs" in cls.mount_type:
ret = export_volume_through_nfs(
mnode=cls.mnode,
volname=cls.volname,
enable_ganesha=cls.enable_nfs_ganesha)
if not ret:
g.log.error("Failed to export volume %s "
"as NFS export", cls.volname)
return False
g.log.info(
"Successful in exporting the volume %s "
"as NFS export", cls.volname)
# Set NFS-Ganesha specific volume options
if cls.enable_nfs_ganesha and cls.nfs_ganesha_export_options:
g.log.info(
"Setting NFS-Ganesha export specific "
"volume options on volume %s", cls.volname)
ret = set_volume_options(
mnode=cls.mnode,
volname=cls.volname,
options=cls.nfs_ganesha_export_options)
if not ret:
g.log.error(
"Failed to set NFS-Ganesha "
"export specific options on "
"volume %s", cls.volname)
return False
g.log.info(
"Successful in setting NFS-Ganesha export "
"specific volume options on volume %s", cls.volname)
if "smb" in cls.mount_type or "cifs" in cls.mount_type:
ret = share_volume_over_smb(mnode=cls.mnode,
volname=cls.volname,
smb_users_info=cls.smb_users_info)
if not ret:
g.log.error("Failed to export volume %s "
"as SMB Share", cls.volname)
return False
g.log.info("Successful in exporting volume %s as SMB Share",
cls.volname)
# Set SMB share specific volume options
if cls.smb_share_options:
g.log.info(
"Setting SMB share specific volume options "
"on volume %s", cls.volname)
ret = set_volume_options(mnode=cls.mnode,
volname=cls.volname,
options=cls.smb_share_options)
if not ret:
g.log.error(
"Failed to set SMB share "
"specific options "
"on volume %s", cls.volname)
return False
g.log.info(
"Successful in setting SMB share specific "
"volume options on volume %s", cls.volname)
# Log Volume Info and Status
g.log.info("Log Volume %s Info and Status", cls.volname)
ret = log_volume_info_and_status(cls.mnode, cls.volname)
if not ret:
g.log.error("Logging volume %s info and status failed",
cls.volname)
return False
g.log.info("Successful in logging volume %s info and status",
cls.volname)
return True
def test_create_vol_used_bricks(self):
'''
-> Create distributed-replica Volume
-> Add 6 bricks to the volume
-> Mount the volume
-> Perform some I/O's on mount point
-> unmount the volume
-> Stop and delete the volume
-> Create another volume using bricks of deleted volume
'''
# Create and start a volume
self.volume['name'] = "test_create_vol_with_fresh_bricks"
self.volname = "test_create_vol_with_fresh_bricks"
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
# Forming brick list
brick_list = form_bricks_list(self.mnode, self.volname, 6,
self.servers, self.all_servers_info)
# Adding bricks to the volume
ret, _, _ = add_brick(self.mnode, self.volname, brick_list)
self.assertEqual(
ret, 0, "Failed to add bricks to the volume %s" % self.volname)
g.log.info("Bricks added successfully to the volume %s", self.volname)
# Mounting the volume.
for mount_obj in self.mounts:
ret, _, _ = mount_volume(self.volname,
mtype=self.mount_type,
mpoint=mount_obj.mountpoint,
mserver=self.mnode,
mclient=mount_obj.client_system)
self.assertEqual(ret, 0,
("Volume %s is not mounted") % (self.volname))
g.log.info("Volume mounted successfully : %s", self.volname)
# run IOs
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = (
"/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d --dir-depth 2 "
"--dir-length 5 --max-num-of-dirs 3 "
"--num-of-files 10 %s" %
(self.script_upload_path, self.counter, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.counter = self.counter + 10
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Unmouting the volume.
for mount_obj in self.mounts:
ret, _, _ = umount_volume(mclient=mount_obj.client_system,
mpoint=mount_obj.mountpoint)
self.assertEqual(ret, 0,
"Volume %s is not unmounted" % (self.volname))
g.log.info("Volume unmounted successfully : %s", self.volname)
# Getting brick list
self.brick_list = get_all_bricks(self.mnode, self.volname)
if not self.brick_list:
raise ExecutionError("Failed to get the brick list of %s" %
self.volname)
# Stop volume
ret, _, _ = volume_stop(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to stop the volume %s" % self.volname)
g.log.info("Volume %s stopped successfully", self.volname)
# Delete Volume
ret, _, _ = g.run(
self.mnode,
"gluster volume delete %s --mode=script" % self.volname)
self.assertEqual(ret, 0, "Failed to delete volume %s" % self.volname)
g.log.info("Volume deleted successfully %s", self.volname)
# Create another volume by using bricks of deleted volume
self.volname = "test_create_vol_used_bricks"
ret, _, err = volume_create(self.mnode,
self.volname,
brick_list[0:6],
replica_count=3)
self.assertNotEqual(
ret, 0, "Volume creation should fail with used "
"bricks but volume creation success")
g.log.info("Failed to create volume with used bricks")
# Checking failed message of volume creation
msg = ' '.join([
'volume create: test_create_vol_used_bricks: failed:',
brick_list[0].split(':')[1], 'is already part of a volume'
])
self.assertIn(
msg, err, "Incorrect error message for volume creation "
"with used bricks")
g.log.info("correct error message for volume creation with "
"used bricks")
def test_nfs_ganesha_export_with_multiple_volumes(self):
"""
Test case to verify multiple volumes gets exported when IO is in
progress.
"""
# Starting IO on the mounts
all_mounts_procs = []
count = 1
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s" %
(self.script_upload_path, count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count = count + 10
# Create and export five new volumes
for i in range(5):
# Check availability of bricks to create new volume
num_of_unused_bricks = 0
servers_unused_bricks_dict = get_servers_unused_bricks_dict(
self.mnode, self.all_servers, self.all_servers_info)
for each_server_unused_bricks_list in list(
servers_unused_bricks_dict.values()):
num_of_unused_bricks = (num_of_unused_bricks +
len(each_server_unused_bricks_list))
if num_of_unused_bricks < 2:
self.assertNotEqual(
i, 0, "New volume cannot be created due "
"to unavailability of bricks.")
g.log.warning(
"Tried to create five new volumes. But could "
"create only %s volume due to unavailability "
"of bricks.", str(i))
break
self.volume['name'] = "nfsvol" + str(i)
self.volume['voltype']['type'] = 'distributed'
self.volume['voltype']['replica_count'] = 1
self.volume['voltype']['dist_count'] = 2
new_vol = self.volume['name']
# Create volume
ret = setup_volume(mnode=self.mnode,
all_servers_info=self.all_servers_info,
volume_config=self.volume,
force=True)
if not ret:
self.assertTrue(ret, "Setup volume [%s] failed" % self.volume)
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, new_vol)
self.assertTrue(
ret, "Volume %s process not online despite "
"waiting for 300 seconds" % new_vol)
# Export volume with nfs ganesha
ret, _, _ = export_nfs_ganesha_volume(mnode=self.mnode,
volname=new_vol)
self.assertEqual(ret, 0, ("Failed to export volume %s "
"using nfs-ganesha" % new_vol))
# Wait for volume to get exported
ret = wait_for_nfs_ganesha_volume_to_get_exported(
self.mnode, new_vol)
self.assertTrue(
ret, "Volume %s is not exported after setting "
"ganesha.enable 'on'" % new_vol)
g.log.info("Exported nfs-ganesha volume %s", new_vol)
# Log Volume Info and Status
ret = log_volume_info_and_status(self.mnode, new_vol)
self.assertTrue(
ret, "Logging volume %s info and status failed" % new_vol)
# Validate IO
g.log.info("Validating IO")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all IO")
def test_volume_create(self):
# create and start a volume
self.volume['name'] = "first_volume"
self.volname = "first_volume"
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
# bring a brick down and volume start force should bring it to online
g.log.info("Get all the bricks of the volume")
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, "Failed to get the brick list")
g.log.info("Successfully got the list of bricks of volume")
ret = bring_bricks_offline(self.volname, bricks_list[0:2])
self.assertTrue(ret, "Failed to bring down the bricks")
g.log.info("Successfully brought the bricks down")
ret, _, _ = volume_start(self.mnode, self.volname, force=True)
self.assertEqual(ret, 0, "Failed to start the volume")
g.log.info("Volume start with force is success")
ret = wait_for_bricks_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, "Failed to bring the bricks online")
g.log.info("Volume start with force successfully brought all the "
"bricks online")
# create volume with previously used bricks and different volume name
self.volname = "second_volume"
ret, _, _ = volume_create(self.mnode, self.volname, bricks_list)
self.assertNotEqual(
ret, 0, "Expected: It should fail to create a "
"volume with previously used bricks. Actual:"
"Successfully created the volume with previously"
" used bricks")
g.log.info("Failed to create the volume with previously used bricks")
# create a volume with already existing volume name
self.volume['name'] = "first_volume"
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(
ret, "Expected: It should fail to create a volume"
" with already existing volume name. Actual: "
"Successfully created the volume with "
"already existing volname")
g.log.info("Failed to create the volume with already existing volname")
# creating a volume with non existing brick path should fail
self.volname = "second_volume"
bricks_list = form_bricks_list(self.mnode, self.volname,
len(self.servers), self.servers,
self.all_servers_info)
nonexisting_brick_index = random.randint(0, len(bricks_list) - 1)
non_existing_brick = bricks_list[nonexisting_brick_index].split(":")[0]
non_existing_path = ":/brick/non_existing_path"
non_existing_brick = non_existing_brick + non_existing_path
bricks_list[nonexisting_brick_index] = non_existing_brick
ret, _, _ = volume_create(self.mnode, self.volname, bricks_list)
self.assertNotEqual(
ret, 0, "Expected: Creating a volume with non "
"existing brick path should fail. Actual: "
"Successfully created the volume with "
"non existing brick path")
g.log.info("Failed to create the volume with non existing brick path")
# cleanup the volume and peer detach all servers. form two clusters,try
# to create a volume with bricks whose nodes are in different clusters
# cleanup volumes
vol_list = get_volume_list(self.mnode)
self.assertIsNotNone(vol_list, "Failed to get the volume list")
for volume in vol_list:
ret = cleanup_volume(self.mnode, volume)
self.assertTrue(ret, "Unable to delete volume % s" % volume)
# peer detach all servers
ret = peer_detach_servers(self.mnode, self.servers)
self.assertTrue(ret, "Peer detach to all servers is failed")
g.log.info("Peer detach to all the servers is success")
# form cluster 1
ret, _, _ = peer_probe(self.servers[0], self.servers[1])
self.assertEqual(
ret, 0, "Peer probe from %s to %s is failed" %
(self.servers[0], self.servers[1]))
g.log.info("Peer probe is success from %s to %s" %
(self.servers[0], self.servers[1]))
# form cluster 2
ret, _, _ = peer_probe(self.servers[2], self.servers[3])
self.assertEqual(
ret, 0, "Peer probe from %s to %s is failed" %
(self.servers[2], self.servers[3]))
g.log.info("Peer probe is success from %s to %s" %
(self.servers[2], self.servers[3]))
# Creating a volume with bricks which are part of another
# cluster should fail
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertFalse(
ret, "Expected: Creating a volume with bricks"
" which are part of another cluster should fail."
" Actual: Successfully created the volume with "
"bricks which are part of another cluster")
g.log.info("Failed to create the volume with bricks which are "
"part of another cluster")
# form a cluster, bring a node down. try to create a volume when one of
# the brick node is down
ret, _, _ = peer_detach(self.servers[2], self.servers[3])
self.assertEqual(ret, 0, "Peer detach is failed")
g.log.info("Peer detach is success")
ret = peer_probe_servers(self.mnode, self.servers)
self.assertTrue(ret, "Peer probe is failed")
g.log.info("Peer probe to all the servers is success")
random_server = self.servers[random.randint(1, len(self.servers) - 1)]
ret = stop_glusterd(random_server)
self.assertTrue(ret, "Glusterd is stopped successfully")
self.volume['name'] = "third_volume"
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertFalse(
ret, "Expected: It should fail to create a volume "
"when one of the node is down. Actual: Successfully "
"created the volume with bbrick whose node is down")
g.log.info("Failed to create the volume with brick whose node is down")
def test_add_brick_functionality(self):
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(
ret, ("Failed to create and start volume %s" % self.volname))
g.log.info("Volume created and started succssfully")
# form bricks list to test add brick functionality
replica_count_of_volume = self.volume['voltype']['replica_count']
num_of_bricks = 4 * replica_count_of_volume
bricks_list = form_bricks_list(self.mnode, self.volname, num_of_bricks,
self.servers, self.all_servers_info)
self.assertIsNotNone(bricks_list, "Bricks list is None")
# Try to add a single brick to volume, which should fail as it is a
# replicated volume, we should pass multiple of replica count number
# of bricks
bricks_list_to_add = [bricks_list[0]]
ret, out, err = add_brick(self.mnode, self.volname, bricks_list_to_add)
self.assertNotEqual(
ret, 0, "Expected: It should fail to add a single"
"brick to a replicated volume. Actual: "
"Successfully added single brick to volume")
g.log.info("failed to add a single brick to replicated volume")
# add brick replica count number of bricks in which one is
# non existing brick
kwargs = {}
kwargs['replica_count'] = replica_count_of_volume
bricks_list_to_add = bricks_list[1:replica_count_of_volume + 1]
num_of_bricks = len(bricks_list_to_add)
index_of_non_existing_brick = random.randint(0, num_of_bricks - 1)
complete_brick = bricks_list_to_add[index_of_non_existing_brick]
non_existing_brick = complete_brick + "/non_existing_brick"
bricks_list_to_add[index_of_non_existing_brick] = non_existing_brick
ret, out, err = add_brick(self.mnode, self.volname, bricks_list_to_add,
False, **kwargs)
self.assertNotEqual(
ret, 0, "Expected: It should fail to add non"
"existing brick to a volume. Actual: "
"Successfully added non existing brick to volume")
g.log.info("failed to add a non existing brick to volume")
# adding brick from node which is not part of cluster
bricks_list_to_add = bricks_list[replica_count_of_volume +
1:(2 * replica_count_of_volume) + 1]
num_of_bricks = len(bricks_list_to_add)
index_of_node = random.randint(0, num_of_bricks - 1)
complete_brick = bricks_list_to_add[index_of_node].split(":")
complete_brick[0] = "abc.def.ghi.jkl"
bricks_list_to_add[index_of_node] = ":".join(complete_brick)
ret, out, err = add_brick(self.mnode, self.volname, bricks_list_to_add,
False, **kwargs)
self.assertNotEqual(
ret, 0, "Expected: It should fail to add brick "
"from a node which is not part of a cluster."
"Actual:Successfully added bricks from node which"
" is not a part of cluster to volume")
g.log.info("Failed to add bricks form node which is not a part of "
"cluster to volume")
# add correct number of valid bricks, it should succeed
bricks_list_to_add = bricks_list[(2 * replica_count_of_volume) +
1:(3 * replica_count_of_volume) + 1]
ret, out, err = add_brick(self.mnode, self.volname, bricks_list_to_add,
False, **kwargs)
self.assertEqual(ret, 0, "Failed to add the bricks to the volume")
g.log.info("Successfully added bricks to volume")
# Perform rebalance start operation
ret, out, err = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Rebalance start is success")
def test_sync_functinality(self):
# create a 2x3 volume
num_of_servers = len(self.servers)
servers_info_from_cluster = {}
for server in self.servers[0:num_of_servers - 1]:
servers_info_from_cluster[server] = self.all_servers_info[server]
self.volume['servers'] = self.servers[0:num_of_servers - 1]
self.volume['voltype']['replica_count'] = 3
self.volume['voltype']['dist_count'] = 2
ret = setup_volume(self.mnode, servers_info_from_cluster, self.volume)
self.assertTrue(ret, ("Failed to create "
"and start volume %s" % self.volname))
g.log.info("Successfully created and started the volume %s",
self.volname)
# stop glusterd on a random node of the cluster
random_server_index = random.randint(1, num_of_servers - 2)
random_server = self.servers[random_server_index]
cmd = "systemctl stop glusterd"
ret = g.run_async(random_server, cmd)
g.log.info("Stopping glusterd on %s", random_server)
# set a option on volume, stat-prefetch on
self.options = {"stat-prefetch": "on"}
ret = set_volume_options(self.mnode, self.volname, self.options)
self.assertTrue(ret, ("Failed to set option stat-prefetch to on"
"for the volume %s" % self.volname))
g.log.info(
"Succeeded in setting stat-prefetch option to on"
"for the volume %s", self.volname)
# start glusterd on the node where glusterd is stopped
ret = start_glusterd(random_server)
self.assertTrue(ret, "Failed to start glusterd on %s" % random_server)
ret = wait_for_glusterd_to_start(random_server)
self.assertTrue(ret, "glusterd is not running on %s" % random_server)
g.log.info("glusterd is started and running on %s", random_server)
# volume info should be synced across the cluster
out1 = get_volume_info(self.mnode, self.volname)
self.assertIsNotNone(
out1, "Failed to get the volume info from %s" % self.mnode)
g.log.info("Getting volume info from %s is success", self.mnode)
count = 0
while count < 60:
out2 = get_volume_info(random_server, self.volname)
self.assertIsNotNone(
out2, "Failed to get the volume info from %s" % random_server)
if out1 == out2:
break
sleep(2)
count += 1
g.log.info("Getting volume info from %s is success", random_server)
self.assertDictEqual(out1, out2, "volume info is not synced")
# stop glusterd on a random server from cluster
random_server_index = random.randint(1, num_of_servers - 2)
random_server = self.servers[random_server_index]
cmd = "systemctl stop glusterd"
ret = g.run_async(random_server, cmd)
g.log.info("Stopping glusterd on node %s", random_server)
# peer probe a new node
ret = peer_probe_servers(self.mnode, self.servers[num_of_servers - 1])
self.assertTrue(
ret, "Failed to peer probe %s from %s" %
(self.servers[num_of_servers - 1], self.mnode))
g.log.info("Peer probe from %s to %s is success", self.mnode,
self.servers[num_of_servers - 1])
# start glusterd on the node where glusterd is stopped
ret = start_glusterd(random_server)
self.assertTrue(ret, "Failed to start glusterd on %s" % random_server)
ret = wait_for_glusterd_to_start(random_server)
self.assertTrue(ret, "glusterd is not running on %s" % random_server)
g.log.info("glusterd is started and running on %s", random_server)
# peer status should be synced across the cluster
list1 = nodes_from_pool_list(self.mnode)
self.assertIsNotNone(
list1, "Failed to get nodes list in the cluster"
"from %s" % self.mnode)
g.log.info("Successfully got the nodes list in the cluster from %s",
self.mnode)
# replacing ip with FQDN
i = 0
for node in list1:
list1[i] = socket.getfqdn(node)
i += 1
list1 = sorted(list1)
count = 0
while count < 60:
list2 = nodes_from_pool_list(random_server)
self.assertIsNotNone(
list2, "Failed to get nodes list in the "
"cluster from %s" % random_server)
# replacing ip with FQDN
i = 0
for node in list2:
list2[i] = socket.getfqdn(node)
i += 1
list2 = sorted(list2)
if list2 == list1:
break
sleep(2)
count += 1
g.log.info("Successfully got the nodes list in the cluster from %s",
random_server)
self.assertListEqual(list1, list2, "Peer status is "
"not synced across the cluster")
g.log.info("Peer status is synced across the cluster")
def setUpClass(cls):
"""
setup volume and initialize necessary variables
which is used in tests
"""
# calling GlusterBaseClass setUpClass
cls.get_super_method(cls, 'setUpClass')()
cls.default_volume_type_config = {
'replicated': {
'type': 'replicated',
'replica_count': 2,
'transport': 'tcp'
},
'dispersed': {
'type': 'dispersed',
'disperse_count': 6,
'redundancy_count': 2,
'transport': 'tcp'
},
'distributed': {
'type': 'distributed',
'dist_count': 2,
'transport': 'tcp'
},
'distributed-replicated': {
'type': 'distributed-replicated',
'dist_count': 2,
'replica_count': 3,
'transport': 'tcp'
}
}
# Setup Volume for all the volume types
cls.volume_configs = []
for volume_type in cls.default_volume_type_config:
cls.volume_configs.append({
'name':
'testvol_%s' % volume_type,
'servers':
cls.servers,
'voltype':
cls.default_volume_type_config[volume_type]
})
for volume_config in cls.volume_configs:
ret = setup_volume(mnode=cls.mnode,
all_servers_info=cls.all_servers_info,
volume_config=volume_config)
volname = volume_config['name']
if not ret:
raise ExecutionError("Failed to setup Volume" " %s" % volname)
g.log.info("Successful in setting volume %s", volname)
# Verify volume's all process are online for 60 sec
g.log.info("Verifying volume's all process are online")
ret = wait_for_volume_process_to_be_online(cls.mnode, volname, 60)
if not ret:
raise ExecutionError("Volume %s : All process are not online" %
volname)
g.log.info("Successfully Verified volume %s processes are online",
volname)
# Verfiy glustershd process releases its parent process
g.log.info("Verifying Self Heal Daemon process is daemonized")
ret = is_shd_daemonized(cls.servers)
if not ret:
raise ExecutionError("Self Heal Daemon process was still"
" holding parent process.")
g.log.info("Self Heal Daemon processes are online")
cls.GLUSTERSHD = "/var/lib/glusterd/glustershd/glustershd-server.vol"
def test_add_brick_when_quorum_not_met(self):
# pylint: disable=too-many-statements
# create and start a volume
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, ("Failed to create "
"and start volume %s" % self.volname))
g.log.info("Volume is created and started successfully")
# set cluster.server-quorum-type as server
ret = set_volume_options(self.mnode, self.volname,
{'cluster.server-quorum-type': 'server'})
self.assertTrue(ret, ("Failed to set the quorum type as a server"
" on volume %s", self.volname))
g.log.info("Able to set server quorum successfully on volume %s",
self.volname)
# Setting quorum ratio to 95%
ret = set_volume_options(self.mnode, 'all',
{'cluster.server-quorum-ratio': '95%'})
self.assertTrue(
ret, "Failed to set server quorum ratio on %s" % self.volname)
g.log.info("Able to set server quorum ratio successfully on %s",
self.servers)
# bring down glusterd of half nodes
num_of_servers = len(self.servers)
num_of_nodes_to_bring_down = num_of_servers / 2
for node in range(num_of_nodes_to_bring_down, num_of_servers):
ret = stop_glusterd(self.servers[node])
self.assertTrue(
ret, ("Failed to stop glusterd on %s" % self.servers[node]))
g.log.info("Glusterd stopped successfully on server %s",
self.servers[node])
for node in range(num_of_nodes_to_bring_down, num_of_servers):
count = 0
while count < 80:
ret = is_glusterd_running(self.servers[node])
if ret:
break
sleep(2)
count += 1
self.assertNotEqual(
ret, 0, "glusterd is still running on %s" % self.servers[node])
# Verifying node count in volume status after glusterd stopped
# on half of the servers, Its not possible to check the brick status
# immediately in volume status after glusterd stop
count = 0
while count < 100:
vol_status = get_volume_status(self.mnode, self.volname)
servers_count = len(vol_status[self.volname])
if servers_count == (num_of_servers - num_of_nodes_to_bring_down):
break
sleep(2)
count += 1
# confirm that quorum is not met, brick process should be down
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, "Failed to get the brick list")
bricks_to_check = bricks_list[0:num_of_nodes_to_bring_down]
ret = are_bricks_offline(self.mnode, self.volname, bricks_to_check)
self.assertTrue(
ret, "Unexpected: Server quorum is not met, "
"Bricks are up")
g.log.info("Server quorum is not met, bricks are down as expected")
# try add brick operation, which should fail
num_bricks_to_add = 1
brick = form_bricks_list(self.mnode, self.volname, num_bricks_to_add,
self.servers, self.all_servers_info)
ret, _, _ = add_brick(self.mnode, self.volname, brick)
self.assertNotEqual(ret, 0, ("Unexpected: add brick is success, "
"when quorum is not met"))
g.log.info("Add brick is failed as expected, when quorum is not met")
# confirm that, newly added brick is not part of volume
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, "Failed to get the brick list")
if brick in bricks_list:
ret = False
self.assertTrue(ret, ("Unexpected: add brick is success, "
"when quorum is not met"))
g.log.info("Add brick is failed as expected, when quorum is not met")
# set cluster.server-quorum-type as none
ret = set_volume_options(self.mnode, self.volname,
{'cluster.server-quorum-type': 'none'})
self.assertTrue(ret, ("Failed to set the quorum type as a server"
" on volume %s", self.volname))
g.log.info("Able to set server quorum successfully on volume %s",
self.volname)
def test_rebalance_status_from_newly_probed_node(self):
# Peer probe first 3 servers
servers_info_from_three_nodes = {}
for server in self.servers[0:3]:
servers_info_from_three_nodes[server] = self.all_servers_info[
server]
# Peer probe the first 3 servers
ret, _, _ = peer_probe(self.mnode, server)
self.assertEqual(ret, 0, "Peer probe failed to %s" % server)
self.volume['servers'] = self.servers[0:3]
# create a volume using the first 3 nodes
ret = setup_volume(self.mnode,
servers_info_from_three_nodes,
self.volume,
force=True)
self.assertTrue(
ret, "Failed to create"
"and start volume %s" % self.volname)
# Mounting a volume
ret = self.mount_volume(self.mounts)
self.assertTrue(ret, "Volume mount failed for %s" % self.volname)
# Checking volume mounted or not
ret = is_mounted(self.volname, self.mounts[0].mountpoint, self.mnode,
self.mounts[0].client_system, self.mount_type)
self.assertTrue(
ret, "Volume not mounted on mount point: %s" %
self.mounts[0].mountpoint)
g.log.info("Volume %s mounted on %s", self.volname,
self.mounts[0].mountpoint)
# run IOs
g.log.info("Starting IO on all mounts...")
self.counter = 1
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = (
"python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 10 "
"--dir-length 5 "
"--max-num-of-dirs 3 "
"--num-of-files 100 %s" %
(self.script_upload_path, self.counter, mount_obj.mountpoint))
ret = g.run(mount_obj.client_system, cmd)
self.assertEqual(ret, 0,
"IO failed on %s" % mount_obj.client_system)
self.counter = self.counter + 10
# add a brick to the volume and start rebalance
brick_to_add = form_bricks_list(self.mnode, self.volname, 1,
self.servers[0:3],
servers_info_from_three_nodes)
ret, _, _ = add_brick(self.mnode, self.volname, brick_to_add)
self.assertEqual(ret, 0, "Failed to add a brick to %s" % self.volname)
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to start rebalance")
# peer probe a new node from existing cluster
ret, _, _ = peer_probe(self.mnode, self.servers[3])
self.assertEqual(ret, 0, "Peer probe failed")
ret = get_rebalance_status(self.servers[3], self.volname)
self.assertIsNone(ret, "Failed to get rebalance status")